diff options
Diffstat (limited to 'src/plugins')
437 files changed, 35627 insertions, 28270 deletions
diff --git a/src/plugins/multimedia/CMakeLists.txt b/src/plugins/multimedia/CMakeLists.txt index 270a5ec6d..5bc39c1f8 100644 --- a/src/plugins/multimedia/CMakeLists.txt +++ b/src/plugins/multimedia/CMakeLists.txt @@ -1,18 +1,24 @@ -if (QT_FEATURE_ffmpeg) +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +if(QT_FEATURE_ffmpeg) add_subdirectory(ffmpeg) -endif () -if (QT_FEATURE_gstreamer) +endif() +if(QT_FEATURE_gstreamer) add_subdirectory(gstreamer) -endif () -if (ANDROID) +endif() +if(ANDROID) add_subdirectory(android) -endif () -if (APPLE AND NOT WATCHOS) +endif() +if(WASM) + add_subdirectory(wasm) +endif() +if(APPLE AND NOT WATCHOS) add_subdirectory(darwin) -endif () -if (QT_FEATURE_wmf) +endif() +if(QT_FEATURE_wmf) add_subdirectory(windows) -endif () -if (QT_FEATURE_mmrenderer) +endif() +if(QT_FEATURE_mmrenderer) add_subdirectory(qnx) -endif () +endif() diff --git a/src/plugins/multimedia/android/CMakeLists.txt b/src/plugins/multimedia/android/CMakeLists.txt index ae3678be2..31a94ff4f 100644 --- a/src/plugins/multimedia/android/CMakeLists.txt +++ b/src/plugins/multimedia/android/CMakeLists.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + qt_internal_add_plugin(QAndroidMediaPlugin OUTPUT_NAME androidmediaplugin PLUGIN_TYPE multimedia @@ -26,6 +29,12 @@ qt_internal_add_plugin(QAndroidMediaPlugin wrappers/jni/androidmultimediautils.cpp wrappers/jni/androidmultimediautils_p.h wrappers/jni/androidsurfacetexture.cpp wrappers/jni/androidsurfacetexture_p.h wrappers/jni/androidsurfaceview.cpp wrappers/jni/androidsurfaceview_p.h + NO_UNITY_BUILD_SOURCES + # Resolves two problems: + # - Collision of `rwLock` with wrappers/jni/androidmediaplayer.cpp + # - and redefinition of `notifyFrameAvailable` with different signature + # with wrappers/jni/androidsurfacetexture.cpp + wrappers/jni/androidcamera.cpp INCLUDE_DIRECTORIES audio common @@ -44,8 +53,7 @@ set_property(TARGET QAndroidMediaPlugin APPEND PROPERTY QT_ANDROID_BUNDLED_JAR_D jar/Qt${QtMultimedia_VERSION_MAJOR}AndroidMultimedia.jar:org.qtproject.qt.android.multimedia.QtMultimediaUtils ) set_property(TARGET QAndroidMediaPlugin APPEND PROPERTY QT_ANDROID_LIB_DEPENDENCIES - plugins/multimedia/libplugins_multimedia_androidmediaplugin.so - lib/libQt6MultimediaQuick.so:Qt6Quick + ${INSTALL_PLUGINSDIR}/multimedia/libplugins_multimedia_androidmediaplugin.so ) set_property(TARGET QAndroidMediaPlugin APPEND PROPERTY QT_ANDROID_PERMISSIONS android.permission.CAMERA android.permission.RECORD_AUDIO diff --git a/src/plugins/multimedia/android/audio/qandroidaudiodecoder.cpp b/src/plugins/multimedia/android/audio/qandroidaudiodecoder.cpp index 7239d7292..d200a72b5 100644 --- a/src/plugins/multimedia/android/audio/qandroidaudiodecoder.cpp +++ b/src/plugins/multimedia/android/audio/qandroidaudiodecoder.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidaudiodecoder_p.h" #include <QtCore/qcoreapplication.h> @@ -55,7 +19,7 @@ QT_BEGIN_NAMESPACE static const char tempFile[] = "encoded.wav"; constexpr int dequeueTimeout = 5000; -Q_LOGGING_CATEGORY(adLogger, "QAndroidAudioDecoder") +static Q_LOGGING_CATEGORY(adLogger, "QAndroidAudioDecoder") Decoder::Decoder() : m_format(AMediaFormat_new()) @@ -95,37 +59,32 @@ void Decoder::setSource(const QUrl &source) "org/qtproject/qt/android/multimedia/QtMultimediaUtils", "getMimeType", "(Landroid/content/Context;Ljava/lang/String;)Ljava/lang/String;", - QNativeInterface::QAndroidApplication::context(), + QNativeInterface::QAndroidApplication::context().object(), QJniObject::fromString(source.path()).object()); const QString mime = path.isValid() ? path.toString() : ""; if (!mime.isEmpty() && !mime.contains("audio", Qt::CaseInsensitive)) { - emit error(QAudioDecoder::FormatError, - tr("Cannot set source, invalid mime type for the provided source.")); + m_formatError = tr("Cannot set source, invalid mime type for the source provided."); return; } if (!m_extractor) m_extractor = AMediaExtractor_new(); - int fd = -1; - if (source.path().contains(QLatin1String("content"))) { - fd = QJniObject::callStaticMethod<jint>("org/qtproject/qt/android/QtNative", - "openFdForContentUrl", - "(Landroid/content/Context;Ljava/lang/String;Ljava/lang/String;)I", - QNativeInterface::QAndroidApplication::context(), - QJniObject::fromString(source.path()).object(), - QJniObject::fromString(QLatin1String("r")).object()); - } else { - fd = open(source.path().toStdString().c_str(), O_RDONLY); - } + QFile file(source.path()); + if (!file.open(QFile::ReadOnly)) { + emit error(QAudioDecoder::ResourceError, tr("Cannot open the file")); + return; + } + + const int fd = file.handle(); if (fd < 0) { - emit error(QAudioDecoder::ResourceError, tr("Invalid fileDescriptor for source.")); - return; - } - const int size = QFile(source.toString()).size(); + emit error(QAudioDecoder::ResourceError, tr("Invalid fileDescriptor for source.")); + return; + } + const int size = file.size(); media_status_t status = AMediaExtractor_setDataSourceFd(m_extractor, fd, 0, size > 0 ? size : LONG_MAX); close(fd); @@ -135,7 +94,7 @@ void Decoder::setSource(const QUrl &source) AMediaExtractor_delete(m_extractor); m_extractor = nullptr; } - emit error(QAudioDecoder::ResourceError, tr("Setting source for Audio Decoder failed.")); + m_formatError = tr("Setting source for Audio Decoder failed."); } } @@ -176,6 +135,11 @@ void Decoder::createDecoder() void Decoder::doDecode() { + if (!m_formatError.isEmpty()) { + emit error(QAudioDecoder::FormatError, m_formatError); + return; + } + if (!m_extractor) { emit error(QAudioDecoder::ResourceError, tr("Cannot decode, source not set.")); return; @@ -326,6 +290,12 @@ void QAndroidAudioDecoder::start() m_position = -1; + if (m_device && (!m_device->isOpen() || !m_device->isReadable())) { + emit error(QAudioDecoder::ResourceError, + QString::fromUtf8("Unable to read from the specified device")); + return; + } + if (!m_threadDecoder) { m_threadDecoder = new QThread(this); m_decoder->moveToThread(m_threadDecoder); @@ -435,11 +405,11 @@ bool QAndroidAudioDecoder::createTempFile() bool success = file.open(QIODevice::QIODevice::ReadWrite); if (!success) - emit error(QAudioDecoder::ResourceError, tr("Error while opening tmp file")); + emit error(QAudioDecoder::ResourceError, tr("Error opening temporary file: %1").arg(file.errorString())); success &= (file.write(m_deviceBuffer) == m_deviceBuffer.size()); if (!success) - emit error(QAudioDecoder::ResourceError, tr("Error while writing data to tmp file")); + emit error(QAudioDecoder::ResourceError, tr("Error while writing data to temporary file")); file.close(); m_deviceBuffer.clear(); @@ -463,3 +433,5 @@ void QAndroidAudioDecoder::readDevice() { } QT_END_NAMESPACE + +#include "moc_qandroidaudiodecoder_p.cpp" diff --git a/src/plugins/multimedia/android/audio/qandroidaudiodecoder_p.h b/src/plugins/multimedia/android/audio/qandroidaudiodecoder_p.h index a1795e39b..65a0f1855 100644 --- a/src/plugins/multimedia/android/audio/qandroidaudiodecoder_p.h +++ b/src/plugins/multimedia/android/audio/qandroidaudiodecoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDAUDIODECODER_P_H #define QANDROIDAUDIODECODER_P_H @@ -90,6 +54,7 @@ private: AMediaFormat *m_format = nullptr; QAudioFormat m_outputFormat; + QString m_formatError; bool m_inputEOS; }; @@ -142,7 +107,6 @@ private: qint64 m_position = -1; qint64 m_duration = -1; - long long m_presentationTimeUs = 0; QByteArray m_deviceBuffer; diff --git a/src/plugins/multimedia/android/common/qandroidaudioinput.cpp b/src/plugins/multimedia/android/common/qandroidaudioinput.cpp index 71be7869d..a1eb9258b 100644 --- a/src/plugins/multimedia/android/common/qandroidaudioinput.cpp +++ b/src/plugins/multimedia/android/common/qandroidaudioinput.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidaudioinput_p.h" @@ -79,3 +43,5 @@ bool QAndroidAudioInput::isMuted() const } QT_END_NAMESPACE + +#include "moc_qandroidaudioinput_p.cpp" diff --git a/src/plugins/multimedia/android/common/qandroidaudioinput_p.h b/src/plugins/multimedia/android/common/qandroidaudioinput_p.h index 100b13aab..ef59da8ec 100644 --- a/src/plugins/multimedia/android/common/qandroidaudioinput_p.h +++ b/src/plugins/multimedia/android/common/qandroidaudioinput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDAUDIOINPUT_H #define QANDROIDAUDIOINPUT_H diff --git a/src/plugins/multimedia/android/common/qandroidaudiooutput_p.h b/src/plugins/multimedia/android/common/qandroidaudiooutput_p.h index e17f158fc..d5d25b458 100644 --- a/src/plugins/multimedia/android/common/qandroidaudiooutput_p.h +++ b/src/plugins/multimedia/android/common/qandroidaudiooutput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDAUDIOOUTPUT_H #define QANDROIDAUDIOOUTPUT_H diff --git a/src/plugins/multimedia/android/common/qandroidglobal_p.h b/src/plugins/multimedia/android/common/qandroidglobal_p.h index 45bd22ffb..1022fa061 100644 --- a/src/plugins/multimedia/android/common/qandroidglobal_p.h +++ b/src/plugins/multimedia/android/common/qandroidglobal_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDGLOBAL_H #define QANDROIDGLOBAL_H diff --git a/src/plugins/multimedia/android/common/qandroidmultimediautils.cpp b/src/plugins/multimedia/android/common/qandroidmultimediautils.cpp index e333f9520..6e4b95fe9 100644 --- a/src/plugins/multimedia/android/common/qandroidmultimediautils.cpp +++ b/src/plugins/multimedia/android/common/qandroidmultimediautils.cpp @@ -1,47 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidmultimediautils_p.h" #include "qandroidglobal_p.h" #include <qlist.h> #include <QtCore/qcoreapplication.h> +#include <QtCore/qpermissions.h> #include <QtCore/private/qandroidextras_p.h> QT_BEGIN_NAMESPACE @@ -124,43 +89,27 @@ static bool androidRequestPermission(const QString &permission) return true; } -static bool androidCheckPermission(const QString &permission) +static bool androidCheckPermission(const QPermission &permission) { - if (QNativeInterface::QAndroidApplication::sdkVersion() < 23) - return true; - - // Permission already granted? - return (QtAndroidPrivate::checkPermission(permission).result() == QtAndroidPrivate::Authorized); + return qApp->checkPermission(permission) == Qt::PermissionStatus::Granted; } bool qt_androidCheckCameraPermission() { - return androidCheckPermission(QStringLiteral("android.permission.CAMERA")); + const QCameraPermission permission; + const auto granted = androidCheckPermission(permission); + if (!granted) + qCDebug(qtAndroidMediaPlugin, "Camera permission not granted!"); + return granted; } bool qt_androidCheckMicrophonePermission() { - return androidCheckPermission(QStringLiteral("android.permission.RECORD_AUDIO")); -} - -bool qt_androidRequestCameraPermission() -{ - if (!androidRequestPermission(QStringLiteral("android.permission.CAMERA"))) { - qCDebug(qtAndroidMediaPlugin, "Camera permission denied by user!"); - return false; - } - - return true; -} - -bool qt_androidRequestRecordingPermission() -{ - if (!androidRequestPermission(QStringLiteral("android.permission.RECORD_AUDIO"))) { - qCDebug(qtAndroidMediaPlugin, "Microphone permission denied by user!"); - return false; - } - - return true; + const QMicrophonePermission permission; + const auto granted = androidCheckPermission(permission); + if (!granted) + qCDebug(qtAndroidMediaPlugin, "Microphone permission not granted!"); + return granted; } bool qt_androidRequestWriteStoragePermission() diff --git a/src/plugins/multimedia/android/common/qandroidmultimediautils_p.h b/src/plugins/multimedia/android/common/qandroidmultimediautils_p.h index 5bc187da3..5fe841e8c 100644 --- a/src/plugins/multimedia/android/common/qandroidmultimediautils_p.h +++ b/src/plugins/multimedia/android/common/qandroidmultimediautils_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDMULTIMEDIAUTILS_H #define QANDROIDMULTIMEDIAUTILS_H @@ -66,8 +30,6 @@ bool qt_sizeLessThan(const QSize &s1, const QSize &s2); QVideoFrameFormat::PixelFormat qt_pixelFormatFromAndroidImageFormat(AndroidCamera::ImageFormat f); AndroidCamera::ImageFormat qt_androidImageFormatFromPixelFormat(QVideoFrameFormat::PixelFormat f); -bool qt_androidRequestCameraPermission(); -bool qt_androidRequestRecordingPermission(); bool qt_androidRequestWriteStoragePermission(); bool qt_androidCheckCameraPermission(); diff --git a/src/plugins/multimedia/android/common/qandroidvideooutput.cpp b/src/plugins/multimedia/android/common/qandroidvideooutput.cpp index a3a4966cc..0724a8359 100644 --- a/src/plugins/multimedia/android/common/qandroidvideooutput.cpp +++ b/src/plugins/multimedia/android/common/qandroidvideooutput.cpp @@ -1,112 +1,256 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidvideooutput_p.h" - #include "androidsurfacetexture_p.h" + +#include <rhi/qrhi.h> +#include <QtGui/private/qopenglextensions_p.h> +#include <private/qhwvideobuffer_p.h> +#include <private/qvideoframeconverter_p.h> +#include <private/qplatformvideosink_p.h> +#include <private/qvideoframe_p.h> #include <qvideosink.h> -#include "private/qabstractvideobuffer_p.h" -#include "private/qplatformvideosink_p.h" -#include <QVideoFrameFormat> -#include <QFile> -#include <QtGui/private/qrhigles2_p.h> -#include <QOpenGLContext> -#include <QPainter> -#include <QPainterPath> -#include <QMutexLocker> -#include <QTextLayout> -#include <QTextFormat> +#include <qopenglcontext.h> +#include <qopenglfunctions.h> +#include <qvideoframeformat.h> +#include <qthread.h> +#include <qfile.h> QT_BEGIN_NAMESPACE -void GraphicsResourceDeleter::deleteResourcesHelper(const QList<QRhiResource *> &res) +class QAndroidVideoFrameTextures : public QVideoFrameTextures { - qDeleteAll(res); -} +public: + QAndroidVideoFrameTextures(QRhi *rhi, QSize size, quint64 handle) + { + m_tex.reset(rhi->newTexture(QRhiTexture::RGBA8, size, 1)); + m_tex->createFrom({quint64(handle), 0}); + } -void GraphicsResourceDeleter::deleteRhiHelper(QRhi *rhi, QOffscreenSurface *surf) -{ - delete rhi; - delete surf; -} + QRhiTexture *texture(uint plane) const override + { + return plane == 0 ? m_tex.get() : nullptr; + } + +private: + std::unique_ptr<QRhiTexture> m_tex; +}; -void GraphicsResourceDeleter::deleteThisHelper() +// QRhiWithThreadGuard keeps QRhi and QThread (that created it) alive to allow proper cleaning +class QRhiWithThreadGuard : public QObject { + Q_OBJECT +public: + QRhiWithThreadGuard(std::shared_ptr<QRhi> r, std::shared_ptr<AndroidTextureThread> t) + : m_guardRhi(std::move(r)), m_thread(std::move(t)) {} + ~QRhiWithThreadGuard(); +protected: + std::shared_ptr<QRhi> m_guardRhi; +private: + std::shared_ptr<AndroidTextureThread> m_thread; +}; + +class AndroidTextureVideoBuffer : public QRhiWithThreadGuard, public QHwVideoBuffer { - delete this; -} +public: + AndroidTextureVideoBuffer(std::shared_ptr<QRhi> rhi, + std::shared_ptr<AndroidTextureThread> thread, + std::unique_ptr<QRhiTexture> tex, const QSize &size) + : QRhiWithThreadGuard(std::move(rhi), std::move(thread)), + QHwVideoBuffer(QVideoFrame::RhiTextureHandle, m_guardRhi.get()), + m_size(size), + m_tex(std::move(tex)) + {} + + MapData map(QtVideo::MapMode mode) override; + + void unmap() override + { + m_image = {}; + m_mapMode = QtVideo::MapMode::NotMapped; + } + + std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *rhi) override + { + return std::make_unique<QAndroidVideoFrameTextures>(rhi, m_size, m_tex->nativeTexture().object); + } -bool AndroidTextureVideoBuffer::updateReadbackFrame() +private: + QSize m_size; + std::unique_ptr<QRhiTexture> m_tex; + QImage m_image; + QtVideo::MapMode m_mapMode = QtVideo::MapMode::NotMapped; +}; + +class ImageFromVideoFrameHelper : public QHwVideoBuffer { - // Even though the texture was updated in a previous call, we need to re-check - // that this has not become a stale buffer, e.g., if the output size changed or - // has since became invalid. - if (!m_output->m_nativeSize.isValid()) - return false; - - // Size changed - if (m_output->m_nativeSize != m_size) - return false; - - // In the unlikely event that we don't have a valid fbo, but have a valid size, - // force an update. - const bool forceUpdate = !m_output->m_readbackTex; - if (m_textureUpdated && !forceUpdate) - return true; - - // update the video texture (called from the render thread) - return (m_textureUpdated = m_output->renderAndReadbackFrame()); -} +public: + ImageFromVideoFrameHelper(AndroidTextureVideoBuffer &atvb) + : QHwVideoBuffer(QVideoFrame::RhiTextureHandle, atvb.rhi()), m_atvb(atvb) + {} + std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *rhi) override + { + return m_atvb.mapTextures(rhi); + } + + MapData map(QtVideo::MapMode) override { return {}; } + void unmap() override {} -QAbstractVideoBuffer::MapData AndroidTextureVideoBuffer::map(QVideoFrame::MapMode mode) +private: + AndroidTextureVideoBuffer &m_atvb; +}; + +QAbstractVideoBuffer::MapData AndroidTextureVideoBuffer::map(QtVideo::MapMode mode) { - MapData mapData; - if (m_mapMode == QVideoFrame::NotMapped && mode == QVideoFrame::ReadOnly && updateReadbackFrame()) { - m_mapMode = mode; - m_image = m_output->m_readbackImage; - mapData.nPlanes = 1; + QAbstractVideoBuffer::MapData mapData; + + if (m_mapMode == QtVideo::MapMode::NotMapped && mode == QtVideo::MapMode::ReadOnly) { + m_mapMode = QtVideo::MapMode::ReadOnly; + m_image = qImageFromVideoFrame(QVideoFramePrivate::createFrame( + std::make_unique<ImageFromVideoFrameHelper>(*this), + QVideoFrameFormat(m_size, QVideoFrameFormat::Format_RGBA8888))); + mapData.planeCount = 1; mapData.bytesPerLine[0] = m_image.bytesPerLine(); - mapData.size[0] = static_cast<int>(m_image.sizeInBytes()); + mapData.dataSize[0] = static_cast<int>(m_image.sizeInBytes()); mapData.data[0] = m_image.bits(); } + return mapData; } +static const float g_quad[] = { + -1.f, -1.f, 0.f, 0.f, + -1.f, 1.f, 0.f, 1.f, + 1.f, 1.f, 1.f, 1.f, + 1.f, -1.f, 1.f, 0.f +}; + +class TextureCopy +{ + static QShader getShader(const QString &name) + { + QFile f(name); + if (f.open(QIODevice::ReadOnly)) + return QShader::fromSerialized(f.readAll()); + return {}; + } + +public: + TextureCopy(QRhi *rhi, QRhiTexture *externalTex) + : m_rhi(rhi) + { + m_vertexBuffer.reset(m_rhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_quad))); + m_vertexBuffer->create(); + + m_uniformBuffer.reset(m_rhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, 64 + 64 + 4 + 4)); + m_uniformBuffer->create(); + + m_sampler.reset(m_rhi->newSampler(QRhiSampler::Nearest, QRhiSampler::Nearest, QRhiSampler::None, + QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge)); + m_sampler->create(); + + m_srb.reset(m_rhi->newShaderResourceBindings()); + m_srb->setBindings({ + QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage, m_uniformBuffer.get()), + QRhiShaderResourceBinding::sampledTexture(1, QRhiShaderResourceBinding::FragmentStage, externalTex, m_sampler.get()) + }); + m_srb->create(); + + m_vertexShader = getShader(QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.vert.qsb")); + Q_ASSERT(m_vertexShader.isValid()); + m_fragmentShader = getShader(QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.frag.qsb")); + Q_ASSERT(m_fragmentShader.isValid()); + } + + std::unique_ptr<QRhiTexture> copyExternalTexture(QSize size, const QMatrix4x4 &externalTexMatrix); + +private: + QRhi *m_rhi = nullptr; + std::unique_ptr<QRhiBuffer> m_vertexBuffer; + std::unique_ptr<QRhiBuffer> m_uniformBuffer; + std::unique_ptr<QRhiSampler> m_sampler; + std::unique_ptr<QRhiShaderResourceBindings> m_srb; + QShader m_vertexShader; + QShader m_fragmentShader; +}; + +static std::unique_ptr<QRhiGraphicsPipeline> newGraphicsPipeline(QRhi *rhi, + QRhiShaderResourceBindings *shaderResourceBindings, + QRhiRenderPassDescriptor *renderPassDescriptor, + QShader vertexShader, + QShader fragmentShader) +{ + std::unique_ptr<QRhiGraphicsPipeline> gp(rhi->newGraphicsPipeline()); + gp->setTopology(QRhiGraphicsPipeline::TriangleFan); + gp->setShaderStages({ + { QRhiShaderStage::Vertex, vertexShader }, + { QRhiShaderStage::Fragment, fragmentShader } + }); + QRhiVertexInputLayout inputLayout; + inputLayout.setBindings({ + { 4 * sizeof(float) } + }); + inputLayout.setAttributes({ + { 0, 0, QRhiVertexInputAttribute::Float2, 0 }, + { 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) } + }); + gp->setVertexInputLayout(inputLayout); + gp->setShaderResourceBindings(shaderResourceBindings); + gp->setRenderPassDescriptor(renderPassDescriptor); + gp->create(); + + return gp; +} + +std::unique_ptr<QRhiTexture> TextureCopy::copyExternalTexture(QSize size, const QMatrix4x4 &externalTexMatrix) +{ + std::unique_ptr<QRhiTexture> tex(m_rhi->newTexture(QRhiTexture::RGBA8, size, 1, QRhiTexture::RenderTarget)); + if (!tex->create()) { + qWarning("Failed to create frame texture"); + return {}; + } + + std::unique_ptr<QRhiTextureRenderTarget> renderTarget(m_rhi->newTextureRenderTarget({ { tex.get() } })); + std::unique_ptr<QRhiRenderPassDescriptor> renderPassDescriptor(renderTarget->newCompatibleRenderPassDescriptor()); + renderTarget->setRenderPassDescriptor(renderPassDescriptor.get()); + renderTarget->create(); + + QRhiResourceUpdateBatch *rub = m_rhi->nextResourceUpdateBatch(); + rub->uploadStaticBuffer(m_vertexBuffer.get(), g_quad); + + QMatrix4x4 identity; + char *p = m_uniformBuffer->beginFullDynamicBufferUpdateForCurrentFrame(); + memcpy(p, identity.constData(), 64); + memcpy(p + 64, externalTexMatrix.constData(), 64); + float opacity = 1.0f; + memcpy(p + 64 + 64, &opacity, 4); + m_uniformBuffer->endFullDynamicBufferUpdateForCurrentFrame(); + + auto graphicsPipeline = newGraphicsPipeline(m_rhi, m_srb.get(), renderPassDescriptor.get(), + m_vertexShader, m_fragmentShader); + + const QRhiCommandBuffer::VertexInput vbufBinding(m_vertexBuffer.get(), 0); + + QRhiCommandBuffer *cb = nullptr; + if (m_rhi->beginOffscreenFrame(&cb) != QRhi::FrameOpSuccess) + return {}; + + cb->beginPass(renderTarget.get(), Qt::transparent, { 1.0f, 0 }, rub); + cb->setGraphicsPipeline(graphicsPipeline.get()); + cb->setViewport({0, 0, float(size.width()), float(size.height())}); + cb->setShaderResources(m_srb.get()); + cb->setVertexInput(0, 1, &vbufBinding); + cb->draw(4); + cb->endPass(); + m_rhi->endOffscreenFrame(); + + QOpenGLContext *ctx = QOpenGLContext::currentContext(); + QOpenGLFunctions *f = ctx->functions(); + static_cast<QOpenGLExtensions *>(f)->flushShared(); + + return tex; +} + static QMatrix4x4 extTransformMatrix(AndroidSurfaceTexture *surfaceTexture) { QMatrix4x4 m = surfaceTexture->getTransformMatrix(); @@ -121,419 +265,204 @@ static QMatrix4x4 extTransformMatrix(AndroidSurfaceTexture *surfaceTexture) return m; } -quint64 AndroidTextureVideoBuffer::textureHandle(int plane) const +class AndroidTextureThread : public QThread { - if (plane != 0 || !m_rhi || !m_output->m_nativeSize.isValid()) - return 0; - - m_output->ensureExternalTexture(m_rhi); - m_output->m_surfaceTexture->updateTexImage(); - m_externalMatrix = extTransformMatrix(m_output->m_surfaceTexture); - return m_output->m_externalTex->nativeTexture().object; -} + Q_OBJECT +public: + AndroidTextureThread(QAndroidTextureVideoOutput * vo) + : QThread() + , m_videoOutput(vo) + { + } -QAndroidTextureVideoOutput::QAndroidTextureVideoOutput(QObject *parent) : QAndroidVideoOutput(parent) { } + ~AndroidTextureThread() { + QMetaObject::invokeMethod(this, + &AndroidTextureThread::clearSurfaceTexture, Qt::BlockingQueuedConnection); + this->quit(); + this->wait(); + } -QAndroidTextureVideoOutput::~QAndroidTextureVideoOutput() -{ - clearSurfaceTexture(); - - if (m_graphicsDeleter) { // Make sure all of these are deleted on the render thread. - m_graphicsDeleter->deleteResources({ - m_externalTex, - m_readbackSrc, - m_readbackTex, - m_readbackVBuf, - m_readbackUBuf, - m_externalTexSampler, - m_readbackSrb, - m_readbackRenderTarget, - m_readbackRpDesc, - m_readbackPs - }); + void start() + { + QThread::start(); + moveToThread(this); + } - m_graphicsDeleter->deleteRhi(m_readbackRhi, m_readbackRhiFallbackSurface); - m_graphicsDeleter->deleteThis(); + void initRhi(QOpenGLContext *context) + { + QRhiGles2InitParams params; + params.shareContext = context; + params.fallbackSurface = QRhiGles2InitParams::newFallbackSurface(); + m_rhi.reset(QRhi::create(QRhi::OpenGLES2, ¶ms)); } -} -void QAndroidTextureVideoOutput::setSubtitle(const QString &subtitle) -{ - if (!m_sink) - return; - auto *sink = m_sink->platformVideoSink(); - sink->setSubtitleText(subtitle); -} +public slots: + void onFrameAvailable(quint64 index) + { + // Check if 'm_surfaceTexture' is not reset and if the current index is the same that + // was used for creating connection because there can be pending frames in queue. + if (m_surfaceTexture && m_surfaceTexture->index() == index) { + m_surfaceTexture->updateTexImage(); + auto matrix = extTransformMatrix(m_surfaceTexture.get()); + auto tex = m_textureCopy->copyExternalTexture(m_size, matrix); + auto *buffer = new AndroidTextureVideoBuffer(m_rhi, m_videoOutput->getSurfaceThread(), std::move(tex), m_size); + QVideoFrame frame(buffer, QVideoFrameFormat(m_size, QVideoFrameFormat::Format_RGBA8888)); + emit newFrame(frame); + } + } -QVideoSink *QAndroidTextureVideoOutput::surface() const -{ - return m_sink; -} + void clearFrame() { emit newFrame({}); } -void QAndroidTextureVideoOutput::setSurface(QVideoSink *surface) -{ - if (surface == m_sink) - return; + void setFrameSize(QSize size) { m_size = size; } - m_sink = surface; -} + void clearSurfaceTexture() + { + m_surfaceTexture.reset(); + m_texture.reset(); + m_textureCopy.reset(); + m_rhi.reset(); + } -bool QAndroidTextureVideoOutput::isReady() -{ - return true; -} + AndroidSurfaceTexture *createSurfaceTexture(QRhi *rhi) + { + if (m_surfaceTexture) + return m_surfaceTexture.get(); -void QAndroidTextureVideoOutput::initSurfaceTexture() -{ - if (m_surfaceTexture) - return; + QOpenGLContext *ctx = rhi + ? static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles())->context + : nullptr; + initRhi(ctx); - if (!m_sink) - return; + m_texture.reset(m_rhi->newTexture(QRhiTexture::RGBA8, m_size, 1, QRhiTexture::ExternalOES)); + m_texture->create(); + m_surfaceTexture = std::make_unique<AndroidSurfaceTexture>(m_texture->nativeTexture().object); + if (m_surfaceTexture->surfaceTexture()) { + const quint64 index = m_surfaceTexture->index(); + connect(m_surfaceTexture.get(), &AndroidSurfaceTexture::frameAvailable, this, + [this, index] () { this->onFrameAvailable(index); }); - QMutexLocker locker(&m_mutex); + m_textureCopy = std::make_unique<TextureCopy>(m_rhi.get(), m_texture.get()); - m_surfaceTexture = new AndroidSurfaceTexture(m_externalTex ? m_externalTex->nativeTexture().object : 0); + } else { + m_texture.reset(); + m_surfaceTexture.reset(); + } - if (m_surfaceTexture->surfaceTexture() != 0) { - connect(m_surfaceTexture, &AndroidSurfaceTexture::frameAvailable, - this, &QAndroidTextureVideoOutput::onFrameAvailable); - } else { - delete m_surfaceTexture; - m_surfaceTexture = nullptr; - if (m_graphicsDeleter) - m_graphicsDeleter->deleteResources({ m_externalTex }); - m_externalTex = nullptr; + return m_surfaceTexture.get(); } -} -void QAndroidTextureVideoOutput::clearSurfaceTexture() -{ - QMutexLocker locker(&m_mutex); - if (m_surfaceTexture) { - delete m_surfaceTexture; - m_surfaceTexture = nullptr; - } +signals: + void newFrame(const QVideoFrame &); - // Also reset the attached texture - if (m_graphicsDeleter) - m_graphicsDeleter->deleteResources({ m_externalTex }); - m_externalTex = nullptr; -} +private: + QAndroidTextureVideoOutput * m_videoOutput; + std::shared_ptr<QRhi> m_rhi; + std::unique_ptr<AndroidSurfaceTexture> m_surfaceTexture; + std::unique_ptr<QRhiTexture> m_texture; + std::unique_ptr<TextureCopy> m_textureCopy; + QSize m_size; +}; -AndroidSurfaceTexture *QAndroidTextureVideoOutput::surfaceTexture() -{ - initSurfaceTexture(); - return m_surfaceTexture; +QRhiWithThreadGuard::~QRhiWithThreadGuard() { + // It may happen that reseting m_rhi shared_ptr will delete it (if it is the last reference) + // QRHI need to be deleted from the thread that created it. + QMetaObject::invokeMethod(m_thread.get(), [&]() {m_guardRhi.reset();}, Qt::BlockingQueuedConnection); } -void QAndroidTextureVideoOutput::setVideoSize(const QSize &size) +QAndroidTextureVideoOutput::QAndroidTextureVideoOutput(QVideoSink *sink, QObject *parent) + : QAndroidVideoOutput(parent) + , m_sink(sink) { - QMutexLocker locker(&m_mutex); - if (m_nativeSize == size) + if (!m_sink) { + qDebug() << "Cannot create QAndroidTextureVideoOutput without a sink."; + m_surfaceThread = nullptr; return; + } - stop(); - - m_nativeSize = size; + startNewSurfaceThread(); } -void QAndroidTextureVideoOutput::start() +void QAndroidTextureVideoOutput::startNewSurfaceThread() { - m_started = true; - renderAndReadbackFrame(); + m_surfaceThread = std::make_shared<AndroidTextureThread>(this); + connect(m_surfaceThread.get(), &AndroidTextureThread::newFrame, + this, &QAndroidTextureVideoOutput::newFrame); + m_surfaceThread->start(); } -void QAndroidTextureVideoOutput::stop() +QAndroidTextureVideoOutput::~QAndroidTextureVideoOutput() { - m_nativeSize = QSize(); - m_started = false; + // Make sure that no more VideFrames will be created by surfaceThread + QMetaObject::invokeMethod(m_surfaceThread.get(), + &AndroidTextureThread::clearSurfaceTexture, Qt::BlockingQueuedConnection); } -void QAndroidTextureVideoOutput::renderFrame() +void QAndroidTextureVideoOutput::setSubtitle(const QString &subtitle) { - if (!m_started) { - m_renderFrame = true; - bool frameok = renderAndReadbackFrame(); - if (!frameok) { - m_renderFrame = true; - renderAndReadbackFrame(); - } + if (m_sink) { + auto *sink = m_sink->platformVideoSink(); + if (sink) + sink->setSubtitleText(subtitle); } } -void QAndroidTextureVideoOutput::reset() +bool QAndroidTextureVideoOutput::shouldTextureBeUpdated() const { - // flush pending frame - if (m_sink) - m_sink->platformVideoSink()->setVideoFrame(QVideoFrame()); - - clearSurfaceTexture(); + return m_sink->rhi() && m_surfaceCreatedWithoutRhi; } -bool QAndroidTextureVideoOutput::moveToOpenGLContextThread() +AndroidSurfaceTexture *QAndroidTextureVideoOutput::surfaceTexture() { if (!m_sink) - return false; - - const auto rhi = m_sink->rhi(); - if (!rhi || rhi->backend() != QRhi::OpenGLES2) - return false; - - const auto nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles()); - if (!nativeHandles) - return false; - - const auto context = nativeHandles->context; - if (!context) - return false; - - // check if QAndroidTextureVideoOutput is already in the OpenGL context thread - if (QThread::currentThread() == context->thread()) - return false; - - // move to the OpenGL context thread; - parent()->moveToThread(context->thread()); - moveToThread(context->thread()); - - return true; + return nullptr; + + AndroidSurfaceTexture *surface = nullptr; + QMetaObject::invokeMethod(m_surfaceThread.get(), [&]() { + auto rhi = m_sink->rhi(); + if (!rhi) { + m_surfaceCreatedWithoutRhi = true; + } + else if (m_surfaceCreatedWithoutRhi) { + m_surfaceThread->clearSurfaceTexture(); + m_surfaceCreatedWithoutRhi = false; + } + surface = m_surfaceThread->createSurfaceTexture(rhi); + }, + Qt::BlockingQueuedConnection); + return surface; } -void QAndroidTextureVideoOutput::onFrameAvailable() +void QAndroidTextureVideoOutput::setVideoSize(const QSize &size) { - if (!(m_nativeSize.isValid() && m_sink) || !(m_started || m_renderFrame)) + if (m_nativeSize == size) return; - m_renderFrame = false; - QRhi *rhi = m_sink ? m_sink->rhi() : nullptr; - - auto *buffer = new AndroidTextureVideoBuffer(rhi, this, m_nativeSize); - const QVideoFrameFormat::PixelFormat format = rhi ? QVideoFrameFormat::Format_SamplerExternalOES - : QVideoFrameFormat::Format_RGBA8888; - QVideoFrame frame(buffer, QVideoFrameFormat(m_nativeSize, format)); - m_sink->platformVideoSink()->setVideoFrame(frame); - - QMetaObject::invokeMethod(m_surfaceTexture, "frameAvailable", Qt::QueuedConnection); + m_nativeSize = size; + QMetaObject::invokeMethod(m_surfaceThread.get(), + [&](){ m_surfaceThread->setFrameSize(size); }, + Qt::BlockingQueuedConnection); } -static const float g_quad[] = { - -1.f, -1.f, 0.f, 0.f, - -1.f, 1.f, 0.f, 1.f, - 1.f, 1.f, 1.f, 1.f, - 1.f, -1.f, 1.f, 0.f -}; - -static QShader getShader(const QString &name) +void QAndroidTextureVideoOutput::stop() { - QFile f(name); - if (f.open(QIODevice::ReadOnly)) - return QShader::fromSerialized(f.readAll()); - - return QShader(); + m_nativeSize = {}; + QMetaObject::invokeMethod(m_surfaceThread.get(), [&](){ m_surfaceThread->clearFrame(); }); } -bool QAndroidTextureVideoOutput::renderAndReadbackFrame() +void QAndroidTextureVideoOutput::reset() { - QMutexLocker locker(&m_mutex); - - if (!m_nativeSize.isValid() || !m_surfaceTexture) - return false; - - if (moveToOpenGLContextThread()) { - // just moved to another thread, must close the execution of this method - QMetaObject::invokeMethod(this, "onFrameAvailable", Qt::ConnectionType::DirectConnection); - return false; - } - - if (!m_readbackRhi) { - QRhi *sinkRhi = m_sink ? m_sink->rhi() : nullptr; - if (sinkRhi && sinkRhi->backend() == QRhi::OpenGLES2) { - // There is an rhi from the sink, e.g. VideoOutput. We lack the necessary - // insight to use that directly, so create our own a QRhi that just wraps the - // same QOpenGLContext. - - const auto constHandles = - static_cast<const QRhiGles2NativeHandles *>(sinkRhi->nativeHandles()); - if (!constHandles) { - qWarning("Failed to get the QRhiGles2NativeHandles to create QRhi readback."); - return false; - } - - auto handles = const_cast<QRhiGles2NativeHandles *>(constHandles); - const auto context = handles->context; - if (!context) { - qWarning("Failed to get the QOpenGLContext to create QRhi readback."); - return false; - } - - sinkRhi->finish(); - m_readbackRhiFallbackSurface = - QRhiGles2InitParams::newFallbackSurface(context->format()); - QRhiGles2InitParams initParams; - initParams.format = context->format(); - initParams.fallbackSurface = m_readbackRhiFallbackSurface; - context->doneCurrent(); - - m_readbackRhi = QRhi::create(QRhi::OpenGLES2, &initParams, {}, handles); - } else { - // No rhi from the sink, e.g. QVideoWidget. - // We will fire up our own QRhi with its own QOpenGLContext. - m_readbackRhiFallbackSurface = QRhiGles2InitParams::newFallbackSurface({}); - QRhiGles2InitParams initParams; - initParams.fallbackSurface = m_readbackRhiFallbackSurface; - m_readbackRhi = QRhi::create(QRhi::OpenGLES2, &initParams); - } - } - - if (!m_readbackRhi) { - qWarning("Failed to create QRhi for video frame readback"); - return false; - } - - QRhiCommandBuffer *cb = nullptr; - if (m_readbackRhi->beginOffscreenFrame(&cb) != QRhi::FrameOpSuccess) - return false; - - if (!m_readbackTex || m_readbackTex->pixelSize() != m_nativeSize) { - delete m_readbackRenderTarget; - delete m_readbackRpDesc; - delete m_readbackTex; - m_readbackTex = m_readbackRhi->newTexture(QRhiTexture::RGBA8, m_nativeSize, 1, QRhiTexture::RenderTarget); - if (!m_readbackTex->create()) { - qWarning("Failed to create readback texture"); - return false; - } - m_readbackRenderTarget = m_readbackRhi->newTextureRenderTarget({ { m_readbackTex } }); - m_readbackRpDesc = m_readbackRenderTarget->newCompatibleRenderPassDescriptor(); - m_readbackRenderTarget->setRenderPassDescriptor(m_readbackRpDesc); - m_readbackRenderTarget->create(); - } - - m_readbackRhi->makeThreadLocalNativeContextCurrent(); - ensureExternalTexture(m_readbackRhi); - m_surfaceTexture->updateTexImage(); - - // The only purpose of m_readbackSrc is to be nice and have a QRhiTexture that belongs - // to m_readbackRhi, not the sink's rhi if there is one. The underlying native object - // (and the rhi's OpenGL context) are the same regardless. - if (!m_readbackSrc) - m_readbackSrc = m_readbackRhi->newTexture(QRhiTexture::RGBA8, m_nativeSize, 1, QRhiTexture::ExternalOES); - - // Keep the object the same (therefore all references to m_readbackSrc in - // the srb or other objects stay valid all the time), just call createFrom - // if the native external texture changes. - const quint64 texId = m_externalTex->nativeTexture().object; - if (m_readbackSrc->nativeTexture().object != texId) - m_readbackSrc->createFrom({ texId, 0 }); - - QRhiResourceUpdateBatch *rub = nullptr; - if (!m_readbackVBuf) { - m_readbackVBuf = m_readbackRhi->newBuffer(QRhiBuffer::Immutable, QRhiBuffer::VertexBuffer, sizeof(g_quad)); - m_readbackVBuf->create(); - if (!rub) - rub = m_readbackRhi->nextResourceUpdateBatch(); - rub->uploadStaticBuffer(m_readbackVBuf, g_quad); - } - - if (!m_readbackUBuf) { - m_readbackUBuf = m_readbackRhi->newBuffer(QRhiBuffer::Dynamic, QRhiBuffer::UniformBuffer, 64 + 64 + 4 + 4); - m_readbackUBuf->create(); - } - - if (!m_externalTexSampler) { - m_externalTexSampler = m_readbackRhi->newSampler(QRhiSampler::Nearest, QRhiSampler::Nearest, QRhiSampler::None, - QRhiSampler::ClampToEdge, QRhiSampler::ClampToEdge); - m_externalTexSampler->create(); - } - - if (!m_readbackSrb) { - m_readbackSrb = m_readbackRhi->newShaderResourceBindings(); - m_readbackSrb->setBindings({ - QRhiShaderResourceBinding::uniformBuffer(0, QRhiShaderResourceBinding::VertexStage | QRhiShaderResourceBinding::FragmentStage, m_readbackUBuf), - QRhiShaderResourceBinding::sampledTexture(1, QRhiShaderResourceBinding::FragmentStage, m_readbackSrc, m_externalTexSampler) - }); - m_readbackSrb->create(); - } - - if (!m_readbackPs) { - m_readbackPs = m_readbackRhi->newGraphicsPipeline(); - m_readbackPs->setTopology(QRhiGraphicsPipeline::TriangleFan); - QShader vs = getShader(QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.vert.qsb")); - Q_ASSERT(vs.isValid()); - QShader fs = getShader(QStringLiteral(":/qt-project.org/multimedia/shaders/externalsampler.frag.qsb")); - Q_ASSERT(fs.isValid()); - m_readbackPs->setShaderStages({ - { QRhiShaderStage::Vertex, vs }, - { QRhiShaderStage::Fragment, fs } - }); - QRhiVertexInputLayout inputLayout; - inputLayout.setBindings({ - { 4 * sizeof(float) } - }); - inputLayout.setAttributes({ - { 0, 0, QRhiVertexInputAttribute::Float2, 0 }, - { 0, 1, QRhiVertexInputAttribute::Float2, 2 * sizeof(float) } - }); - m_readbackPs->setVertexInputLayout(inputLayout); - m_readbackPs->setShaderResourceBindings(m_readbackSrb); - m_readbackPs->setRenderPassDescriptor(m_readbackRpDesc); - m_readbackPs->create(); - } - - QMatrix4x4 identity; - char *p = m_readbackUBuf->beginFullDynamicBufferUpdateForCurrentFrame(); - memcpy(p, identity.constData(), 64); - QMatrix4x4 extMatrix = extTransformMatrix(m_surfaceTexture); - memcpy(p + 64, extMatrix.constData(), 64); - float opacity = 1.0f; - memcpy(p + 64 + 64, &opacity, 4); - m_readbackUBuf->endFullDynamicBufferUpdateForCurrentFrame(); - - cb->beginPass(m_readbackRenderTarget, Qt::transparent, { 1.0f, 0 }, rub); - cb->setGraphicsPipeline(m_readbackPs); - cb->setViewport(QRhiViewport(0, 0, m_nativeSize.width(), m_nativeSize.height())); - cb->setShaderResources(); - const QRhiCommandBuffer::VertexInput vbufBinding(m_readbackVBuf, 0); - cb->setVertexInput(0, 1, &vbufBinding); - cb->draw(4); - - QRhiReadbackDescription readDesc(m_readbackTex); - QRhiReadbackResult readResult; - bool readCompleted = false; - // invoked at latest in the endOffscreenFrame() below - readResult.completed = [&readCompleted] { readCompleted = true; }; - rub = m_readbackRhi->nextResourceUpdateBatch(); - rub->readBackTexture(readDesc, &readResult); - - cb->endPass(rub); - - m_readbackRhi->endOffscreenFrame(); - - if (!readCompleted) - return false; - - // implicit sharing, keep the data alive - m_readbackImageData = readResult.data; - // the QImage does not own the data - m_readbackImage = QImage(reinterpret_cast<const uchar *>(m_readbackImageData.constData()), - readResult.pixelSize.width(), readResult.pixelSize.height(), - QImage::Format_ARGB32_Premultiplied); - - return true; + if (m_sink) + m_sink->platformVideoSink()->setVideoFrame({}); + QMetaObject::invokeMethod(m_surfaceThread.get(), &AndroidTextureThread::clearSurfaceTexture); } -void QAndroidTextureVideoOutput::ensureExternalTexture(QRhi *rhi) +void QAndroidTextureVideoOutput::newFrame(const QVideoFrame &frame) { - if (!m_graphicsDeleter) - m_graphicsDeleter = new GraphicsResourceDeleter; - - if (!m_externalTex) { - m_surfaceTexture->detachFromGLContext(); - m_externalTex = rhi->newTexture(QRhiTexture::RGBA8, m_nativeSize, 1, QRhiTexture::ExternalOES); - if (!m_externalTex->create()) - qWarning("Failed to create native texture object"); - m_surfaceTexture->attachToGLContext(m_externalTex->nativeTexture().object); - } + if (m_sink) + m_sink->setVideoFrame(frame); } QT_END_NAMESPACE + +#include "qandroidvideooutput.moc" +#include "moc_qandroidvideooutput_p.cpp" diff --git a/src/plugins/multimedia/android/common/qandroidvideooutput_p.h b/src/plugins/multimedia/android/common/qandroidvideooutput_p.h index 377f3a4eb..7c9be5aee 100644 --- a/src/plugins/multimedia/android/common/qandroidvideooutput_p.h +++ b/src/plugins/multimedia/android/common/qandroidvideooutput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDVIDEOOUTPUT_H #define QANDROIDVIDEOOUTPUT_H @@ -51,15 +15,13 @@ // We mean it. // -#include <qobject.h> #include <qsize.h> #include <qmutex.h> #include <qreadwritelock.h> -#include <private/qabstractvideobuffer_p.h> +#include <qabstractvideobuffer.h> #include <qmatrix4x4.h> -#include <QtGui/private/qrhi_p.h> -#include <QtGui/qoffscreensurface.h> -#include <QPixmap> +#include <qoffscreensurface.h> +#include <rhi/qrhi.h> QT_BEGIN_NAMESPACE @@ -82,6 +44,7 @@ public: virtual void start() { } virtual void stop() { } virtual void reset() { } + virtual QSize getVideoSize() const { return QSize(0, 0); } Q_SIGNALS: void readyChanged(bool); @@ -90,126 +53,36 @@ protected: QAndroidVideoOutput(QObject *parent) : QObject(parent) { } }; -class GraphicsResourceDeleter : public QObject -{ - Q_OBJECT -public: - void deleteResources(const QList<QRhiResource *> &res) { QMetaObject::invokeMethod(this, "deleteResourcesHelper", Qt::AutoConnection, Q_ARG(QList<QRhiResource*>, res)); } - void deleteRhi(QRhi *rhi, QOffscreenSurface *surf) { QMetaObject::invokeMethod(this, "deleteRhiHelper", Qt::AutoConnection, Q_ARG(QRhi*, rhi), Q_ARG(QOffscreenSurface*, surf)); } - void deleteThis() { QMetaObject::invokeMethod(this, "deleteThisHelper"); } - -private: - Q_INVOKABLE void deleteResourcesHelper(const QList<QRhiResource *> &res); - Q_INVOKABLE void deleteRhiHelper(QRhi *rhi, QOffscreenSurface *surf); - Q_INVOKABLE void deleteThisHelper(); -}; - +class AndroidTextureThread; class QAndroidTextureVideoOutput : public QAndroidVideoOutput { Q_OBJECT public: - explicit QAndroidTextureVideoOutput(QObject *parent = 0); + explicit QAndroidTextureVideoOutput(QVideoSink *sink, QObject *parent = 0); ~QAndroidTextureVideoOutput() override; - QVideoSink *surface() const; - void setSurface(QVideoSink *surface); + QVideoSink *surface() const { return m_sink; } + bool shouldTextureBeUpdated() const; AndroidSurfaceTexture *surfaceTexture() override; - bool isReady() override; void setVideoSize(const QSize &) override; - void start() override; void stop() override; void reset() override; - void renderFrame(); + QSize getVideoSize() const override { return m_nativeSize; } void setSubtitle(const QString &subtitle); + std::shared_ptr<AndroidTextureThread> getSurfaceThread() { return m_surfaceThread; } private Q_SLOTS: - void onFrameAvailable(); + void newFrame(const QVideoFrame &); private: - void initSurfaceTexture(); - bool renderAndReadbackFrame(); - void ensureExternalTexture(QRhi *rhi); - - bool moveToOpenGLContextThread(); - - QMutex m_mutex; - QReadWriteLock m_subtitleLock; - - void clearSurfaceTexture(); - + void startNewSurfaceThread(); QVideoSink *m_sink = nullptr; QSize m_nativeSize; - bool m_started = false; - bool m_renderFrame = false; - - AndroidSurfaceTexture *m_surfaceTexture = nullptr; - - QRhiTexture *m_externalTex = nullptr; - - QRhi *m_readbackRhi = nullptr; - QOffscreenSurface *m_readbackRhiFallbackSurface = nullptr; - QRhiTexture *m_readbackSrc = nullptr; - QRhiTexture *m_readbackTex = nullptr; - QRhiBuffer *m_readbackVBuf = nullptr; - QRhiBuffer *m_readbackUBuf = nullptr; - QRhiSampler *m_externalTexSampler = nullptr; - QRhiShaderResourceBindings *m_readbackSrb = nullptr; - QRhiTextureRenderTarget *m_readbackRenderTarget = nullptr; - QRhiRenderPassDescriptor *m_readbackRpDesc = nullptr; - QRhiGraphicsPipeline *m_readbackPs = nullptr; - - QImage m_readbackImage; - QByteArray m_readbackImageData; + bool m_surfaceCreatedWithoutRhi = false; - QString m_subtitleText; - QPixmap m_subtitlePixmap; - - GraphicsResourceDeleter *m_graphicsDeleter = nullptr; - - friend class AndroidTextureVideoBuffer; -}; - - -class AndroidTextureVideoBuffer : public QAbstractVideoBuffer -{ -public: - AndroidTextureVideoBuffer(QRhi *rhi, QAndroidTextureVideoOutput *output, const QSize &size) - : QAbstractVideoBuffer(rhi ? QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, rhi) - , m_output(output) - , m_size(size) - { - } - - virtual ~AndroidTextureVideoBuffer() {} - - QVideoFrame::MapMode mapMode() const override { return m_mapMode; } - - MapData map(QVideoFrame::MapMode mode) override; - - void unmap() override - { - m_image = QImage(); - m_mapMode = QVideoFrame::NotMapped; - } - - quint64 textureHandle(int plane) const override; - - QMatrix4x4 externalTextureMatrix() const override - { - return m_externalMatrix; - } - -private: - bool updateReadbackFrame(); - - QVideoFrame::MapMode m_mapMode = QVideoFrame::NotMapped; - QAndroidTextureVideoOutput *m_output = nullptr; - QImage m_image; - QSize m_size; - mutable QMatrix4x4 m_externalMatrix; - bool m_textureUpdated = false; + std::shared_ptr<AndroidTextureThread> m_surfaceThread; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/android/common/qandroidvideosink.cpp b/src/plugins/multimedia/android/common/qandroidvideosink.cpp index 7cc0fefe4..3da5eab31 100644 --- a/src/plugins/multimedia/android/common/qandroidvideosink.cpp +++ b/src/plugins/multimedia/android/common/qandroidvideosink.cpp @@ -1,44 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidvideosink_p.h" -#include <QtGui/private/qrhi_p.h> +#include <rhi/qrhi.h> #include <QtCore/qdebug.h> @@ -46,8 +10,6 @@ QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcMediaVideoSink, "qt.multimedia.videosink") - QAndroidVideoSink::QAndroidVideoSink(QVideoSink *parent) : QPlatformVideoSink(parent) { @@ -65,6 +27,9 @@ void QAndroidVideoSink::setRhi(QRhi *rhi) return; m_rhi = rhi; + emit rhiChanged(rhi); } QT_END_NAMESPACE + +#include "moc_qandroidvideosink_p.cpp" diff --git a/src/plugins/multimedia/android/common/qandroidvideosink_p.h b/src/plugins/multimedia/android/common/qandroidvideosink_p.h index d653234f1..9afc58f65 100644 --- a/src/plugins/multimedia/android/common/qandroidvideosink_p.h +++ b/src/plugins/multimedia/android/common/qandroidvideosink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDVIDEOSINK_P_H #define QANDROIDVIDEOSINK_P_H diff --git a/src/plugins/multimedia/android/mediacapture/qandroidcamera.cpp b/src/plugins/multimedia/android/mediacapture/qandroidcamera.cpp index 3bcc93564..52d2e00f6 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidcamera.cpp +++ b/src/plugins/multimedia/android/mediacapture/qandroidcamera.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidcamera_p.h" #include "qandroidcamerasession_p.h" @@ -60,8 +24,11 @@ QAndroidCamera::~QAndroidCamera() void QAndroidCamera::setActive(bool active) { - if (m_cameraSession) + if (m_cameraSession) { m_cameraSession->setActive(active); + } else { + isPendingSetActive = active; + } } bool QAndroidCamera::isActive() const @@ -135,6 +102,11 @@ void QAndroidCamera::setCaptureSession(QPlatformMediaCaptureSession *session) connect(m_cameraSession, &QAndroidCameraSession::activeChanged, this, &QAndroidCamera::activeChanged); connect(m_cameraSession, &QAndroidCameraSession::error, this, &QAndroidCamera::error); connect(m_cameraSession, &QAndroidCameraSession::opened, this, &QAndroidCamera::onCameraOpened); + + if (isPendingSetActive) { + setActive(true); + isPendingSetActive = false; + } } void QAndroidCamera::setFocusMode(QCamera::FocusMode mode) @@ -225,13 +197,11 @@ void QAndroidCamera::onCameraOpened() if (m_cameraSession->camera()->isZoomSupported()) { m_zoomRatios = m_cameraSession->camera()->getZoomRatios(); qreal maxZoom = m_zoomRatios.last() / qreal(100); - if (m_maximumZoom != maxZoom) { - m_maximumZoom = maxZoom; - } + maximumZoomFactorChanged(maxZoom); zoomTo(1, -1); } else { m_zoomRatios.clear(); - m_maximumZoom = 1.0; + maximumZoomFactorChanged(1.0); } m_minExposureCompensationIndex = m_cameraSession->camera()->getMinExposureCompensation(); @@ -588,3 +558,5 @@ void QAndroidCamera::setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) } QT_END_NAMESPACE + +#include "moc_qandroidcamera_p.cpp" diff --git a/src/plugins/multimedia/android/mediacapture/qandroidcamera_p.h b/src/plugins/multimedia/android/mediacapture/qandroidcamera_p.h index 9ee775012..77bbc3133 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidcamera_p.h +++ b/src/plugins/multimedia/android/mediacapture/qandroidcamera_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDCAMERACONTROL_H @@ -112,7 +76,6 @@ private: bool m_continuousVideoFocusSupported = false; bool m_focusPointSupported = false; - float m_maximumZoom; QList<int> m_zoomRatios; QList<QCamera::ExposureMode> m_supportedExposureModes; @@ -123,6 +86,7 @@ private: bool isFlashSupported = false; bool isFlashAutoSupported = false; bool isTorchSupported = false; + bool isPendingSetActive = false; QCameraDevice m_cameraDev; QMap<QCamera::WhiteBalanceMode, QString> m_supportedWhiteBalanceModes; diff --git a/src/plugins/multimedia/android/mediacapture/qandroidcamerasession.cpp b/src/plugins/multimedia/android/mediacapture/qandroidcamerasession.cpp index 83ac04545..7eda1175f 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidcamerasession.cpp +++ b/src/plugins/multimedia/android/mediacapture/qandroidcamerasession.cpp @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Copyright (C) 2016 Ruslan Baratov -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// Copyright (C) 2016 Ruslan Baratov +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidcamerasession_p.h" @@ -53,9 +17,11 @@ #include <qdebug.h> #include <qvideoframe.h> #include <private/qplatformimagecapture_p.h> +#include <private/qplatformvideosink_p.h> #include <private/qmemoryvideobuffer_p.h> #include <private/qcameradevice_p.h> #include <private/qmediastoragelocation_p.h> +#include <private/qvideoframe_p.h> #include <QImageWriter> QT_BEGIN_NAMESPACE @@ -89,6 +55,8 @@ QAndroidCameraSession::QAndroidCameraSession(QObject *parent) QAndroidCameraSession::~QAndroidCameraSession() { + if (m_sink) + disconnect(m_retryPreviewConnection); close(); } @@ -269,6 +237,8 @@ void QAndroidCameraSession::applyResolution(const QSize &captureSize, bool resta // -- adjust resolution QSize adjustedViewfinderResolution; + const QList<QSize> previewSizes = m_camera->getSupportedPreviewSizes(); + const bool validCaptureSize = captureSize.width() > 0 && captureSize.height() > 0; if (validCaptureSize && m_camera->getPreferredPreviewSizeForVideo().isEmpty()) { @@ -280,8 +250,6 @@ void QAndroidCameraSession::applyResolution(const QSize &captureSize, bool resta if (validCaptureSize) captureAspectRatio = qreal(captureSize.width()) / qreal(captureSize.height()); - const QList<QSize> previewSizes = m_camera->getSupportedPreviewSizes(); - if (validCaptureSize) { // search for viewfinder resolution with the same aspect ratio qreal minAspectDiff = 1; @@ -319,31 +287,37 @@ void QAndroidCameraSession::applyResolution(const QSize &captureSize, bool resta // -- adjust FPS - AndroidCamera::FpsRange adjustedFps = m_requestedFpsRange;; + AndroidCamera::FpsRange adjustedFps = m_requestedFpsRange; if (adjustedFps.min == 0 || adjustedFps.max == 0) adjustedFps = currentFpsRange; // -- Set values on camera // fix the resolution of output based on the orientation - QSize outputResolution = adjustedViewfinderResolution; + QSize cameraOutputResolution = adjustedViewfinderResolution; + QSize videoOutputResolution = adjustedViewfinderResolution; + QSize currentVideoOutputResolution = m_videoOutput ? m_videoOutput->getVideoSize() : QSize(0, 0); const int rotation = currentCameraRotation(); - if (rotation == 90 || rotation == 270) - outputResolution.transpose(); + // only transpose if it's valid for the preview + if (rotation == 90 || rotation == 270) { + videoOutputResolution.transpose(); + if (previewSizes.contains(cameraOutputResolution.transposed())) + cameraOutputResolution.transpose(); + } - if (currentViewfinderResolution != outputResolution + if (currentViewfinderResolution != cameraOutputResolution + || (m_videoOutput && currentVideoOutputResolution != videoOutputResolution) || currentPreviewFormat != adjustedPreviewFormat || currentFpsRange.min != adjustedFps.min || currentFpsRange.max != adjustedFps.max) { - if (m_videoOutput) { - m_videoOutput->setVideoSize(outputResolution); + m_videoOutput->setVideoSize(videoOutputResolution); } // if preview is started, we have to stop it first before changing its size if (m_previewStarted && restartPreview) m_camera->stopPreview(); - m_camera->setPreviewSize(outputResolution); + m_camera->setPreviewSize(cameraOutputResolution); m_camera->setPreviewFormat(adjustedPreviewFormat); m_camera->setPreviewFpsRange(adjustedFps); @@ -444,7 +418,6 @@ void QAndroidCameraSession::stopPreview() if (m_videoOutput) { m_videoOutput->stop(); - m_videoOutput->reset(); } m_previewStarted = false; } @@ -612,7 +585,6 @@ int QAndroidCameraSession::captureImage() m_currentImageCaptureId = newImageCaptureId; - applyImageSettings(); applyResolution(m_actualImageSettings.resolution()); m_camera->takePicture(); @@ -689,12 +661,16 @@ void QAndroidCameraSession::onNewPreviewFrame(const QVideoFrame &frame) m_videoFrameCallbackMutex.unlock(); } -void QAndroidCameraSession::onCameraPictureCaptured(const QVideoFrame &frame) +void QAndroidCameraSession::onCameraPictureCaptured(const QByteArray &bytes, + QVideoFrameFormat::PixelFormat format, QSize size,int bytesPerLine) { - // Loading and saving the captured image can be slow, do it in a separate thread - (void)QtConcurrent::run(&QAndroidCameraSession::processCapturedImage, this, - m_currentImageCaptureId, frame, m_imageCaptureToBuffer, - m_currentImageCaptureFileName); + if (m_imageCaptureToBuffer) { + processCapturedImageToBuffer(m_currentImageCaptureId, bytes, format, size, bytesPerLine); + } else { + // Loading and saving the captured image can be slow, do it in a separate thread + (void)QtConcurrent::run(&QAndroidCameraSession::processCapturedImage, this, + m_currentImageCaptureId, bytes, m_currentImageCaptureFileName); + } // Preview needs to be restarted after taking a picture if (m_camera) @@ -732,37 +708,39 @@ void QAndroidCameraSession::onCameraPreviewStopped() setReadyForCapture(false); } -void QAndroidCameraSession::processCapturedImage(int id, const QVideoFrame &frame, - bool captureToBuffer, const QString &fileName) +void QAndroidCameraSession::processCapturedImage(int id, const QByteArray &bytes, const QString &fileName) { - if (captureToBuffer) { - emit imageAvailable(id, frame); - return; - } - const QString actualFileName = QMediaStorageLocation::generateFileName( fileName, QStandardPaths::PicturesLocation, QLatin1String("jpg")); - QImageWriter writer(actualFileName); - - if (!writer.canWrite()) { + QFile writer(actualFileName); + if (!writer.open(QIODeviceBase::WriteOnly)) { const QString errorMessage = tr("File is not available: %1").arg(writer.errorString()); emit imageCaptureError(id, QImageCapture::Error::ResourceError, errorMessage); return; } - const bool written = writer.write(frame.toImage()); - if (!written) { + if (writer.write(bytes) < 0) { const QString errorMessage = tr("Could not save to file: %1").arg(writer.errorString()); emit imageCaptureError(id, QImageCapture::Error::ResourceError, errorMessage); return; } + writer.close(); if (fileName.isEmpty() || QFileInfo(fileName).isRelative()) AndroidMultimediaUtils::registerMediaFile(actualFileName); emit imageSaved(id, actualFileName); } +void QAndroidCameraSession::processCapturedImageToBuffer(int id, const QByteArray &bytes, + QVideoFrameFormat::PixelFormat format, QSize size, int bytesPerLine) +{ + QVideoFrame frame = QVideoFramePrivate::createFrame( + std::make_unique<QMemoryVideoBuffer>(bytes, bytesPerLine), + QVideoFrameFormat(size, format)); + emit imageAvailable(id, frame); +} + void QAndroidCameraSession::onVideoOutputReady(bool ready) { if (ready && m_active) @@ -801,17 +779,30 @@ void QAndroidCameraSession::setVideoSink(QVideoSink *sink) if (m_sink == sink) return; + if (m_sink) + disconnect(m_retryPreviewConnection); + m_sink = sink; + if (m_sink) + m_retryPreviewConnection = + connect(m_sink->platformVideoSink(), &QPlatformVideoSink::rhiChanged, this, [&]() + { + if (m_active) { + setActive(false); + setActive(true); + } + }, Qt::DirectConnection); if (m_sink) { delete m_textureOutput; m_textureOutput = nullptr; - m_textureOutput = new QAndroidTextureVideoOutput(this); - m_textureOutput->setSurface(m_sink); + m_textureOutput = new QAndroidTextureVideoOutput(m_sink, this); } setVideoOutput(m_textureOutput); } QT_END_NAMESPACE + +#include "moc_qandroidcamerasession_p.cpp" diff --git a/src/plugins/multimedia/android/mediacapture/qandroidcamerasession_p.h b/src/plugins/multimedia/android/mediacapture/qandroidcamerasession_p.h index 2fdd9d33b..3b56d9c3b 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidcamerasession_p.h +++ b/src/plugins/multimedia/android/mediacapture/qandroidcamerasession_p.h @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Copyright (C) 2016 Ruslan Baratov -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// Copyright (C) 2016 Ruslan Baratov +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDCAMERASESSION_H #define QANDROIDCAMERASESSION_H @@ -138,7 +102,7 @@ private Q_SLOTS: void onCameraTakePictureFailed(); void onCameraPictureExposed(); - void onCameraPictureCaptured(const QVideoFrame &frame); + void onCameraPictureCaptured(const QByteArray &bytes, QVideoFrameFormat::PixelFormat format, QSize size, int bytesPerLine); void onLastPreviewFrameFetched(const QVideoFrame &frame); void onNewPreviewFrame(const QVideoFrame &frame); void onCameraPreviewStarted(); @@ -157,8 +121,9 @@ private: void applyImageSettings(); void processPreviewImage(int id, const QVideoFrame &frame, int rotation); - void processCapturedImage(int id, const QVideoFrame &frame, bool captureToBuffer, - const QString &fileName); + void processCapturedImage(int id, const QByteArray &bytes, const QString &fileName); + void processCapturedImageToBuffer(int id, const QByteArray &bytes, + QVideoFrameFormat::PixelFormat format, QSize size, int bytesPerLine); void setActiveHelper(bool active); @@ -193,6 +158,7 @@ private: QMutex m_videoFrameCallbackMutex; PreviewCallback *m_previewCallback; bool m_keepActive; + QMetaObject::Connection m_retryPreviewConnection; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/android/mediacapture/qandroidcapturesession.cpp b/src/plugins/multimedia/android/mediacapture/qandroidcapturesession.cpp index d274fc9e9..3b005e4a5 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidcapturesession.cpp +++ b/src/plugins/multimedia/android/mediacapture/qandroidcapturesession.cpp @@ -1,46 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidcapturesession_p.h" #include "androidcamera_p.h" #include "qandroidcamerasession_p.h" +#include "qaudioinput.h" +#include "qaudiooutput.h" #include "androidmediaplayer_p.h" #include "androidmultimediautils_p.h" #include "qandroidmultimediautils_p.h" @@ -74,6 +40,8 @@ QAndroidCaptureSession::~QAndroidCaptureSession() { stop(); m_mediaRecorder = nullptr; + if (m_audioInput && m_audioOutput) + AndroidMediaPlayer::stopSoundStreaming(); } void QAndroidCaptureSession::setCameraSession(QAndroidCameraSession *cameraSession) @@ -97,7 +65,23 @@ void QAndroidCaptureSession::setCameraSession(QAndroidCameraSession *cameraSessi void QAndroidCaptureSession::setAudioInput(QPlatformAudioInput *input) { + if (m_audioInput == input) + return; + + if (m_audioInput) { + disconnect(m_audioInputChanged); + } + m_audioInput = input; + + if (m_audioInput) { + m_audioInputChanged = connect(m_audioInput->q, &QAudioInput::deviceChanged, this, [this]() { + if (m_state == QMediaRecorder::RecordingState) + m_mediaRecorder->setAudioInput(m_audioInput->device.id()); + updateStreamingState(); + }); + } + updateStreamingState(); } void QAndroidCaptureSession::setAudioOutput(QPlatformAudioOutput *output) @@ -105,10 +89,30 @@ void QAndroidCaptureSession::setAudioOutput(QPlatformAudioOutput *output) if (m_audioOutput == output) return; + if (m_audioOutput) + disconnect(m_audioOutputChanged); + m_audioOutput = output; - if (m_audioOutput) + if (m_audioOutput) { + m_audioOutputChanged = connect(m_audioOutput->q, &QAudioOutput::deviceChanged, this, + [this] () { + AndroidMediaPlayer::setAudioOutput(m_audioOutput->device.id()); + updateStreamingState(); + }); AndroidMediaPlayer::setAudioOutput(m_audioOutput->device.id()); + } + updateStreamingState(); +} + +void QAndroidCaptureSession::updateStreamingState() +{ + if (m_audioInput && m_audioOutput) { + AndroidMediaPlayer::startSoundStreaming(m_audioInput->device.id().toInt(), + m_audioOutput->device.id().toInt()); + } else { + AndroidMediaPlayer::stopSoundStreaming(); + } } QMediaRecorder::RecorderState QAndroidCaptureSession::state() const @@ -129,7 +133,7 @@ void QAndroidCaptureSession::start(QMediaEncoderSettings &settings, const QUrl & return; if (!m_cameraSession && !m_audioInput) { - emit error(QMediaRecorder::ResourceError, QLatin1String("No devices are set")); + updateError(QMediaRecorder::ResourceError, QLatin1String("No devices are set")); return; } @@ -137,14 +141,14 @@ void QAndroidCaptureSession::start(QMediaEncoderSettings &settings, const QUrl & const bool validCameraSession = m_cameraSession && m_cameraSession->camera(); - if (validCameraSession && !qt_androidRequestCameraPermission()) { - emit error(QMediaRecorder::ResourceError, QLatin1String("Camera permission denied.")); + if (validCameraSession && !qt_androidCheckCameraPermission()) { + updateError(QMediaRecorder::ResourceError, QLatin1String("Camera permission denied.")); setKeepAlive(false); return; } - if (m_audioInput && !qt_androidRequestRecordingPermission()) { - emit error(QMediaRecorder::ResourceError, QLatin1String("Microphone permission denied.")); + if (m_audioInput && !qt_androidCheckMicrophonePermission()) { + updateError(QMediaRecorder::ResourceError, QLatin1String("Microphone permission denied.")); setKeepAlive(false); return; } @@ -160,7 +164,6 @@ void QAndroidCaptureSession::start(QMediaEncoderSettings &settings, const QUrl & // Set audio/video sources if (validCameraSession) { m_cameraSession->camera()->stopPreviewSynchronous(); - m_cameraSession->applyResolution(settings.videoResolution(), false); m_cameraSession->camera()->unlock(); m_mediaRecorder->setCamera(m_cameraSession->camera()); @@ -168,7 +171,6 @@ void QAndroidCaptureSession::start(QMediaEncoderSettings &settings, const QUrl & } if (m_audioInput) { - m_mediaRecorder->setAudioSource(AndroidMediaRecorder::Camcorder); m_mediaRecorder->setAudioInput(m_audioInput->device.id()); if (!m_mediaRecorder->isAudioSourceSet()) m_mediaRecorder->setAudioSource(AndroidMediaRecorder::DefaultAudioSource); @@ -214,36 +216,20 @@ void QAndroidCaptureSession::start(QMediaEncoderSettings &settings, const QUrl & m_outputLocationIsStandard = location.isEmpty() || QFileInfo(location).isRelative(); m_mediaRecorder->setOutputFile(filePath); - // Even though the Android doc explicitly says that calling MediaRecorder.setPreviewDisplay() - // is not necessary when the Camera already has a Surface, it doesn't actually work on some - // devices. For example on the Samsung Galaxy Tab 2, the camera server dies after prepare() - // and start() if MediaRecorder.setPreviewDisplay() is not called. if (validCameraSession) { - - if (m_cameraSession->videoOutput()) { - // When using a SurfaceTexture, we need to pass a new one to the MediaRecorder, not the - // same one that is set on the Camera or it will crash, hence the reset(). - m_cameraSession->videoOutput()->reset(); - if (m_cameraSession->videoOutput()->surfaceTexture()) - m_mediaRecorder->setSurfaceTexture( - m_cameraSession->videoOutput()->surfaceTexture()); - else if (m_cameraSession->videoOutput()->surfaceHolder()) - m_mediaRecorder->setSurfaceHolder(m_cameraSession->videoOutput()->surfaceHolder()); - } - m_cameraSession->disableRotation(); } if (!m_mediaRecorder->prepare()) { - emit error(QMediaRecorder::FormatError, QLatin1String("Unable to prepare the media recorder.")); + updateError(QMediaRecorder::FormatError, + QLatin1String("Unable to prepare the media recorder.")); restartViewfinder(); return; } if (!m_mediaRecorder->start()) { - emit error(QMediaRecorder::FormatError, - QMediaRecorderPrivate::msgFailedStartRecording()); + updateError(QMediaRecorder::FormatError, QMediaRecorderPrivate::msgFailedStartRecording()); restartViewfinder(); return; @@ -380,23 +366,7 @@ void QAndroidCaptureSession::restartViewfinder() if (m_cameraSession && m_cameraSession->camera()) { m_cameraSession->camera()->reconnect(); - - // This is not necessary on most devices, but it crashes on some if we don't stop the - // preview and reset the preview display on the camera when recording is over. m_cameraSession->camera()->stopPreviewSynchronous(); - - if (m_cameraSession->videoOutput()) { - // When using a SurfaceTexture, we need to pass a new one to the MediaRecorder, not the - // same one that is set on the Camera or it will crash, hence the reset(). - m_cameraSession->videoOutput()->reset(); - if (m_cameraSession->videoOutput()->surfaceTexture()) - m_cameraSession->camera()->setPreviewTexture( - m_cameraSession->videoOutput()->surfaceTexture()); - else if (m_cameraSession->videoOutput()->surfaceHolder()) - m_cameraSession->camera()->setPreviewDisplay( - m_cameraSession->videoOutput()->surfaceHolder()); - } - m_cameraSession->camera()->startPreview(); m_cameraSession->setReadyForCapture(true); m_cameraSession->enableRotation(); @@ -434,6 +404,10 @@ void QAndroidCaptureSession::onCameraOpened() std::sort(m_supportedResolutions.begin(), m_supportedResolutions.end(), qt_sizeLessThan); std::sort(m_supportedFramerates.begin(), m_supportedFramerates.end()); + + QMediaEncoderSettings defaultSettings; + applySettings(defaultSettings); + m_cameraSession->applyResolution(defaultSettings.videoResolution()); } QAndroidCaptureSession::CaptureProfile QAndroidCaptureSession::getProfile(int id) @@ -477,7 +451,7 @@ void QAndroidCaptureSession::onError(int what, int extra) Q_UNUSED(what); Q_UNUSED(extra); stop(true); - emit error(QMediaRecorder::ResourceError, QLatin1String("Unknown error.")); + updateError(QMediaRecorder::ResourceError, QLatin1String("Unknown error.")); } void QAndroidCaptureSession::onInfo(int what, int extra) @@ -486,12 +460,14 @@ void QAndroidCaptureSession::onInfo(int what, int extra) if (what == 800) { // MEDIA_RECORDER_INFO_MAX_DURATION_REACHED stop(); - emit error(QMediaRecorder::OutOfSpaceError, QLatin1String("Maximum duration reached.")); + updateError(QMediaRecorder::OutOfSpaceError, QLatin1String("Maximum duration reached.")); } else if (what == 801) { // MEDIA_RECORDER_INFO_MAX_FILESIZE_REACHED stop(); - emit error(QMediaRecorder::OutOfSpaceError, QLatin1String("Maximum file size reached.")); + updateError(QMediaRecorder::OutOfSpaceError, QLatin1String("Maximum file size reached.")); } } QT_END_NAMESPACE + +#include "moc_qandroidcapturesession_p.cpp" diff --git a/src/plugins/multimedia/android/mediacapture/qandroidcapturesession_p.h b/src/plugins/multimedia/android/mediacapture/qandroidcapturesession_p.h index e91e5b210..161d47994 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidcapturesession_p.h +++ b/src/plugins/multimedia/android/mediacapture/qandroidcapturesession_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDCAPTURESESSION_H #define QANDROIDCAPTURESESSION_H @@ -103,10 +67,10 @@ public: if (m_mediaEncoder) m_mediaEncoder->actualLocationChanged(location); } - void error(int error, const QString &errorString) + void updateError(int error, const QString &errorString) { if (m_mediaEncoder) - m_mediaEncoder->error(QMediaRecorder::Error(error), errorString); + m_mediaEncoder->updateError(QMediaRecorder::Error(error), errorString); } private Q_SLOTS: @@ -153,6 +117,7 @@ private: CaptureProfile getProfile(int id); void restartViewfinder(); + void updateStreamingState(); QAndroidMediaEncoder *m_mediaEncoder = nullptr; std::shared_ptr<AndroidMediaRecorder> m_mediaRecorder; @@ -179,6 +144,8 @@ private: QList<QSize> m_supportedResolutions; QList<qreal> m_supportedFramerates; + QMetaObject::Connection m_audioInputChanged; + QMetaObject::Connection m_audioOutputChanged; QMetaObject::Connection m_connOpenCamera; QMetaObject::Connection m_connActiveChangedCamera; diff --git a/src/plugins/multimedia/android/mediacapture/qandroidimagecapture.cpp b/src/plugins/multimedia/android/mediacapture/qandroidimagecapture.cpp index a3fd5cd16..4105851ed 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidimagecapture.cpp +++ b/src/plugins/multimedia/android/mediacapture/qandroidimagecapture.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidimagecapture_p.h" @@ -105,3 +69,5 @@ void QAndroidImageCapture::setCaptureSession(QPlatformMediaCaptureSession *sessi this, &QAndroidImageCapture::error); } QT_END_NAMESPACE + +#include "moc_qandroidimagecapture_p.cpp" diff --git a/src/plugins/multimedia/android/mediacapture/qandroidimagecapture_p.h b/src/plugins/multimedia/android/mediacapture/qandroidimagecapture_p.h index 83d15445f..ac273c195 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidimagecapture_p.h +++ b/src/plugins/multimedia/android/mediacapture/qandroidimagecapture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDCAMERAIMAGECAPTURECONTROL_H #define QANDROIDCAMERAIMAGECAPTURECONTROL_H diff --git a/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession.cpp b/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession.cpp index ddc690d77..e2b551d35 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession.cpp +++ b/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession.cpp @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Copyright (C) 2016 Ruslan Baratov -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// Copyright (C) 2016 Ruslan Baratov +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidmediacapturesession_p.h" @@ -83,10 +47,8 @@ void QAndroidMediaCaptureSession::setCamera(QPlatformCamera *camera) m_cameraControl->setCaptureSession(nullptr); m_cameraControl = control; - if (m_cameraControl) { + if (m_cameraControl) m_cameraControl->setCaptureSession(this); - m_cameraControl->setActive(true); - } emit cameraChanged(); } @@ -149,3 +111,5 @@ void QAndroidMediaCaptureSession::setVideoPreview(QVideoSink *sink) } QT_END_NAMESPACE + +#include "moc_qandroidmediacapturesession_p.cpp" diff --git a/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession_p.h b/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession_p.h index bd32bb834..90c792c32 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession_p.h +++ b/src/plugins/multimedia/android/mediacapture/qandroidmediacapturesession_p.h @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Copyright (C) 2016 Ruslan Baratov -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// Copyright (C) 2016 Ruslan Baratov +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDCAPTURESERVICE_H #define QANDROIDCAPTURESERVICE_H diff --git a/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder.cpp b/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder.cpp index b3543bd91..d3449312d 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder.cpp +++ b/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidmediaencoder_p.h" #include "qandroidmultimediautils_p.h" diff --git a/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder_p.h b/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder_p.h index 281c10317..b46268449 100644 --- a/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder_p.h +++ b/src/plugins/multimedia/android/mediacapture/qandroidmediaencoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDMEDIAENCODER_H #define QANDROIDMEDIAENCODER_H diff --git a/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer.cpp b/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer.cpp index 01a57c298..b257a8986 100644 --- a/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer.cpp +++ b/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidmediaplayer_p.h" #include "androidmediaplayer_p.h" @@ -44,11 +8,12 @@ #include "qandroidaudiooutput_p.h" #include "qaudiooutput.h" +#include <private/qplatformvideosink_p.h> #include <qloggingcategory.h> QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(lcMediaPlayer, "qt.multimedia.mediaplayer.android") +static Q_LOGGING_CATEGORY(lcMediaPlayer, "qt.multimedia.mediaplayer.android") class StateChangeNotifier { @@ -84,6 +49,8 @@ QAndroidMediaPlayer::QAndroidMediaPlayer(QMediaPlayer *parent) mMediaPlayer(new AndroidMediaPlayer), mState(AndroidMediaPlayer::Uninitialized) { + // Set seekable to True by default. It changes if MEDIA_INFO_NOT_SEEKABLE is received + seekableChanged(true); connect(mMediaPlayer, &AndroidMediaPlayer::bufferingChanged, this, &QAndroidMediaPlayer::onBufferingChanged); connect(mMediaPlayer, &AndroidMediaPlayer::info, this, &QAndroidMediaPlayer::onInfo); @@ -102,6 +69,9 @@ QAndroidMediaPlayer::QAndroidMediaPlayer(QMediaPlayer *parent) QAndroidMediaPlayer::~QAndroidMediaPlayer() { + if (m_videoSink) + disconnect(m_videoSink->platformVideoSink(), nullptr, this, nullptr); + mMediaPlayer->disconnect(); mMediaPlayer->release(); delete mMediaPlayer; @@ -249,43 +219,24 @@ void QAndroidMediaPlayer::updateAvailablePlaybackRanges() qreal QAndroidMediaPlayer::playbackRate() const { - if (mHasPendingPlaybackRate || - (mState & (AndroidMediaPlayer::Initialized - | AndroidMediaPlayer::Prepared - | AndroidMediaPlayer::Started - | AndroidMediaPlayer::Paused - | AndroidMediaPlayer::PlaybackCompleted - | AndroidMediaPlayer::Error)) == 0) { - return mPendingPlaybackRate; - } - - return mMediaPlayer->playbackRate(); + return mCurrentPlaybackRate; } void QAndroidMediaPlayer::setPlaybackRate(qreal rate) { - if ((mState & (AndroidMediaPlayer::Initialized - | AndroidMediaPlayer::Prepared - | AndroidMediaPlayer::Started - | AndroidMediaPlayer::Paused - | AndroidMediaPlayer::PlaybackCompleted - | AndroidMediaPlayer::Error)) == 0) { - if (mPendingPlaybackRate != rate) { - mPendingPlaybackRate = rate; + if (mState != AndroidMediaPlayer::Started) { + // If video isn't playing, changing speed rate may start it automatically + // It need to be postponed + if (mCurrentPlaybackRate != rate) { + mCurrentPlaybackRate = rate; mHasPendingPlaybackRate = true; Q_EMIT playbackRateChanged(rate); } return; } - bool succeeded = mMediaPlayer->setPlaybackRate(rate); - - if (mHasPendingPlaybackRate) { - mHasPendingPlaybackRate = false; - mPendingPlaybackRate = qreal(1.0); - if (!succeeded) - Q_EMIT playbackRateChanged(playbackRate()); - } else if (succeeded) { + if (mMediaPlayer->setPlaybackRate(rate)) { + mCurrentPlaybackRate = rate; Q_EMIT playbackRateChanged(rate); } } @@ -325,7 +276,8 @@ void QAndroidMediaPlayer::setMedia(const QUrl &mediaContent, if (mVideoSize.isValid() && mVideoOutput) mVideoOutput->setVideoSize(mVideoSize); - if ((mMediaPlayer->display() == 0) && mVideoOutput) + if (mVideoOutput && + (mMediaPlayer->display() == 0 || mVideoOutput->shouldTextureBeUpdated())) mMediaPlayer->setDisplay(mVideoOutput->surfaceTexture()); mMediaPlayer->setDataSource(QNetworkRequest(mediaContent)); mMediaPlayer->prepareAsync(); @@ -344,6 +296,9 @@ void QAndroidMediaPlayer::setVideoSink(QVideoSink *sink) if (m_videoSink == sink) return; + if (m_videoSink) + disconnect(m_videoSink->platformVideoSink(), nullptr, this, nullptr); + m_videoSink = sink; if (!m_videoSink) { @@ -356,16 +311,17 @@ void QAndroidMediaPlayer::setVideoSink(QVideoSink *sink) mMediaPlayer->setDisplay(nullptr); } - mVideoOutput = new QAndroidTextureVideoOutput(this); + mVideoOutput = new QAndroidTextureVideoOutput(sink, this); connect(mVideoOutput, &QAndroidTextureVideoOutput::readyChanged, this, &QAndroidMediaPlayer::onVideoOutputReady); connect(mMediaPlayer, &AndroidMediaPlayer::timedTextChanged, mVideoOutput, &QAndroidTextureVideoOutput::setSubtitle); - mVideoOutput->setSurface(sink); - if (mVideoOutput->isReady()) mMediaPlayer->setDisplay(mVideoOutput->surfaceTexture()); + + connect(m_videoSink->platformVideoSink(), &QPlatformVideoSink::rhiChanged, this, [&]() + { mMediaPlayer->setDisplay(mVideoOutput->surfaceTexture()); }); } void QAndroidMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) @@ -416,6 +372,14 @@ void QAndroidMediaPlayer::play() updateAudioDevice(); + if (mHasPendingPlaybackRate) { + mHasPendingPlaybackRate = false; + if (mMediaPlayer->setPlaybackRate(mCurrentPlaybackRate)) + return; + mCurrentPlaybackRate = mMediaPlayer->playbackRate(); + Q_EMIT playbackRateChanged(mCurrentPlaybackRate); + } + mMediaPlayer->play(); } @@ -437,8 +401,6 @@ void QAndroidMediaPlayer::pause() mPendingState = QMediaPlayer::PausedState; return; } - if (mVideoOutput) - mVideoOutput->renderFrame(); const qint64 currentPosition = mMediaPlayer->getCurrentPosition(); setPosition(currentPosition); @@ -462,17 +424,16 @@ void QAndroidMediaPlayer::stop() return; } + if (mCurrentPlaybackRate != 1.) + // Playback rate need to by reapplied + mHasPendingPlaybackRate = true; + if (mVideoOutput) mVideoOutput->stop(); mMediaPlayer->stop(); } -bool QAndroidMediaPlayer::isSeekable() const -{ - return true; -} - void QAndroidMediaPlayer::onInfo(qint32 what, qint32 extra) { StateChangeNotifier notifier(this); @@ -552,7 +513,9 @@ void QAndroidMediaPlayer::onError(qint32 what, qint32 extra) setMediaStatus(QMediaPlayer::InvalidMedia); break; case AndroidMediaPlayer::MEDIA_ERROR_BAD_THINGS_ARE_GOING_TO_HAPPEN: - errorString += QLatin1String(" (Unknown error/Insufficient resources)"); + errorString += mMediaContent.scheme() == QLatin1String("rtsp") + ? QLatin1String(" (Unknown error/Insufficient resources or RTSP may not be supported)") + : QLatin1String(" (Unknown error/Insufficient resources)"); error = QMediaPlayer::ResourceError; break; } @@ -685,7 +648,6 @@ void QAndroidMediaPlayer::onStateChanged(qint32 state) mMediaPlayer->setDisplay(0); if (mVideoOutput) { mVideoOutput->stop(); - mVideoOutput->reset(); } } } @@ -981,8 +943,6 @@ void QAndroidMediaPlayer::flushPendingStates() setVolume(mPendingVolume); if (mPendingMute != -1) setMuted((mPendingMute == 1)); - if (mHasPendingPlaybackRate) - setPlaybackRate(mPendingPlaybackRate); switch (newState) { case QMediaPlayer::PlayingState: @@ -1035,3 +995,5 @@ void QAndroidMediaPlayer::updateTrackInfo() } QT_END_NAMESPACE + +#include "moc_qandroidmediaplayer_p.cpp" diff --git a/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer_p.h b/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer_p.h index 9eec31dc6..dd2a3469d 100644 --- a/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer_p.h +++ b/src/plugins/multimedia/android/mediaplayer/qandroidmediaplayer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDMEDIAPLAYERCONTROL_H #define QANDROIDMEDIAPLAYERCONTROL_H @@ -97,8 +61,6 @@ public: void pause() override; void stop() override; - bool isSeekable() const override; - int trackCount(TrackType trackType) override; QMediaMetaData trackMetaData(TrackType trackType, int streamNumber) override; int activeTrack(TrackType trackType) override; @@ -138,7 +100,7 @@ private: int mPendingMute = -1; bool mReloadingMedia = false; int mActiveStateChangeNotifiers = 0; - qreal mPendingPlaybackRate = 1.; + qreal mCurrentPlaybackRate = 1.; bool mHasPendingPlaybackRate = false; // we need this because the rate can theoretically be negative QMap<TrackType, QList<QAndroidMetaData>> mTracksMetadata; diff --git a/src/plugins/multimedia/android/mediaplayer/qandroidmetadata.cpp b/src/plugins/multimedia/android/mediaplayer/qandroidmetadata.cpp index 4765fa0ad..b01845fa7 100644 --- a/src/plugins/multimedia/android/mediaplayer/qandroidmetadata.cpp +++ b/src/plugins/multimedia/android/mediaplayer/qandroidmetadata.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidmetadata_p.h" @@ -49,8 +13,6 @@ QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(lcaMetadata, "qt.multimedia.android.metadata") - // Genre name ordered by ID // see: http://id3.org/id3v2.3.0#Appendix_A_-_Genre_List_from_ID3v1 static const char* qt_ID3GenreNames[] = @@ -170,15 +132,7 @@ QLocale::Language getLocaleLanguage(const QString &language) if (language == QLatin1String("und") || language == QStringLiteral("mis")) return QLocale::AnyLanguage; - QLocale locale(language); - - if (locale == QLocale::c()) { - qCWarning(lcaMetadata) << "Could not parse language:" << language - << ". It is not a valid Unicode CLDR language code."; - return QLocale::AnyLanguage; - } - - return locale.language(); + return QLocale::codeToLanguage(language, QLocale::ISO639Part2); } QAndroidMetaData::QAndroidMetaData(int trackType, int androidTrackType, int androidTrackNumber, diff --git a/src/plugins/multimedia/android/mediaplayer/qandroidmetadata_p.h b/src/plugins/multimedia/android/mediaplayer/qandroidmetadata_p.h index 812edb062..1bbad92dd 100644 --- a/src/plugins/multimedia/android/mediaplayer/qandroidmetadata_p.h +++ b/src/plugins/multimedia/android/mediaplayer/qandroidmetadata_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDMETADATA_H #define QANDROIDMETADATA_H diff --git a/src/plugins/multimedia/android/qandroidformatsinfo.cpp b/src/plugins/multimedia/android/qandroidformatsinfo.cpp index 584c7a122..3b23340ce 100644 --- a/src/plugins/multimedia/android/qandroidformatsinfo.cpp +++ b/src/plugins/multimedia/android/qandroidformatsinfo.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidformatsinfo_p.h" @@ -108,7 +72,6 @@ QAndroidFormatInfo::QAndroidFormatInfo() { const QMediaFormat::AudioCodec aac = hasEncoder(QMediaFormat::AudioCodec::AAC); const QMediaFormat::AudioCodec mp3 = hasEncoder(QMediaFormat::AudioCodec::MP3); - const QMediaFormat::AudioCodec flac = hasEncoder(QMediaFormat::AudioCodec::FLAC); const QMediaFormat::AudioCodec opus = hasEncoder(QMediaFormat::AudioCodec::Opus); const QMediaFormat::AudioCodec vorbis = hasEncoder(QMediaFormat::AudioCodec::Vorbis); @@ -123,11 +86,12 @@ QAndroidFormatInfo::QAndroidFormatInfo() encoders = { { QMediaFormat::AAC, {aac}, {} }, { QMediaFormat::MP3, {mp3}, {} }, - { QMediaFormat::FLAC, {flac}, {} }, - { QMediaFormat::Mpeg4Audio, {mp3, aac, flac, vorbis}, {} }, - { QMediaFormat::MPEG4, {mp3, aac, flac, vorbis}, {h264, h265, av1} }, - { QMediaFormat::Ogg, {opus, vorbis, flac}, {} }, - { QMediaFormat::Matroska, {mp3, opus, flac}, {vp8, vp9, h264, h265, av1} }, + // FLAC encoder is not supported by the MediaRecorder used for recording + // { QMediaFormat::FLAC, {flac}, {} }, + { QMediaFormat::Mpeg4Audio, {mp3, aac, vorbis}, {} }, + { QMediaFormat::MPEG4, {mp3, aac, vorbis}, {h264, h265, av1} }, + { QMediaFormat::Ogg, {opus, vorbis}, {} }, + { QMediaFormat::Matroska, {mp3, opus}, {vp8, vp9, h264, h265, av1} }, // NOTE: WebM seems to be documented to supported with VP8 encoder, // but the Camera API doesn't work with it, keep it commented for now. // { QMediaFormat::WebM, {vorbis, opus}, {vp8, vp9} } diff --git a/src/plugins/multimedia/android/qandroidformatsinfo_p.h b/src/plugins/multimedia/android/qandroidformatsinfo_p.h index fc349189a..2d14ad181 100644 --- a/src/plugins/multimedia/android/qandroidformatsinfo_p.h +++ b/src/plugins/multimedia/android/qandroidformatsinfo_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDFORMATINFO_H #define QANDROIDFORMATINFO_H diff --git a/src/plugins/multimedia/android/qandroidintegration.cpp b/src/plugins/multimedia/android/qandroidintegration.cpp index 5289f7f04..c7077e49d 100644 --- a/src/plugins/multimedia/android/qandroidintegration.cpp +++ b/src/plugins/multimedia/android/qandroidintegration.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qandroidintegration_p.h" #include "qandroidglobal_p.h" @@ -73,72 +37,60 @@ public: QPlatformMediaIntegration* create(const QString &name) override { - if (name == QLatin1String("android")) + if (name == u"android") return new QAndroidIntegration; return nullptr; } }; +QAndroidIntegration::QAndroidIntegration() : QPlatformMediaIntegration(QLatin1String("android")) { } -QAndroidIntegration::QAndroidIntegration() -{ - -} - -QAndroidIntegration::~QAndroidIntegration() -{ - delete m_formatInfo; -} - -QPlatformAudioDecoder *QAndroidIntegration::createAudioDecoder(QAudioDecoder *decoder) +QMaybe<QPlatformAudioDecoder *> QAndroidIntegration::createAudioDecoder(QAudioDecoder *decoder) { return new QAndroidAudioDecoder(decoder); } -QPlatformMediaFormatInfo *QAndroidIntegration::formatInfo() +QPlatformMediaFormatInfo *QAndroidIntegration::createFormatInfo() { - if (!m_formatInfo) - m_formatInfo = new QAndroidFormatInfo(); - return m_formatInfo; - + return new QAndroidFormatInfo; } -QPlatformMediaCaptureSession *QAndroidIntegration::createCaptureSession() +QMaybe<QPlatformMediaCaptureSession *> QAndroidIntegration::createCaptureSession() { return new QAndroidMediaCaptureSession(); } -QPlatformMediaPlayer *QAndroidIntegration::createPlayer(QMediaPlayer *player) +QMaybe<QPlatformMediaPlayer *> QAndroidIntegration::createPlayer(QMediaPlayer *player) { return new QAndroidMediaPlayer(player); } -QPlatformCamera *QAndroidIntegration::createCamera(QCamera *camera) +QMaybe<QPlatformCamera *> QAndroidIntegration::createCamera(QCamera *camera) { return new QAndroidCamera(camera); } -QPlatformMediaRecorder *QAndroidIntegration::createRecorder(QMediaRecorder *recorder) +QMaybe<QPlatformMediaRecorder *> QAndroidIntegration::createRecorder(QMediaRecorder *recorder) { return new QAndroidMediaEncoder(recorder); } -QPlatformImageCapture *QAndroidIntegration::createImageCapture(QImageCapture *imageCapture) +QMaybe<QPlatformImageCapture *> QAndroidIntegration::createImageCapture(QImageCapture *imageCapture) { return new QAndroidImageCapture(imageCapture); } -QPlatformAudioOutput *QAndroidIntegration::createAudioOutput(QAudioOutput *q) +QMaybe<QPlatformAudioOutput *> QAndroidIntegration::createAudioOutput(QAudioOutput *q) { return new QAndroidAudioOutput(q); } -QPlatformAudioInput *QAndroidIntegration::createAudioInput(QAudioInput *audioInput) +QMaybe<QPlatformAudioInput *> QAndroidIntegration::createAudioInput(QAudioInput *audioInput) { return new QAndroidAudioInput(audioInput); } -QPlatformVideoSink *QAndroidIntegration::createVideoSink(QVideoSink *sink) +QMaybe<QPlatformVideoSink *> QAndroidIntegration::createVideoSink(QVideoSink *sink) { return new QAndroidVideoSink(sink); } diff --git a/src/plugins/multimedia/android/qandroidintegration_p.h b/src/plugins/multimedia/android/qandroidintegration_p.h index 3c6dde15f..9ef5a3267 100644 --- a/src/plugins/multimedia/android/qandroidintegration_p.h +++ b/src/plugins/multimedia/android/qandroidintegration_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QANDROIDINTEGRATION_H #define QANDROIDINTEGRATION_H @@ -61,24 +25,22 @@ class QAndroidIntegration : public QPlatformMediaIntegration { public: QAndroidIntegration(); - ~QAndroidIntegration(); - QPlatformMediaFormatInfo *formatInfo() override; + QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *decoder) override; + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *player) override; + QMaybe<QPlatformCamera *> createCamera(QCamera *camera) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *recorder) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *imageCapture) override; - QPlatformAudioDecoder *createAudioDecoder(QAudioDecoder *decoder) override; - QPlatformMediaCaptureSession *createCaptureSession() override; - QPlatformMediaPlayer *createPlayer(QMediaPlayer *player) override; - QPlatformCamera *createCamera(QCamera *camera) override; - QPlatformMediaRecorder *createRecorder(QMediaRecorder *recorder) override; - QPlatformImageCapture *createImageCapture(QImageCapture *imageCapture) override; + QMaybe<QPlatformAudioOutput *> createAudioOutput(QAudioOutput *q) override; + QMaybe<QPlatformAudioInput *> createAudioInput(QAudioInput *audioInput) override; - QPlatformAudioOutput *createAudioOutput(QAudioOutput *q) override; - QPlatformAudioInput *createAudioInput(QAudioInput *audioInput) override; - - QPlatformVideoSink *createVideoSink(QVideoSink *) override; + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *) override; QList<QCameraDevice> videoInputs() override; - QPlatformMediaFormatInfo *m_formatInfo = nullptr; +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/android/wrappers/jni/androidcamera.cpp b/src/plugins/multimedia/android/wrappers/jni/androidcamera.cpp index dbdffc8bb..cef36d7ad 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidcamera.cpp +++ b/src/plugins/multimedia/android/wrappers/jni/androidcamera.cpp @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Copyright (C) 2016 Ruslan Baratov -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// Copyright (C) 2016 Ruslan Baratov +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "androidcamera_p.h" #include "androidsurfacetexture_p.h" @@ -44,6 +8,8 @@ #include "qandroidmultimediautils_p.h" #include "qandroidglobal_p.h" +#include <private/qvideoframe_p.h> + #include <qhash.h> #include <qstringlist.h> #include <qdebug.h> @@ -57,7 +23,7 @@ QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(lcAndroidCamera, "qt.multimedia.android.camera") +static Q_LOGGING_CATEGORY(lcAndroidCamera, "qt.multimedia.android.camera") static const char QtCameraListenerClassName[] = "org/qtproject/qt/android/multimedia/QtCameraListener"; @@ -161,10 +127,9 @@ static void notifyPictureCaptured(JNIEnv *env, jobject, int id, jbyteArray data) bytesPerLine = -1; } - QVideoFrame frame(new QMemoryVideoBuffer(bytes, bytesPerLine), - QVideoFrameFormat(pictureSize, qt_pixelFormatFromAndroidImageFormat(format))); + auto pictureFormat = qt_pixelFormatFromAndroidImageFormat(format); - emit camera->pictureCaptured(frame); + emit camera->pictureCaptured(bytes, pictureFormat, pictureSize, bytesPerLine); } static void notifyNewPreviewFrame(JNIEnv *env, jobject, int id, jbyteArray data, @@ -182,9 +147,12 @@ static void notifyNewPreviewFrame(JNIEnv *env, jobject, int id, jbyteArray data, QByteArray bytes(arrayLength, Qt::Uninitialized); env->GetByteArrayRegion(data, 0, arrayLength, (jbyte*)bytes.data()); - QVideoFrame frame(new QMemoryVideoBuffer(bytes, bpl), - QVideoFrameFormat(QSize(width, height), - qt_pixelFormatFromAndroidImageFormat(AndroidCamera::ImageFormat(format)))); + QVideoFrameFormat frameFormat( + QSize(width, height), + qt_pixelFormatFromAndroidImageFormat(AndroidCamera::ImageFormat(format))); + + QVideoFrame frame = QVideoFramePrivate::createFrame( + std::make_unique<QMemoryVideoBuffer>(std::move(bytes), bpl), std::move(frameFormat)); Q_EMIT (*it)->newPreviewFrame(frame); } @@ -352,7 +320,7 @@ AndroidCamera::~AndroidCamera() AndroidCamera *AndroidCamera::open(int cameraId) { - if (!qt_androidRequestCameraPermission()) + if (!qt_androidCheckCameraPermission()) return nullptr; AndroidCameraPrivate *d = new AndroidCameraPrivate(); @@ -801,7 +769,7 @@ QJniObject AndroidCamera::getCameraObject() int AndroidCamera::getNumberOfCameras() { - if (!qt_androidRequestCameraPermission()) + if (!qt_androidCheckCameraPermission()) return 0; return QJniObject::callStaticMethod<jint>("android/hardware/Camera", @@ -837,6 +805,12 @@ void AndroidCamera::getCameraInfo(int id, QCameraDevicePrivate *info) default: break; } + // Add a number to allow correct access to cameras on systems with two + // (and more) front/back cameras + if (id > 1) { + info->id.append(QByteArray::number(id)); + info->description.append(QString(" %1").arg(id)); + } } QVideoFrameFormat::PixelFormat AndroidCamera::QtPixelFormatFromAndroidImageFormat(AndroidCamera::ImageFormat format) @@ -1235,6 +1209,7 @@ bool AndroidCameraPrivate::setPreviewDisplay(void *surfaceHolder) void AndroidCameraPrivate::setDisplayOrientation(int degrees) { m_camera.callMethod<void>("setDisplayOrientation", "(I)V", degrees); + m_cameraListener.callMethod<void>("setPhotoRotation", "(I)V", degrees); } bool AndroidCameraPrivate::isZoomSupported() @@ -1760,9 +1735,12 @@ void AndroidCameraPrivate::fetchLastPreviewFrame() const int format = m_cameraListener.callMethod<jint>("previewFormat"); const int bpl = m_cameraListener.callMethod<jint>("previewBytesPerLine"); - QVideoFrame frame(new QMemoryVideoBuffer(bytes, bpl), - QVideoFrameFormat(QSize(width, height), - qt_pixelFormatFromAndroidImageFormat(AndroidCamera::ImageFormat(format)))); + QVideoFrameFormat frameFormat( + QSize(width, height), + qt_pixelFormatFromAndroidImageFormat(AndroidCamera::ImageFormat(format))); + + QVideoFrame frame = QVideoFramePrivate::createFrame( + std::make_unique<QMemoryVideoBuffer>(std::move(bytes), bpl), std::move(frameFormat)); emit lastPreviewFrameFetched(frame); } @@ -1816,3 +1794,4 @@ bool AndroidCamera::registerNativeMethods() QT_END_NAMESPACE #include "androidcamera.moc" +#include "moc_androidcamera_p.cpp" diff --git a/src/plugins/multimedia/android/wrappers/jni/androidcamera_p.h b/src/plugins/multimedia/android/wrappers/jni/androidcamera_p.h index ff018335b..8375cf3b1 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidcamera_p.h +++ b/src/plugins/multimedia/android/wrappers/jni/androidcamera_p.h @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Copyright (C) 2016 Ruslan Baratov -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// Copyright (C) 2016 Ruslan Baratov +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef ANDROIDCAMERA_H #define ANDROIDCAMERA_H @@ -225,7 +189,7 @@ Q_SIGNALS: void takePictureFailed(); void pictureExposed(); - void pictureCaptured(const QVideoFrame &frame); + void pictureCaptured(const QByteArray &frame, QVideoFrameFormat::PixelFormat format, QSize size, int bytesPerLine); void lastPreviewFrameFetched(const QVideoFrame &frame); void newPreviewFrame(const QVideoFrame &frame); diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever.cpp b/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever.cpp index 733717ae7..25e1efdb0 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever.cpp +++ b/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "androidmediametadataretriever_p.h" @@ -160,7 +124,8 @@ bool AndroidMediaMetadataRetriever::setDataSource(const QUrl &url) auto methodId = env->GetMethodID(m_metadataRetriever.objectClass(), "setDataSource", "(Landroid/content/Context;Landroid/net/Uri;)V"); env->CallVoidMethod(m_metadataRetriever.object(), methodId, - QNativeInterface::QAndroidApplication::context(), uri.object()); + QNativeInterface::QAndroidApplication::context().object(), + uri.object()); if (env.checkAndClearExceptions()) return false; } diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever_p.h b/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever_p.h index 69727bbd3..68e346336 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever_p.h +++ b/src/plugins/multimedia/android/wrappers/jni/androidmediametadataretriever_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef ANDROIDMEDIAMETADATARETRIEVER_H #define ANDROIDMEDIAMETADATARETRIEVER_H diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer.cpp b/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer.cpp index 1378cfbeb..91f489f9e 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer.cpp +++ b/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "androidmediaplayer_p.h" #include "androidsurfacetexture_p.h" @@ -53,7 +17,7 @@ Q_GLOBAL_STATIC(QReadWriteLock, rwLock) QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(lcAudio, "qt.multimedia.audio") +static Q_LOGGING_CATEGORY(lcAudio, "qt.multimedia.audio") AndroidMediaPlayer::AndroidMediaPlayer() : QObject() @@ -63,7 +27,7 @@ AndroidMediaPlayer::AndroidMediaPlayer() const jlong id = reinterpret_cast<jlong>(this); mMediaPlayer = QJniObject(QtAndroidMediaPlayerClassName, "(Landroid/content/Context;J)V", - context, + context.object(), id); mediaPlayers->append(this); } @@ -288,6 +252,20 @@ void AndroidMediaPlayer::unblockAudio() mAudioBlocked = false; } +void AndroidMediaPlayer::startSoundStreaming(const int inputId, const int outputId) +{ + QJniObject::callStaticMethod<void>("org/qtproject/qt/android/multimedia/QtAudioDeviceManager", + "startSoundStreaming", + inputId, + outputId); +} + +void AndroidMediaPlayer::stopSoundStreaming() +{ + QJniObject::callStaticMethod<void>( + "org/qtproject/qt/android/multimedia/QtAudioDeviceManager", "stopSoundStreaming"); +} + bool AndroidMediaPlayer::setPlaybackRate(qreal rate) { if (QNativeInterface::QAndroidApplication::sdkVersion() < 23) { @@ -296,34 +274,7 @@ bool AndroidMediaPlayer::setPlaybackRate(qreal rate) return false; } - QJniObject player = mMediaPlayer.callObjectMethod("getMediaPlayerHandle", - "()Landroid/media/MediaPlayer;"); - if (player.isValid()) { - QJniObject playbackParams = player.callObjectMethod("getPlaybackParams", - "()Landroid/media/PlaybackParams;"); - if (playbackParams.isValid()) { - playbackParams.callObjectMethod("setSpeed", "(F)Landroid/media/PlaybackParams;", - jfloat(rate)); - // pitch can only be > 0 - if (!qFuzzyIsNull(rate)) - playbackParams.callObjectMethod("setPitch", "(F)Landroid/media/PlaybackParams;", - jfloat(qAbs(rate))); - - QJniEnvironment env; - auto methodId = env->GetMethodID(player.objectClass(), "setPlaybackParams", - "(Landroid/media/PlaybackParams;)V"); - env->CallVoidMethod(player.object(), methodId, playbackParams.object()); - - if (env.checkAndClearExceptions()) { - qWarning() << "Invalid playback rate" << rate; - return false; - } else { - return true; - } - } - } - - return false; + return mMediaPlayer.callMethod<jboolean>("setPlaybackRate", jfloat(rate)); } void AndroidMediaPlayer::setDisplay(AndroidSurfaceTexture *surfaceTexture) @@ -580,3 +531,5 @@ bool AndroidMediaPlayer::registerNativeMethods() } QT_END_NAMESPACE + +#include "moc_androidmediaplayer_p.cpp" diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer_p.h b/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer_p.h index d5cf07f9c..66095b114 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer_p.h +++ b/src/plugins/multimedia/android/wrappers/jni/androidmediaplayer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef ANDROIDMEDIAPLAYER_H #define ANDROIDMEDIAPLAYER_H @@ -136,6 +100,8 @@ public: void setDataSource(const QNetworkRequest &request); void prepareAsync(); void setVolume(int volume); + static void startSoundStreaming(const int inputId, const int outputId); + static void stopSoundStreaming(); bool setPlaybackRate(qreal rate); void setDisplay(AndroidSurfaceTexture *surfaceTexture); static bool setAudioOutput(const QByteArray &deviceId); diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder.cpp b/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder.cpp index 61b0f6df8..a3c9f4556 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder.cpp +++ b/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "androidmediarecorder_p.h" #include "androidcamera_p.h" @@ -51,7 +15,7 @@ QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(lcMediaRecorder, "qt.multimedia.mediarecorder.android") +static Q_LOGGING_CATEGORY(lcMediaRecorder, "qt.multimedia.mediarecorder.android") typedef QMap<QString, QJniObject> CamcorderProfiles; Q_GLOBAL_STATIC(CamcorderProfiles, g_camcorderProfiles) @@ -323,7 +287,7 @@ void AndroidMediaRecorder::setOutputFile(const QString &path) "org/qtproject/qt/android/QtNative", "openFdObjectForContentUrl", "(Landroid/content/Context;Ljava/lang/String;Ljava/lang/String;)Ljava/io/FileDescriptor;", - QNativeInterface::QAndroidApplication::context(), + QNativeInterface::QAndroidApplication::context().object(), QJniObject::fromString(path).object(), QJniObject::fromString(QLatin1String("rw")).object()); @@ -369,3 +333,5 @@ bool AndroidMediaRecorder::registerNativeMethods() } QT_END_NAMESPACE + +#include "moc_androidmediarecorder_p.cpp" diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder_p.h b/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder_p.h index 77fd3ff90..ffdbcc149 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder_p.h +++ b/src/plugins/multimedia/android/wrappers/jni/androidmediarecorder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef ANDROIDMEDIARECORDER_H #define ANDROIDMEDIARECORDER_H diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils.cpp b/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils.cpp index 1fe482f30..9606bd6bb 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils.cpp +++ b/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "androidmultimediautils_p.h" diff --git a/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils_p.h b/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils_p.h index 8a1fd7328..ee72c3c61 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils_p.h +++ b/src/plugins/multimedia/android/wrappers/jni/androidmultimediautils_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef ANDROIDMULTIMEDIAUTILS_H #define ANDROIDMULTIMEDIAUTILS_H diff --git a/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture.cpp b/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture.cpp index d3a5b56ff..c5860b265 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture.cpp +++ b/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "androidsurfacetexture_p.h" #include <QtCore/qmutex.h> @@ -48,6 +12,8 @@ typedef QList<jlong> SurfaceTextures; Q_GLOBAL_STATIC(SurfaceTextures, g_surfaceTextures); Q_GLOBAL_STATIC(QMutex, g_textureMutex); +static QAtomicInteger<quint64> indexCounter = 0u; + // native method for QtSurfaceTexture.java static void notifyFrameAvailable(JNIEnv* , jobject, jlong id) { @@ -63,6 +29,7 @@ static void notifyFrameAvailable(JNIEnv* , jobject, jlong id) AndroidSurfaceTexture::AndroidSurfaceTexture(quint32 texName) : QObject() + , m_index(indexCounter.fetchAndAddRelaxed(1)) { Q_STATIC_ASSERT(sizeof (jlong) >= sizeof (void *)); m_surfaceTexture = QJniObject("android/graphics/SurfaceTexture", "(I)V", jint(texName)); @@ -181,3 +148,5 @@ void AndroidSurfaceTexture::setOnFrameAvailableListener(const QJniObject &listen } QT_END_NAMESPACE + +#include "moc_androidsurfacetexture_p.cpp" diff --git a/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture_p.h b/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture_p.h index d31df972b..24581ca8d 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture_p.h +++ b/src/plugins/multimedia/android/wrappers/jni/androidsurfacetexture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef ANDROIDSURFACETEXTURE_H #define ANDROIDSURFACETEXTURE_H @@ -79,6 +43,7 @@ public: static bool registerNativeMethods(); + quint64 index() const { return m_index; } Q_SIGNALS: void frameAvailable(); @@ -88,6 +53,7 @@ private: QJniObject m_surfaceTexture; QJniObject m_surface; QJniObject m_surfaceHolder; + const quint64 m_index = 0; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview.cpp b/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview.cpp index 33d15a91b..dae9516c3 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview.cpp +++ b/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "androidsurfaceview_p.h" @@ -131,7 +95,7 @@ AndroidSurfaceView::AndroidSurfaceView() QNativeInterface::QAndroidApplication::runOnAndroidMainThread([this] { m_surfaceView = QJniObject("android/view/SurfaceView", "(Landroid/content/Context;)V", - QNativeInterface::QAndroidApplication::context()); + QNativeInterface::QAndroidApplication::context().object()); }).waitForFinished(); Q_ASSERT(m_surfaceView.isValid()); @@ -184,3 +148,5 @@ void AndroidSurfaceView::setGeometry(int x, int y, int width, int height) } QT_END_NAMESPACE + +#include "moc_androidsurfaceview_p.cpp" diff --git a/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview_p.h b/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview_p.h index db767069c..e6be60ef1 100644 --- a/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview_p.h +++ b/src/plugins/multimedia/android/wrappers/jni/androidsurfaceview_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef ANDROIDSURFACEVIEW_H #define ANDROIDSURFACEVIEW_H diff --git a/src/plugins/multimedia/darwin/CMakeLists.txt b/src/plugins/multimedia/darwin/CMakeLists.txt index a1a0cb13c..0bbc054eb 100644 --- a/src/plugins/multimedia/darwin/CMakeLists.txt +++ b/src/plugins/multimedia/darwin/CMakeLists.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + qt_internal_find_apple_system_framework(FWCoreMedia CoreMedia) # special case qt_internal_find_apple_system_framework(FWCoreAudio CoreAudio) # special case diff --git a/src/plugins/multimedia/darwin/avfaudiodecoder.mm b/src/plugins/multimedia/darwin/avfaudiodecoder.mm index 85ca7c273..3191b7db0 100644 --- a/src/plugins/multimedia/darwin/avfaudiodecoder.mm +++ b/src/plugins/multimedia/darwin/avfaudiodecoder.mm @@ -1,99 +1,37 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfaudiodecoder_p.h" #include <QtCore/qmutex.h> #include <QtCore/qiodevice.h> #include <QMimeDatabase> +#include <QThread> #include "private/qcoreaudioutils_p.h" +#include <QtCore/qloggingcategory.h> #include <AVFoundation/AVFoundation.h> -#define MAX_BUFFERS_IN_QUEUE 10 - QT_USE_NAMESPACE -@interface AVFResourceReaderDelegate : NSObject <AVAssetResourceLoaderDelegate> -{ - AVFAudioDecoder *m_decoder; - QMutex m_mutex; -} - --(void)handleNextSampleBuffer:(CMSampleBufferRef)sampleBuffer; - --(BOOL)resourceLoader:(AVAssetResourceLoader *)resourceLoader - shouldWaitForLoadingOfRequestedResource:(AVAssetResourceLoadingRequest *)loadingRequest; - -@end +static Q_LOGGING_CATEGORY(qLcAVFAudioDecoder, "qt.multimedia.darwin.AVFAudioDecoder") +constexpr static int MAX_BUFFERS_IN_QUEUE = 5; -@implementation AVFResourceReaderDelegate - --(id)initWithDecoder: (AVFAudioDecoder *)decoder { - if (!(self = [super init])) - return nil; - - m_decoder = decoder; - - return self; -} - --(void)dealloc { - m_decoder = nil; - [super dealloc]; -} - --(void)handleNextSampleBuffer:(CMSampleBufferRef)sampleBuffer +QAudioBuffer handleNextSampleBuffer(CMSampleBufferRef sampleBuffer) { if (!sampleBuffer) - return; + return {}; // Check format CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer); if (!formatDescription) - return; + return {}; const AudioStreamBasicDescription* const asbd = CMAudioFormatDescriptionGetStreamBasicDescription(formatDescription); QAudioFormat qtFormat = CoreAudioUtils::toQAudioFormat(*asbd); if (qtFormat.sampleFormat() == QAudioFormat::Unknown && asbd->mBitsPerChannel == 8) qtFormat.setSampleFormat(QAudioFormat::UInt8); if (!qtFormat.isValid()) - return; + return {}; // Get the required size to allocate to audioBufferList size_t audioBufferListSize = 0; @@ -106,7 +44,7 @@ QT_USE_NAMESPACE kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, NULL); if (err != noErr) - return; + return {}; CMBlockBufferRef blockBuffer = NULL; AudioBufferList* audioBufferList = (AudioBufferList*) malloc(audioBufferListSize); @@ -121,7 +59,7 @@ QT_USE_NAMESPACE &blockBuffer); if (err != noErr) { free(audioBufferList); - return; + return {}; } QByteArray abuf; @@ -137,12 +75,29 @@ QT_USE_NAMESPACE CMTime sampleStartTime = (CMSampleBufferGetPresentationTimeStamp(sampleBuffer)); float sampleStartTimeSecs = CMTimeGetSeconds(sampleStartTime); - QAudioBuffer audioBuffer; - audioBuffer = QAudioBuffer(abuf, qtFormat, qint64(sampleStartTimeSecs * 1000000)); - if (!audioBuffer.isValid()) - return; + return QAudioBuffer(abuf, qtFormat, qint64(sampleStartTimeSecs * 1000000)); +} + +@interface AVFResourceReaderDelegate : NSObject <AVAssetResourceLoaderDelegate> { + AVFAudioDecoder *m_decoder; + QMutex m_mutex; +} + +- (BOOL)resourceLoader:(AVAssetResourceLoader *)resourceLoader + shouldWaitForLoadingOfRequestedResource:(AVAssetResourceLoadingRequest *)loadingRequest; + +@end + +@implementation AVFResourceReaderDelegate + +- (id)initWithDecoder:(AVFAudioDecoder *)decoder +{ + if (!(self = [super init])) + return nil; + + m_decoder = decoder; - emit m_decoder->newAudioBuffer(audioBuffer); + return self; } -(BOOL)resourceLoader:(AVAssetResourceLoader *)resourceLoader @@ -226,6 +181,22 @@ QAudioFormat qt_format_for_audio_track(AVAssetTrack *track) } +struct AVFAudioDecoder::DecodingContext +{ + AVAssetReader *m_reader = nullptr; + AVAssetReaderTrackOutput *m_readerOutput = nullptr; + + ~DecodingContext() + { + if (m_reader) { + [m_reader cancelReading]; + [m_reader release]; + } + + [m_readerOutput release]; + } +}; + AVFAudioDecoder::AVFAudioDecoder(QAudioDecoder *parent) : QPlatformAudioDecoder(parent) { @@ -233,31 +204,17 @@ AVFAudioDecoder::AVFAudioDecoder(QAudioDecoder *parent) m_decodingQueue = dispatch_queue_create("decoder_queue", DISPATCH_QUEUE_SERIAL); m_readerDelegate = [[AVFResourceReaderDelegate alloc] initWithDecoder:this]; - - connect(this, &AVFAudioDecoder::readyToRead, this, &AVFAudioDecoder::startReading); - connect(this, &AVFAudioDecoder::newAudioBuffer, this, &AVFAudioDecoder::handleNewAudioBuffer); } AVFAudioDecoder::~AVFAudioDecoder() { stop(); - [m_readerOutput release]; - m_readerOutput = nil; - - [m_reader release]; - m_reader = nil; - [m_readerDelegate release]; - m_readerDelegate = nil; - [m_asset release]; - m_asset = nil; - if (m_readingQueue) - dispatch_release(m_readingQueue); - if (m_decodingQueue) - dispatch_release(m_decodingQueue); + dispatch_release(m_readingQueue); + dispatch_release(m_decodingQueue); } QUrl AVFAudioDecoder::source() const @@ -282,7 +239,7 @@ void AVFAudioDecoder::setSource(const QUrl &fileName) m_asset = [[AVURLAsset alloc] initWithURL:nsURL options:nil]; } - emit sourceChanged(); + sourceChanged(); } QIODevice *AVFAudioDecoder::sourceDevice() const @@ -309,76 +266,83 @@ void AVFAudioDecoder::setSourceDevice(QIODevice *device) NSURL *nsURL = [NSURL URLWithString:urlString]; m_asset = [[AVURLAsset alloc] initWithURL:nsURL options:nil]; - [m_asset.resourceLoader setDelegate:m_readerDelegate queue:m_readingQueue]; - m_loadingSource = true; + // use decoding queue instead of reading queue in order to fix random stucks. + // Anyway, decoding queue is empty in the moment. + [m_asset.resourceLoader setDelegate:m_readerDelegate queue:m_decodingQueue]; } - emit sourceChanged(); + sourceChanged(); } void AVFAudioDecoder::start() { - Q_ASSERT(!m_buffersAvailable); - if (isDecoding()) + if (m_decodingContext) { + qCDebug(qLcAVFAudioDecoder()) << "AVFAudioDecoder has been already started"; return; - - if (m_position != -1) { - m_position = -1; - emit positionChanged(-1); } + positionChanged(-1); + if (m_device && (!m_device->isOpen() || !m_device->isReadable())) { processInvalidMedia(QAudioDecoder::ResourceError, tr("Unable to read from specified device")); return; } - [m_asset loadValuesAsynchronouslyForKeys:@[@"tracks"] completionHandler: - ^{ - dispatch_async(m_readingQueue, - ^{ - NSError *error = nil; - AVKeyValueStatus status = [m_asset statusOfValueForKey:@"tracks" error:&error]; - if (status != AVKeyValueStatusLoaded) { - if (status == AVKeyValueStatusFailed) { - if (error.domain == NSURLErrorDomain) - processInvalidMedia(QAudioDecoder::ResourceError, QString::fromNSString(error.localizedDescription)); - else - processInvalidMedia(QAudioDecoder::FormatError, tr("Could not load media source's tracks")); - } - return; - } - initAssetReader(); - }); + m_decodingContext = std::make_shared<DecodingContext>(); + std::weak_ptr<DecodingContext> weakContext(m_decodingContext); + + auto handleLoadingResult = [=]() { + NSError *error = nil; + AVKeyValueStatus status = [m_asset statusOfValueForKey:@"tracks" error:&error]; + + if (status == AVKeyValueStatusFailed) { + if (error.domain == NSURLErrorDomain) + processInvalidMedia(QAudioDecoder::ResourceError, + QString::fromNSString(error.localizedDescription)); + else + processInvalidMedia(QAudioDecoder::FormatError, + tr("Could not load media source's tracks")); + } else if (status != AVKeyValueStatusLoaded) { + qWarning() << "Unexpected AVKeyValueStatus:" << status; + stop(); } - ]; + else { + initAssetReader(); + } + }; - if (m_device && m_loadingSource) { - setIsDecoding(true); - return; + [m_asset loadValuesAsynchronouslyForKeys:@[ @"tracks" ] + completionHandler:[=]() { + invokeWithDecodingContext(weakContext, handleLoadingResult); + }]; +} + +void AVFAudioDecoder::decBuffersCounter(uint val) +{ + if (val) { + QMutexLocker locker(&m_buffersCounterMutex); + m_buffersCounter -= val; } + + Q_ASSERT(m_buffersCounter >= 0); + + m_buffersCounterCondition.wakeAll(); } void AVFAudioDecoder::stop() { + qCDebug(qLcAVFAudioDecoder()) << "stop decoding"; + + m_decodingContext.reset(); + decBuffersCounter(m_cachedBuffers.size()); m_cachedBuffers.clear(); - if (m_reader) - [m_reader cancelReading]; + bufferAvailableChanged(false); + positionChanged(-1); + durationChanged(-1); - if (m_buffersAvailable != 0) { - m_buffersAvailable = 0; - emit bufferAvailableChanged(false); - } - if (m_position != -1) { - m_position = -1; - emit positionChanged(m_position); - } - if (m_duration != -1) { - m_duration = -1; - emit durationChanged(m_duration); - } - setIsDecoding(false); + onFinished(); } QAudioFormat AVFAudioDecoder::audioFormat() const @@ -390,140 +354,191 @@ void AVFAudioDecoder::setAudioFormat(const QAudioFormat &format) { if (m_format != format) { m_format = format; - emit formatChanged(m_format); + formatChanged(m_format); } } QAudioBuffer AVFAudioDecoder::read() { - if (!m_buffersAvailable) + if (m_cachedBuffers.empty()) return QAudioBuffer(); Q_ASSERT(m_cachedBuffers.size() > 0); - QAudioBuffer buffer = m_cachedBuffers.takeFirst(); + QAudioBuffer buffer = m_cachedBuffers.dequeue(); + decBuffersCounter(1); - m_position = qint64(buffer.startTime() / 1000); - emit positionChanged(m_position); - - m_buffersAvailable--; - if (!m_buffersAvailable) - emit bufferAvailableChanged(false); + positionChanged(buffer.startTime() / 1000); + bufferAvailableChanged(!m_cachedBuffers.empty()); return buffer; } -bool AVFAudioDecoder::bufferAvailable() const +void AVFAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, + const QString &errorString) { - return m_buffersAvailable > 0; -} + qCDebug(qLcAVFAudioDecoder()) << "Invalid media. Error code:" << errorCode + << "Description:" << errorString; -qint64 AVFAudioDecoder::position() const -{ - return m_position; -} + Q_ASSERT(QThread::currentThread() == thread()); -qint64 AVFAudioDecoder::duration() const -{ - return m_duration; + error(int(errorCode), errorString); + + // TODO: may be check if decodingCondext was changed by + // user's action (restart) from the emitted error. + // We should handle it somehow (don't run stop, print warning or etc...) + + stop(); } -void AVFAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString) +void AVFAudioDecoder::onFinished() { - stop(); - emit error(int(errorCode), errorString); + m_decodingContext.reset(); + + if (isDecoding()) + finished(); } void AVFAudioDecoder::initAssetReader() { - if (!m_asset) - return; + qCDebug(qLcAVFAudioDecoder()) << "Init asset reader"; + + Q_ASSERT(m_asset); + Q_ASSERT(QThread::currentThread() == thread()); NSArray<AVAssetTrack *> *tracks = [m_asset tracksWithMediaType:AVMediaTypeAudio]; if (!tracks.count) { processInvalidMedia(QAudioDecoder::FormatError, tr("No audio tracks found")); return; } - AVAssetTrack *track = [tracks objectAtIndex:0]; - // Set format - QAudioFormat format; - if (m_format.isValid()) { - format = m_format; - } else { - format = qt_format_for_audio_track(track); - if (!format.isValid()) - { - processInvalidMedia(QAudioDecoder::FormatError, tr("Unsupported source format")); - return; - } - } - - // Set duration - qint64 duration = CMTimeGetSeconds(track.timeRange.duration) * 1000; - if (m_duration != duration) { - m_duration = duration; - emit durationChanged(m_duration); + AVAssetTrack *track = [tracks objectAtIndex:0]; + QAudioFormat format = m_format.isValid() ? m_format : qt_format_for_audio_track(track); + if (!format.isValid()) { + processInvalidMedia(QAudioDecoder::FormatError, tr("Unsupported source format")); + return; } - // Initialize asset reader and output - [m_reader release]; - m_reader = nil; - [m_readerOutput release]; - m_readerOutput = nil; + durationChanged(CMTimeGetSeconds(track.timeRange.duration) * 1000); NSError *error = nil; NSDictionary *audioSettings = av_audio_settings_for_format(format); - m_readerOutput = [[AVAssetReaderTrackOutput alloc] initWithTrack:track outputSettings:audioSettings]; - m_reader = [[AVAssetReader alloc] initWithAsset:m_asset error:&error]; + + AVAssetReaderTrackOutput *readerOutput = + [[AVAssetReaderTrackOutput alloc] initWithTrack:track outputSettings:audioSettings]; + AVAssetReader *reader = [[AVAssetReader alloc] initWithAsset:m_asset error:&error]; if (error) { processInvalidMedia(QAudioDecoder::ResourceError, QString::fromNSString(error.localizedDescription)); return; } - if (![m_reader canAddOutput:m_readerOutput]) { + if (![reader canAddOutput:readerOutput]) { processInvalidMedia(QAudioDecoder::ResourceError, tr("Failed to add asset reader output")); return; } - [m_reader addOutput:m_readerOutput]; - emit readyToRead(); + [reader addOutput:readerOutput]; + + Q_ASSERT(m_decodingContext); + m_decodingContext->m_reader = reader; + m_decodingContext->m_readerOutput = readerOutput; + + startReading(); } void AVFAudioDecoder::startReading() { - m_loadingSource = false; + Q_ASSERT(m_decodingContext); + Q_ASSERT(m_decodingContext->m_reader); + Q_ASSERT(QThread::currentThread() == thread()); // Prepares the receiver for obtaining sample buffers from the asset. - if (!m_reader || ![m_reader startReading]) { + if (![m_decodingContext->m_reader startReading]) { processInvalidMedia(QAudioDecoder::ResourceError, tr("Could not start reading")); return; } setIsDecoding(true); + std::weak_ptr<DecodingContext> weakContext = m_decodingContext; + // Since copyNextSampleBuffer is synchronous, submit it to an async dispatch queue // to run in a separate thread. Call the handleNextSampleBuffer "callback" on another // thread when new audio sample is read. - dispatch_async(m_readingQueue, ^{ - CMSampleBufferRef sampleBuffer; - while ((sampleBuffer = [m_readerOutput copyNextSampleBuffer])) { - dispatch_async(m_decodingQueue, ^{ - if (CMSampleBufferDataIsReady(sampleBuffer)) - [m_readerDelegate handleNextSampleBuffer:sampleBuffer]; - CFRelease(sampleBuffer); - }); - } - if (m_reader.status == AVAssetReaderStatusCompleted) - emit finished(); + auto copyNextSampleBuffer = [=]() { + auto decodingContext = weakContext.lock(); + if (!decodingContext) + return false; + + CMSampleBufferRef sampleBuffer = [decodingContext->m_readerOutput copyNextSampleBuffer]; + if (!sampleBuffer) + return false; + + dispatch_async(m_decodingQueue, [=]() { + if (!weakContext.expired() && CMSampleBufferDataIsReady(sampleBuffer)) { + auto audioBuffer = handleNextSampleBuffer(sampleBuffer); + + if (audioBuffer.isValid()) + invokeWithDecodingContext(weakContext, + [=]() { handleNewAudioBuffer(audioBuffer); }); + } + + CFRelease(sampleBuffer); + }); + + return true; + }; + + dispatch_async(m_readingQueue, [=]() { + qCDebug(qLcAVFAudioDecoder()) << "start reading thread"; + + do { + // Note, waiting here doesn't ensure strong contol of the counter. + // However, it doesn't affect the logic: the reading flow works fine + // even if the counter is time-to-time more than max value + waitUntilBuffersCounterLessMax(); + } while (copyNextSampleBuffer()); + + // TODO: check m_reader.status == AVAssetReaderStatusFailed + invokeWithDecodingContext(weakContext, [this]() { onFinished(); }); }); } +void AVFAudioDecoder::waitUntilBuffersCounterLessMax() +{ + if (m_buffersCounter >= MAX_BUFFERS_IN_QUEUE) { + // the check avoids extra mutex lock. + + QMutexLocker locker(&m_buffersCounterMutex); + + while (m_buffersCounter >= MAX_BUFFERS_IN_QUEUE) + m_buffersCounterCondition.wait(&m_buffersCounterMutex); + } +} + void AVFAudioDecoder::handleNewAudioBuffer(QAudioBuffer buffer) { - Q_ASSERT(m_cachedBuffers.size() <= MAX_BUFFERS_IN_QUEUE); - m_cachedBuffers.push_back(buffer); + m_cachedBuffers.enqueue(buffer); + ++m_buffersCounter; + + Q_ASSERT(m_cachedBuffers.size() == m_buffersCounter); - m_buffersAvailable++; - Q_ASSERT(m_buffersAvailable <= MAX_BUFFERS_IN_QUEUE); + bufferAvailableChanged(true); + bufferReady(); +} - emit bufferAvailableChanged(true); - emit bufferReady(); +/* + * The method calls the passed functor in the thread of AVFAudioDecoder and guarantees that + * the passed decoding context is not expired. In other words, it helps avoiding all callbacks + * after stopping of the decoder. + */ +template<typename F> +void AVFAudioDecoder::invokeWithDecodingContext(std::weak_ptr<DecodingContext> weakContext, F &&f) +{ + if (!weakContext.expired()) + QMetaObject::invokeMethod(this, [=]() { + // strong check: compare with actual decoding context. + // Otherwise, the context can be temporary locked by one of dispatch queues. + if (auto context = weakContext.lock(); context && context == m_decodingContext) + f(); + }); } + +#include "moc_avfaudiodecoder_p.cpp" diff --git a/src/plugins/multimedia/darwin/avfaudiodecoder_p.h b/src/plugins/multimedia/darwin/avfaudiodecoder_p.h index 73a8cbd38..81ef3f49e 100644 --- a/src/plugins/multimedia/darwin/avfaudiodecoder_p.h +++ b/src/plugins/multimedia/darwin/avfaudiodecoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFAUDIODECODER_H #define AVFAUDIODECODER_H @@ -54,6 +18,9 @@ #include <QtMultimedia/private/qtmultimediaglobal_p.h> #include <QObject> #include <QtCore/qurl.h> +#include <QWaitCondition> +#include <QMutex> +#include <QQueue> #include "private/qplatformaudiodecoder_p.h" #include "qaudiodecoder.h" @@ -71,6 +38,8 @@ class AVFAudioDecoder : public QPlatformAudioDecoder { Q_OBJECT + struct DecodingContext; + public: AVFAudioDecoder(QAudioDecoder *parent); virtual ~AVFAudioDecoder(); @@ -88,41 +57,41 @@ public: void setAudioFormat(const QAudioFormat &format) override; QAudioBuffer read() override; - bool bufferAvailable() const override; - - qint64 position() const override; - qint64 duration() const override; -private slots: +private: void handleNewAudioBuffer(QAudioBuffer); void startReading(); -signals: - void newAudioBuffer(QAudioBuffer); - void readyToRead(); - -private: void processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString); void initAssetReader(); + void onFinished(); + + void waitUntilBuffersCounterLessMax(); + + void decBuffersCounter(uint val); + template<typename F> + void invokeWithDecodingContext(std::weak_ptr<DecodingContext> weakContext, F &&f); + +private: QUrl m_source; QIODevice *m_device = nullptr; QAudioFormat m_format; - int m_buffersAvailable = 0; - QList<QAudioBuffer> m_cachedBuffers; - - qint64 m_position = -1; - qint64 m_duration = -1; - - bool m_loadingSource = false; + // Use a separate counter instead of buffers queue size in order to + // ensure atomic access and also make mutex locking shorter + std::atomic<int> m_buffersCounter = 0; + QQueue<QAudioBuffer> m_cachedBuffers; AVURLAsset *m_asset = nullptr; - AVAssetReader *m_reader = nullptr; - AVAssetReaderTrackOutput *m_readerOutput = nullptr; + AVFResourceReaderDelegate *m_readerDelegate = nullptr; dispatch_queue_t m_readingQueue; dispatch_queue_t m_decodingQueue; + + std::shared_ptr<DecodingContext> m_decodingContext; + QMutex m_buffersCounterMutex; + QWaitCondition m_buffersCounterCondition; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/darwin/avfvideobuffer.mm b/src/plugins/multimedia/darwin/avfvideobuffer.mm index ea57051b0..0f5e3bcb3 100644 --- a/src/plugins/multimedia/darwin/avfvideobuffer.mm +++ b/src/plugins/multimedia/darwin/avfvideobuffer.mm @@ -1,46 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfvideobuffer_p.h" -#include <private/qrhi_p.h> -#include <private/qrhimetal_p.h> -#include <private/qrhigles2_p.h> +#include <rhi/qrhi.h> #include <CoreVideo/CVMetalTexture.h> #include <CoreVideo/CVMetalTextureCache.h> #include <QtGui/qopenglcontext.h> @@ -54,7 +16,8 @@ QT_USE_NAMESPACE AVFVideoBuffer::AVFVideoBuffer(AVFVideoSinkInterface *sink, CVImageBufferRef buffer) - : QAbstractVideoBuffer(sink->rhi() ? QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, sink->rhi()), + : QHwVideoBuffer(sink->rhi() ? QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, + sink->rhi()), sink(sink), m_buffer(buffer) { @@ -67,7 +30,7 @@ AVFVideoBuffer::AVFVideoBuffer(AVFVideoSinkInterface *sink, CVImageBufferRef buf AVFVideoBuffer::~AVFVideoBuffer() { - AVFVideoBuffer::unmap(); + Q_ASSERT(m_mode == QtVideo::MapMode::NotMapped); for (int i = 0; i < 3; ++i) if (cvMetalTexture[i]) CFRelease(cvMetalTexture[i]); @@ -81,33 +44,33 @@ AVFVideoBuffer::~AVFVideoBuffer() CVPixelBufferRelease(m_buffer); } -AVFVideoBuffer::MapData AVFVideoBuffer::map(QVideoFrame::MapMode mode) +AVFVideoBuffer::MapData AVFVideoBuffer::map(QtVideo::MapMode mode) { MapData mapData; - if (m_mode == QVideoFrame::NotMapped) { - CVPixelBufferLockBaseAddress(m_buffer, mode == QVideoFrame::ReadOnly + if (m_mode == QtVideo::MapMode::NotMapped) { + CVPixelBufferLockBaseAddress(m_buffer, mode == QtVideo::MapMode::ReadOnly ? kCVPixelBufferLock_ReadOnly : 0); m_mode = mode; } - mapData.nPlanes = CVPixelBufferGetPlaneCount(m_buffer); - Q_ASSERT(mapData.nPlanes <= 3); + mapData.planeCount = CVPixelBufferGetPlaneCount(m_buffer); + Q_ASSERT(mapData.planeCount <= 3); - if (!mapData.nPlanes) { + if (!mapData.planeCount) { // single plane mapData.bytesPerLine[0] = CVPixelBufferGetBytesPerRow(m_buffer); mapData.data[0] = static_cast<uchar*>(CVPixelBufferGetBaseAddress(m_buffer)); - mapData.size[0] = CVPixelBufferGetDataSize(m_buffer); - mapData.nPlanes = mapData.data[0] ? 1 : 0; + mapData.dataSize[0] = CVPixelBufferGetDataSize(m_buffer); + mapData.planeCount = mapData.data[0] ? 1 : 0; return mapData; } // For a bi-planar or tri-planar format we have to set the parameters correctly: - for (int i = 0; i < mapData.nPlanes; ++i) { + for (int i = 0; i < mapData.planeCount; ++i) { mapData.bytesPerLine[i] = CVPixelBufferGetBytesPerRowOfPlane(m_buffer, i); - mapData.size[i] = mapData.bytesPerLine[i]*CVPixelBufferGetHeightOfPlane(m_buffer, i); + mapData.dataSize[i] = mapData.bytesPerLine[i]*CVPixelBufferGetHeightOfPlane(m_buffer, i); mapData.data[i] = static_cast<uchar*>(CVPixelBufferGetBaseAddressOfPlane(m_buffer, i)); } @@ -116,11 +79,11 @@ AVFVideoBuffer::MapData AVFVideoBuffer::map(QVideoFrame::MapMode mode) void AVFVideoBuffer::unmap() { - if (m_mode != QVideoFrame::NotMapped) { - CVPixelBufferUnlockBaseAddress(m_buffer, m_mode == QVideoFrame::ReadOnly + if (m_mode != QtVideo::MapMode::NotMapped) { + CVPixelBufferUnlockBaseAddress(m_buffer, m_mode == QtVideo::MapMode::ReadOnly ? kCVPixelBufferLock_ReadOnly : 0); - m_mode = QVideoFrame::NotMapped; + m_mode = QtVideo::MapMode::NotMapped; } } @@ -155,7 +118,7 @@ static MTLPixelFormat rhiTextureFormatToMetalFormat(QRhiTexture::Format f) } -quint64 AVFVideoBuffer::textureHandle(int plane) const +quint64 AVFVideoBuffer::textureHandle(QRhi *, int plane) const { auto *textureDescription = QVideoTextureHelper::textureDescription(m_format.pixelFormat()); int bufferPlanes = CVPixelBufferGetPlaneCount(m_buffer); @@ -172,9 +135,16 @@ quint64 AVFVideoBuffer::textureHandle(int plane) const height = textureDescription->heightForPlane(height, plane); // Create a CoreVideo pixel buffer backed Metal texture image from the texture cache. + QMutexLocker locker(sink->textureCacheMutex()); + if (!metalCache && sink->cvMetalTextureCache) + metalCache = CVMetalTextureCacheRef(CFRetain(sink->cvMetalTextureCache)); + if (!metalCache) { + qWarning("cannot create texture, Metal texture cache was released?"); + return {}; + } auto ret = CVMetalTextureCacheCreateTextureFromImage( kCFAllocatorDefault, - sink->cvMetalTextureCache, + metalCache, m_buffer, nil, rhiTextureFormatToMetalFormat(textureDescription->textureFormat[plane]), width, height, diff --git a/src/plugins/multimedia/darwin/avfvideobuffer_p.h b/src/plugins/multimedia/darwin/avfvideobuffer_p.h index 1958b00e4..f70961c15 100644 --- a/src/plugins/multimedia/darwin/avfvideobuffer_p.h +++ b/src/plugins/multimedia/darwin/avfvideobuffer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFVIDEOBUFFER_H #define AVFVIDEOBUFFER_H @@ -51,8 +15,8 @@ // We mean it. // -#include <QtMultimedia/qvideoframe.h> -#include <private/qabstractvideobuffer_p.h> +#include <private/qhwvideobuffer_p.h> +#include <private/qcore_mac_p.h> #include <QtCore/qobject.h> #include <QtCore/qmutex.h> @@ -66,17 +30,16 @@ QT_BEGIN_NAMESPACE struct AVFMetalTexture; -class AVFVideoBuffer : public QAbstractVideoBuffer +class AVFVideoBuffer : public QHwVideoBuffer { public: AVFVideoBuffer(AVFVideoSinkInterface *sink, CVImageBufferRef buffer); ~AVFVideoBuffer(); - QVideoFrame::MapMode mapMode() const { return m_mode; } - MapData map(QVideoFrame::MapMode mode); + MapData map(QtVideo::MapMode mode); void unmap(); - virtual quint64 textureHandle(int plane) const; + virtual quint64 textureHandle(QRhi *, int plane) const; QVideoFrameFormat videoFormat() const { return m_format; } @@ -84,7 +47,7 @@ private: AVFVideoSinkInterface *sink = nullptr; mutable CVMetalTextureRef cvMetalTexture[3] = {}; - + mutable QCFType<CVMetalTextureCacheRef> metalCache; #if defined(Q_OS_MACOS) mutable CVOpenGLTextureRef cvOpenGLTexture = nullptr; #elif defined(Q_OS_IOS) @@ -92,7 +55,7 @@ private: #endif CVImageBufferRef m_buffer = nullptr; - QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped; + QtVideo::MapMode m_mode = QtVideo::MapMode::NotMapped; QVideoFrameFormat m_format; }; diff --git a/src/plugins/multimedia/darwin/avfvideosink.mm b/src/plugins/multimedia/darwin/avfvideosink.mm index f44a65683..f4c8bdb2e 100644 --- a/src/plugins/multimedia/darwin/avfvideosink.mm +++ b/src/plugins/multimedia/darwin/avfvideosink.mm @@ -1,57 +1,19 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfvideosink_p.h" -#include <private/qrhi_p.h> -#include <private/qrhimetal_p.h> -#include <private/qrhigles2_p.h> +#include <rhi/qrhi.h> #include <QtGui/qopenglcontext.h> #include <AVFoundation/AVFoundation.h> #import <QuartzCore/CATransaction.h> -#if QT_HAS_INCLUDE(<AppKit/AppKit.h>) +#if __has_include(<AppKit/AppKit.h>) #include <AppKit/AppKit.h> #endif -#if QT_HAS_INCLUDE(<UIKit/UIKit.h>) +#if __has_include(<UIKit/UIKit.h>) #include <UIKit/UIKit.h> #endif @@ -91,18 +53,12 @@ void AVFVideoSink::setVideoSinkInterface(AVFVideoSinkInterface *interface) m_interface->setRhi(m_rhi); } -// The OpengGL texture cache can apparently only handle single plane formats, so lets simply restrict to BGRA -static NSDictionary* const AVF_OUTPUT_SETTINGS_OPENGL = @{ - (NSString *)kCVPixelBufferPixelFormatTypeKey: @[ - @(kCVPixelFormatType_32BGRA), - ], - (NSString *)kCVPixelBufferOpenGLCompatibilityKey: @true -}; - AVFVideoSinkInterface::~AVFVideoSinkInterface() { if (m_layer) [m_layer release]; + if (m_outputSettings) + [m_outputSettings release]; freeTextureCaches(); } @@ -127,7 +83,11 @@ void AVFVideoSinkInterface::setVideoSink(AVFVideoSink *sink) if (sink == m_sink) return; + if (m_sink) + m_sink->setVideoSinkInterface(nullptr); + m_sink = sink; + if (m_sink) { m_sink->setVideoSinkInterface(this); reconfigure(); @@ -136,6 +96,7 @@ void AVFVideoSinkInterface::setVideoSink(AVFVideoSink *sink) void AVFVideoSinkInterface::setRhi(QRhi *rhi) { + QMutexLocker locker(&m_textureCacheMutex); if (m_rhi == rhi) return; freeTextureCaches(); @@ -159,8 +120,6 @@ void AVFVideoSinkInterface::setRhi(QRhi *rhi) } } else if (rhi->backend() == QRhi::OpenGLES2) { #if QT_CONFIG(opengl) - setOutputSettings(AVF_OUTPUT_SETTINGS_OPENGL); - #ifdef Q_OS_MACOS const auto *gl = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles()); @@ -195,6 +154,7 @@ void AVFVideoSinkInterface::setRhi(QRhi *rhi) m_rhi = nullptr; #endif // QT_CONFIG(opengl) } + setOutputSettings(); } void AVFVideoSinkInterface::setLayer(CALayer *layer) @@ -212,9 +172,46 @@ void AVFVideoSinkInterface::setLayer(CALayer *layer) reconfigure(); } -void AVFVideoSinkInterface::setOutputSettings(NSDictionary *settings) +void AVFVideoSinkInterface::setOutputSettings() { - m_outputSettings = settings; + if (m_outputSettings) + [m_outputSettings release]; + m_outputSettings = nil; + + // Set pixel format + NSDictionary *dictionary = nil; + if (m_rhi && m_rhi->backend() == QRhi::OpenGLES2) { +#if QT_CONFIG(opengl) + dictionary = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: + @(kCVPixelFormatType_32BGRA) +#ifndef Q_OS_IOS // On iOS this key generates a warning about unsupported key. + , (NSString *)kCVPixelBufferOpenGLCompatibilityKey: @true +#endif // Q_OS_IOS + }; +#endif + } else { + dictionary = @{(NSString *)kCVPixelBufferPixelFormatTypeKey: + @[ + @(kCVPixelFormatType_32BGRA), + @(kCVPixelFormatType_32RGBA), + @(kCVPixelFormatType_422YpCbCr8), + @(kCVPixelFormatType_422YpCbCr8_yuvs), + @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), + @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), + @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange), + @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange), + @(kCVPixelFormatType_OneComponent8), + @(kCVPixelFormatType_OneComponent16), + @(kCVPixelFormatType_420YpCbCr8Planar), + @(kCVPixelFormatType_420YpCbCr8PlanarFullRange) + ] +#ifndef Q_OS_IOS // This key is not supported and generates a warning. + , (NSString *)kCVPixelBufferMetalCompatibilityKey: @true +#endif // Q_OS_IOS + }; + } + + m_outputSettings = [[NSDictionary alloc] initWithDictionary:dictionary]; } void AVFVideoSinkInterface::updateLayerBounds() diff --git a/src/plugins/multimedia/darwin/avfvideosink_p.h b/src/plugins/multimedia/darwin/avfvideosink_p.h index 9191d19f1..9b66e79f2 100644 --- a/src/plugins/multimedia/darwin/avfvideosink_p.h +++ b/src/plugins/multimedia/darwin/avfvideosink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFVIDEOWINDOWCONTROL_H #define AVFVIDEOWINDOWCONTROL_H @@ -51,6 +15,7 @@ // We mean it. // +#include <QtCore/QMutex> #include "private/qplatformvideosink_p.h" Q_FORWARD_DECLARE_OBJC_CLASS(CALayer); @@ -100,7 +65,9 @@ public: virtual void reconfigure() = 0; virtual void setRhi(QRhi *); virtual void setLayer(CALayer *layer); - virtual void setOutputSettings(NSDictionary *settings); + virtual void setOutputSettings(); + + QMutex *textureCacheMutex() { return &m_textureCacheMutex; } QRhi *rhi() const { return m_rhi; } @@ -123,6 +90,7 @@ protected: QRhi *m_rhi = nullptr; CALayer *m_layer = nullptr; NSDictionary *m_outputSettings = nullptr; + QMutex m_textureCacheMutex; }; diff --git a/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate.mm b/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate.mm index 27261b8a0..1b2d4b15d 100644 --- a/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate.mm +++ b/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfcamerasession_p.h" #include "avfaudiopreviewdelegate_p.h" diff --git a/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate_p.h b/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate_p.h index e993637dd..8fa06ef39 100644 --- a/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate_p.h +++ b/src/plugins/multimedia/darwin/camera/avfaudiopreviewdelegate_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFAUDIOPREVIEWDELEGATE_P_H #define AVFAUDIOPREVIEWDELEGATE_P_H diff --git a/src/plugins/multimedia/darwin/camera/avfcamera.mm b/src/plugins/multimedia/darwin/camera/avfcamera.mm index 20f7eed65..05cdbae17 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamera.mm +++ b/src/plugins/multimedia/darwin/camera/avfcamera.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfcameradebug_p.h" #include "avfcamera_p.h" @@ -47,91 +11,6 @@ QT_USE_NAMESPACE - -namespace { - -// All these methods to work with exposure/ISO/SS in custom mode do not support macOS. - -#ifdef Q_OS_IOS - -// Misc. helpers to check values/ranges: - -bool qt_check_exposure_duration(AVCaptureDevice *captureDevice, CMTime duration) -{ - Q_ASSERT(captureDevice); - - AVCaptureDeviceFormat *activeFormat = captureDevice.activeFormat; - if (!activeFormat) { - qCDebug(qLcCamera) << Q_FUNC_INFO << "failed to obtain capture device format"; - return false; - } - - return CMTimeCompare(duration, activeFormat.minExposureDuration) != -1 - && CMTimeCompare(activeFormat.maxExposureDuration, duration) != -1; -} - -bool qt_check_ISO_value(AVCaptureDevice *captureDevice, int newISO) -{ - Q_ASSERT(captureDevice); - - AVCaptureDeviceFormat *activeFormat = captureDevice.activeFormat; - if (!activeFormat) { - qCDebug(qLcCamera) << Q_FUNC_INFO << "failed to obtain capture device format"; - return false; - } - - return !(newISO < activeFormat.minISO || newISO > activeFormat.maxISO); -} - -bool qt_exposure_duration_equal(AVCaptureDevice *captureDevice, qreal qDuration) -{ - Q_ASSERT(captureDevice); - const CMTime avDuration = CMTimeMakeWithSeconds(qDuration, captureDevice.exposureDuration.timescale); - return !CMTimeCompare(avDuration, captureDevice.exposureDuration); -} - -bool qt_iso_equal(AVCaptureDevice *captureDevice, int iso) -{ - Q_ASSERT(captureDevice); - return qFuzzyCompare(float(iso), captureDevice.ISO); -} - -bool qt_exposure_bias_equal(AVCaptureDevice *captureDevice, qreal bias) -{ - Q_ASSERT(captureDevice); - return qFuzzyCompare(bias, qreal(captureDevice.exposureTargetBias)); -} - -// Converters: - -bool qt_convert_exposure_mode(AVCaptureDevice *captureDevice, QCamera::ExposureMode mode, - AVCaptureExposureMode &avMode) -{ - // Test if mode supported and convert. - Q_ASSERT(captureDevice); - - if (mode == QCamera::ExposureAuto) { - if ([captureDevice isExposureModeSupported:AVCaptureExposureModeContinuousAutoExposure]) { - avMode = AVCaptureExposureModeContinuousAutoExposure; - return true; - } - } - - if (mode == QCamera::ExposureManual) { - if ([captureDevice isExposureModeSupported:AVCaptureExposureModeCustom]) { - avMode = AVCaptureExposureModeCustom; - return true; - } - } - - return false; -} - -#endif // defined(Q_OS_IOS) - -} // Unnamed namespace. - - AVFCamera::AVFCamera(QCamera *camera) : QAVFCameraBase(camera) { diff --git a/src/plugins/multimedia/darwin/camera/avfcamera_p.h b/src/plugins/multimedia/darwin/camera/avfcamera_p.h index 05faaab55..3c3e6da09 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamera_p.h +++ b/src/plugins/multimedia/darwin/camera/avfcamera_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFCAMERA_H #define AVFCAMERA_H diff --git a/src/plugins/multimedia/darwin/camera/avfcameradebug_p.h b/src/plugins/multimedia/darwin/camera/avfcameradebug_p.h index f52257bde..f93c85142 100644 --- a/src/plugins/multimedia/darwin/camera/avfcameradebug_p.h +++ b/src/plugins/multimedia/darwin/camera/avfcameradebug_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFDEBUG_H #define AVFDEBUG_H diff --git a/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm b/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm index 5d388359f..0c9ab3f2c 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm +++ b/src/plugins/multimedia/darwin/camera/avfcamerarenderer.mm @@ -1,43 +1,9 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "private/qabstractvideobuffer_p.h" +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qabstractvideobuffer.h" +#include "private/qcameradevice_p.h" +#include "private/qvideoframe_p.h" #include "avfcamerarenderer_p.h" #include "avfcamerasession_p.h" #include "avfcameraservice_p.h" @@ -48,7 +14,7 @@ #include "qvideosink.h" #include "qavfhelpers_p.h" -#include <QtGui/private/qrhi_p.h> +#include <rhi/qrhi.h> #import <AVFoundation/AVFoundation.h> @@ -56,8 +22,6 @@ #include <QtGui/qopengl.h> #endif -#include <private/qabstractvideobuffer_p.h> - #include <QtMultimedia/qvideoframeformat.h> QT_USE_NAMESPACE @@ -98,14 +62,13 @@ QT_USE_NAMESPACE // avfmediaassetwriter). CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); - AVFVideoBuffer *buffer = new AVFVideoBuffer(m_renderer, imageBuffer); + auto buffer = std::make_unique<AVFVideoBuffer>(m_renderer, imageBuffer); auto format = buffer->videoFormat(); if (!format.isValid()) { - delete buffer; return; } - QVideoFrame frame(buffer, format); + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(buffer), format); m_renderer->syncHandleViewfinderFrame(frame); } @@ -123,6 +86,8 @@ AVFCameraRenderer::~AVFCameraRenderer() { [m_cameraSession->captureSession() removeOutput:m_videoDataOutput]; [m_viewfinderFramesDelegate release]; + [m_videoDataOutput release]; + if (m_delegateQueue) dispatch_release(m_delegateQueue); #ifdef Q_OS_IOS @@ -142,13 +107,26 @@ void AVFCameraRenderer::reconfigure() deviceOrientationChanged(); } -void AVFCameraRenderer::setOutputSettings(NSDictionary *settings) +void AVFCameraRenderer::setOutputSettings() { if (!m_videoDataOutput) return; - m_videoDataOutput.videoSettings = settings; - AVFVideoSinkInterface::setOutputSettings(settings); + if (m_cameraSession) { + const auto format = m_cameraSession->cameraFormat(); + if (format.pixelFormat() != QVideoFrameFormat::Format_Invalid) + setPixelFormat(format.pixelFormat(), QCameraFormatPrivate::getColorRange(format)); + } + + // If no output settings set from above, + // it's most likely because the rhi is OpenGL + // and the pixel format is not BGRA. + // We force this in the base class implementation + if (!m_outputSettings) + AVFVideoSinkInterface::setOutputSettings(); + + if (m_outputSettings) + m_videoDataOutput.videoSettings = m_outputSettings; } void AVFCameraRenderer::configureAVCaptureSession(AVFCameraSession *cameraSession) @@ -159,7 +137,7 @@ void AVFCameraRenderer::configureAVCaptureSession(AVFCameraSession *cameraSessio m_needsHorizontalMirroring = false; - m_videoDataOutput = [[[AVCaptureVideoDataOutput alloc] init] autorelease]; + m_videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; // Configure video output m_delegateQueue = dispatch_queue_create("vf_queue", nullptr); @@ -260,16 +238,13 @@ void AVFCameraRenderer::handleViewfinderFrame() } if (m_sink && frame.isValid()) { - // ### pass format to surface - QVideoFrameFormat format = frame.surfaceFormat(); - if (m_needsHorizontalMirroring) - format.setMirrored(true); - + // frame.setMirroed(m_needsHorizontalMirroring) ? m_sink->setVideoFrame(frame); } } -void AVFCameraRenderer::setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat) +void AVFCameraRenderer::setPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat, + QVideoFrameFormat::ColorRange colorRange) { if (rhi() && rhi()->backend() == QRhi::OpenGLES2) { if (pixelFormat != QVideoFrameFormat::Format_BGRA8888) @@ -280,26 +255,34 @@ void AVFCameraRenderer::setPixelFormat(const QVideoFrameFormat::PixelFormat pixe // Default to 32BGRA pixel formats on the viewfinder, in case the requested // format can't be used (shouldn't happen unless the developers sets a wrong camera // format on the camera). - unsigned avPixelFormat = kCVPixelFormatType_32BGRA; - if (!QAVFHelpers::toCVPixelFormat(pixelFormat, avPixelFormat)) + auto cvPixelFormat = QAVFHelpers::toCVPixelFormat(pixelFormat, colorRange); + if (cvPixelFormat == CvPixelFormatInvalid) { + cvPixelFormat = kCVPixelFormatType_32BGRA; qWarning() << "QCamera::setCameraFormat: couldn't convert requested pixel format, using ARGB32"; + } bool isSupported = false; NSArray *supportedPixelFormats = m_videoDataOutput.availableVideoCVPixelFormatTypes; for (NSNumber *currentPixelFormat in supportedPixelFormats) { - if ([currentPixelFormat unsignedIntValue] == avPixelFormat) { + if ([currentPixelFormat unsignedIntValue] == cvPixelFormat) { isSupported = true; break; } } if (isSupported) { - NSDictionary* outputSettings = @{ - (NSString *)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:avPixelFormat], - (NSString *)kCVPixelBufferMetalCompatibilityKey: @true + NSDictionary *outputSettings = @{ + (NSString *) + kCVPixelBufferPixelFormatTypeKey : [NSNumber numberWithUnsignedInt:cvPixelFormat] +#ifndef Q_OS_IOS // On iOS this key generates a warning about 'unsupported key'. + , + (NSString *)kCVPixelBufferMetalCompatibilityKey : @true +#endif // Q_OS_IOS }; - setOutputSettings(outputSettings); + if (m_outputSettings) + [m_outputSettings release]; + m_outputSettings = [[NSDictionary alloc] initWithDictionary:outputSettings]; } else { qWarning() << "QCamera::setCameraFormat: requested pixel format not supported. Did you use a camera format from another camera?"; } diff --git a/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h b/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h index 8e583a6f9..57f665cd6 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h +++ b/src/plugins/multimedia/darwin/camera/avfcamerarenderer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFCAMERARENDERER_H #define AVFCAMERARENDERER_H @@ -85,7 +49,7 @@ public: ~AVFCameraRenderer(); void reconfigure() override; - void setOutputSettings(NSDictionary *settings) override; + void setOutputSettings() override; void configureAVCaptureSession(AVFCameraSession *cameraSession); void syncHandleViewfinderFrame(const QVideoFrame &frame); @@ -95,7 +59,8 @@ public: AVFCaptureFramesDelegate *captureDelegate() const; void resetCaptureDelegate() const; - void setPixelFormat(const QVideoFrameFormat::PixelFormat format); + void setPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat, + QVideoFrameFormat::ColorRange colorRange); Q_SIGNALS: void newViewfinderFrame(const QVideoFrame &frame); diff --git a/src/plugins/multimedia/darwin/camera/avfcameraservice.mm b/src/plugins/multimedia/darwin/camera/avfcameraservice.mm index 5c7cc5848..b25fb50a9 100644 --- a/src/plugins/multimedia/darwin/camera/avfcameraservice.mm +++ b/src/plugins/multimedia/darwin/camera/avfcameraservice.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <QtCore/qvariant.h> #include <QtCore/qdebug.h> @@ -77,8 +41,11 @@ void AVFCameraService::setCamera(QPlatformCamera *camera) if (m_cameraControl == control) return; - if (m_cameraControl) + if (m_cameraControl) { + if (m_encoder) + m_cameraControl->disconnect(m_encoder); m_cameraControl->setCaptureSession(nullptr); + } m_cameraControl = control; diff --git a/src/plugins/multimedia/darwin/camera/avfcameraservice_p.h b/src/plugins/multimedia/darwin/camera/avfcameraservice_p.h index 6b617d32c..f3ef8d08e 100644 --- a/src/plugins/multimedia/darwin/camera/avfcameraservice_p.h +++ b/src/plugins/multimedia/darwin/camera/avfcameraservice_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFCAMERASERVICE_H #define AVFCAMERASERVICE_H diff --git a/src/plugins/multimedia/darwin/camera/avfcamerasession.mm b/src/plugins/multimedia/darwin/camera/avfcamerasession.mm index bedd34f48..52e2eadfa 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamerasession.mm +++ b/src/plugins/multimedia/darwin/camera/avfcamerasession.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfcameradebug_p.h" #include "avfcamerasession_p.h" @@ -50,9 +14,11 @@ #include <CoreFoundation/CoreFoundation.h> #include <Foundation/Foundation.h> +#include <QtCore/qcoreapplication.h> #include <QtCore/qdatetime.h> #include <QtCore/qurl.h> #include <QtCore/qelapsedtimer.h> +#include <QtCore/qpermissions.h> #include <QtCore/qpointer.h> #include <private/qplatformaudioinput_p.h> @@ -180,8 +146,7 @@ void AVFCameraSession::setActiveCamera(const QCameraDevice &info) if (m_activeCameraDevice != info) { m_activeCameraDevice = info; - requestCameraPermissionIfNeeded(); - if (m_cameraAuthorizationStatus == AVAuthorizationStatusAuthorized) + if (checkCameraPermission()) updateVideoInput(); } } @@ -194,6 +159,11 @@ void AVFCameraSession::setCameraFormat(const QCameraFormat &format) updateCameraFormat(format); } +QCameraFormat AVFCameraSession::cameraFormat() const +{ + return m_cameraFormat; +} + void AVFCameraSession::updateCameraFormat(const QCameraFormat &format) { m_cameraFormat = format; @@ -203,11 +173,8 @@ void AVFCameraSession::updateCameraFormat(const QCameraFormat &format) return; AVCaptureDeviceFormat *newFormat = qt_convert_to_capture_device_format(captureDevice, format); - if (newFormat) { + if (newFormat) qt_set_active_format(captureDevice, newFormat, false); - if (m_videoOutput) - m_videoOutput->setPixelFormat(format.pixelFormat()); - } } void AVFCameraSession::setVideoOutput(AVFCameraRenderer *output) @@ -372,8 +339,7 @@ AVCaptureDevice *AVFCameraSession::createAudioCaptureDevice() void AVFCameraSession::attachVideoInputDevice() { - requestCameraPermissionIfNeeded(); - if (m_cameraAuthorizationStatus != AVAuthorizationStatusAuthorized) + if (!checkCameraPermission()) return; if (m_videoInput) { @@ -399,9 +365,6 @@ void AVFCameraSession::attachVideoInputDevice() void AVFCameraSession::attachAudioInputDevice() { - if (m_microphoneAuthorizationStatus != AVAuthorizationStatusAuthorized) - return; - if (m_audioInput) { [m_captureSession removeInput:m_audioInput]; [m_audioInput release]; @@ -484,8 +447,7 @@ void AVFCameraSession::updateVideoInput() void AVFCameraSession::updateAudioInput() { - requestMicrophonePermissionIfNeeded(); - if (m_microphoneAuthorizationStatus != AVAuthorizationStatusAuthorized) + if (!checkMicrophonePermission()) return; auto recorder = m_service->recorderControl(); @@ -528,93 +490,24 @@ void AVFCameraSession::updateVideoOutput() m_videoOutput->setVideoSink(m_videoSink); } -void AVFCameraSession::requestCameraPermissionIfNeeded() -{ - if (m_cameraAuthorizationStatus == AVAuthorizationStatusAuthorized) - return; - - switch ([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]) - { - case AVAuthorizationStatusAuthorized: - { - m_cameraAuthorizationStatus = AVAuthorizationStatusAuthorized; - break; - } - case AVAuthorizationStatusNotDetermined: - { - m_cameraAuthorizationStatus = AVAuthorizationStatusNotDetermined; - QPointer<AVFCameraSession> guard(this); - [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL granted) { - dispatch_async(dispatch_get_main_queue(), ^{ - if (guard) - cameraAuthorizationChanged(granted); - }); - }]; - break; - } - case AVAuthorizationStatusDenied: - case AVAuthorizationStatusRestricted: - { - m_cameraAuthorizationStatus = AVAuthorizationStatusDenied; - return; - } - } -} - -void AVFCameraSession::requestMicrophonePermissionIfNeeded() +bool AVFCameraSession::checkCameraPermission() { - if (m_microphoneAuthorizationStatus == AVAuthorizationStatusAuthorized) - return; + const QCameraPermission permission; + const bool granted = qApp->checkPermission(permission) == Qt::PermissionStatus::Granted; + if (!granted) + qWarning() << "Access to camera not granted"; - switch ([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeAudio]) - { - case AVAuthorizationStatusAuthorized: - { - m_microphoneAuthorizationStatus = AVAuthorizationStatusAuthorized; - break; - } - case AVAuthorizationStatusNotDetermined: - { - m_microphoneAuthorizationStatus = AVAuthorizationStatusNotDetermined; - QPointer<AVFCameraSession> guard(this); - [AVCaptureDevice requestAccessForMediaType:AVMediaTypeAudio completionHandler:^(BOOL granted) { - dispatch_async(dispatch_get_main_queue(), ^{ - if (guard) - microphoneAuthorizationChanged(granted); - }); - }]; - break; - } - case AVAuthorizationStatusDenied: - case AVAuthorizationStatusRestricted: - { - m_microphoneAuthorizationStatus = AVAuthorizationStatusDenied; - return; - } - } + return granted; } -void AVFCameraSession::cameraAuthorizationChanged(bool authorized) +bool AVFCameraSession::checkMicrophonePermission() { - if (authorized) { - m_cameraAuthorizationStatus = AVAuthorizationStatusAuthorized; - updateVideoInput(); - updateCameraFormat(m_cameraFormat); - } else { - m_cameraAuthorizationStatus = AVAuthorizationStatusDenied; - qWarning() << "User has denied access to camera"; - } -} + const QMicrophonePermission permission; + const bool granted = qApp->checkPermission(permission) == Qt::PermissionStatus::Granted; + if (!granted) + qWarning() << "Access to microphone not granted"; -void AVFCameraSession::microphoneAuthorizationChanged(bool authorized) -{ - if (authorized) { - m_microphoneAuthorizationStatus = AVAuthorizationStatusAuthorized; - updateAudioInput(); - } else { - m_microphoneAuthorizationStatus = AVAuthorizationStatusDenied; - qWarning() << "User has denied access to microphone"; - } + return granted; } #include "moc_avfcamerasession_p.cpp" diff --git a/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h b/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h index c1a5e69a4..76e31ab48 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h +++ b/src/plugins/multimedia/darwin/camera/avfcamerasession_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFCAMERASESSION_H #define AVFCAMERASESSION_H @@ -80,6 +44,7 @@ public: void setActiveCamera(const QCameraDevice &info); void setCameraFormat(const QCameraFormat &format); + QCameraFormat cameraFormat() const; AVFCameraRenderer *videoOutput() const { return m_videoOutput; } AVCaptureAudioDataOutput *audioOutput() const { return m_audioOutput; } @@ -115,9 +80,6 @@ public Q_SLOTS: void processSessionStarted(); void processSessionStopped(); - void cameraAuthorizationChanged(bool authorized); - void microphoneAuthorizationChanged(bool authorized); - Q_SIGNALS: void readyToConfigureConnections(); void activeChanged(bool); @@ -136,8 +98,8 @@ private: AVCaptureDevice *createAudioCaptureDevice(); void attachVideoInputDevice(); void attachAudioInputDevice(); - void requestCameraPermissionIfNeeded(); - void requestMicrophonePermissionIfNeeded(); + bool checkCameraPermission(); + bool checkMicrophonePermission(); bool applyImageEncoderSettings(); @@ -163,9 +125,6 @@ private: bool m_inputMuted = false; FourCharCode m_defaultCodec; - - AVAuthorizationStatus m_cameraAuthorizationStatus = AVAuthorizationStatusNotDetermined; - AVAuthorizationStatus m_microphoneAuthorizationStatus = AVAuthorizationStatusNotDetermined; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/darwin/camera/avfcamerautility.mm b/src/plugins/multimedia/darwin/camera/avfcamerautility.mm index 614b99e06..1864eb0e8 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamerautility.mm +++ b/src/plugins/multimedia/darwin/camera/avfcamerautility.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfcamerautility_p.h" #include "avfcameradebug_p.h" @@ -43,6 +7,7 @@ #include <QtCore/qvector.h> #include <QtCore/qpair.h> #include <private/qmultimediautils_p.h> +#include <private/qcameradevice_p.h> #include "avfvideobuffer_p.h" #include "qavfhelpers_p.h" @@ -105,9 +70,9 @@ struct ByResolution } }; -struct FormatHasNoFPSRange : std::unary_function<AVCaptureDeviceFormat *, bool> +struct FormatHasNoFPSRange { - bool operator() (AVCaptureDeviceFormat *format) + bool operator() (AVCaptureDeviceFormat *format) const { Q_ASSERT(format); return !format.videoSupportedFrameRateRanges || !format.videoSupportedFrameRateRanges.count; @@ -131,28 +96,47 @@ Float64 qt_find_min_framerate_distance(AVCaptureDeviceFormat *format, Float64 fp } // Unnamed namespace. -AVCaptureDeviceFormat *qt_convert_to_capture_device_format(AVCaptureDevice *captureDevice, - const QCameraFormat &cameraFormat) +AVCaptureDeviceFormat * +qt_convert_to_capture_device_format(AVCaptureDevice *captureDevice, + const QCameraFormat &cameraFormat, + const std::function<bool(uint32_t)> &cvFormatValidator) { + const auto cameraFormatPrivate = QCameraFormatPrivate::handle(cameraFormat); + if (!cameraFormatPrivate) + return nil; + + const auto requiredCvPixFormat = QAVFHelpers::toCVPixelFormat(cameraFormatPrivate->pixelFormat, + cameraFormatPrivate->colorRange); + + if (requiredCvPixFormat == CvPixelFormatInvalid) + return nil; + AVCaptureDeviceFormat *newFormat = nil; + Float64 newFormatMaxFrameRate = {}; NSArray<AVCaptureDeviceFormat *> *formats = captureDevice.formats; for (AVCaptureDeviceFormat *format in formats) { CMFormatDescriptionRef formatDesc = format.formatDescription; CMVideoDimensions dim = CMVideoFormatDescriptionGetDimensions(formatDesc); - FourCharCode formatCodec = CMVideoFormatDescriptionGetCodecType(formatDesc); - if (QAVFHelpers::fromCVPixelFormat(formatCodec) == cameraFormat.pixelFormat() - && cameraFormat.resolution().width() == dim.width - && cameraFormat.resolution().height() == dim.height) { - for (AVFrameRateRange *frameRateRange in format.videoSupportedFrameRateRanges) { - if (frameRateRange.minFrameRate >= cameraFormat.minFrameRate() - && frameRateRange.maxFrameRate <= cameraFormat.maxFrameRate()) { - newFormat = format; - break; - } + FourCharCode cvPixFormat = CMVideoFormatDescriptionGetCodecType(formatDesc); + + if (cvPixFormat != requiredCvPixFormat) + continue; + + if (cameraFormatPrivate->resolution != QSize(dim.width, dim.height)) + continue; + + if (cvFormatValidator && !cvFormatValidator(cvPixFormat)) + continue; + + const float epsilon = 0.001f; + for (AVFrameRateRange *frameRateRange in format.videoSupportedFrameRateRanges) { + if (frameRateRange.minFrameRate >= cameraFormatPrivate->minFrameRate - epsilon + && frameRateRange.maxFrameRate <= cameraFormatPrivate->maxFrameRate + epsilon + && newFormatMaxFrameRate < frameRateRange.maxFrameRate) { + newFormat = format; + newFormatMaxFrameRate = frameRateRange.maxFrameRate; } } - if (newFormat) - break; } return newFormat; } @@ -258,13 +242,10 @@ QSize qt_device_format_pixel_aspect_ratio(AVCaptureDeviceFormat *format) if (!res.width || !resPAR.width) return QSize(); - int n, d; - qt_real_to_fraction(resPAR.width > res.width - ? res.width / qreal(resPAR.width) - : resPAR.width / qreal(res.width), - &n, &d); + auto frac = qRealToFraction(resPAR.width > res.width ? res.width / qreal(resPAR.width) + : resPAR.width / qreal(res.width)); - return QSize(n, d); + return QSize(frac.numerator, frac.denominator); } AVCaptureDeviceFormat *qt_find_best_resolution_match(AVCaptureDevice *captureDevice, @@ -520,9 +501,8 @@ CMTime qt_adjusted_frame_duration(AVFrameRateRange *range, qreal fps) if (fps >= range.maxFrameRate) return range.minFrameDuration; - int n, d; - qt_real_to_fraction(1. / fps, &n, &d); - return CMTimeMake(n, d); + auto frac = qRealToFraction(1. / fps); + return CMTimeMake(frac.numerator, frac.denominator); } void qt_set_framerate_limits(AVCaptureDevice *captureDevice, qreal minFPS, qreal maxFPS) diff --git a/src/plugins/multimedia/darwin/camera/avfcamerautility_p.h b/src/plugins/multimedia/darwin/camera/avfcamerautility_p.h index cdfff4905..b5c9e9bda 100644 --- a/src/plugins/multimedia/darwin/camera/avfcamerautility_p.h +++ b/src/plugins/multimedia/darwin/camera/avfcamerautility_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFCAMERAUTILITY_H #define AVFCAMERAUTILITY_H @@ -167,8 +131,9 @@ private: typedef QPair<qreal, qreal> AVFPSRange; AVFPSRange qt_connection_framerates(AVCaptureConnection *videoConnection); -AVCaptureDeviceFormat *qt_convert_to_capture_device_format(AVCaptureDevice *captureDevice, - const QCameraFormat &format); +AVCaptureDeviceFormat *qt_convert_to_capture_device_format( + AVCaptureDevice *captureDevice, const QCameraFormat &format, + const std::function<bool(uint32_t)> &cvFormatValidator = nullptr); QList<AVCaptureDeviceFormat *> qt_unique_device_formats(AVCaptureDevice *captureDevice, FourCharCode preferredFormat); QSize qt_device_format_resolution(AVCaptureDeviceFormat *format); diff --git a/src/plugins/multimedia/darwin/camera/avfimagecapture.mm b/src/plugins/multimedia/darwin/camera/avfimagecapture.mm index 8aab1b605..2ee7b0597 100644 --- a/src/plugins/multimedia/darwin/camera/avfimagecapture.mm +++ b/src/plugins/multimedia/darwin/camera/avfimagecapture.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfcameradebug_p.h" #include "avfimagecapture_p.h" @@ -47,6 +11,7 @@ #include "private/qmediastoragelocation_p.h" #include <private/qplatformimagecapture_p.h> #include <private/qmemoryvideobuffer_p.h> +#include <private/qvideoframe_p.h> #include <QtCore/qurl.h> #include <QtCore/qfile.h> @@ -154,8 +119,10 @@ int AVFImageCapture::doCapture(const QString &actualFileName) QBuffer data(&jpgData); QImageReader reader(&data, "JPEG"); QSize size = reader.size(); - QVideoFrame frame(new QMemoryVideoBuffer(QByteArray(jpgData.constData(), jpgData.size()), -1), - QVideoFrameFormat(size, QVideoFrameFormat::Format_Jpeg)); + auto buffer = std::make_unique<QMemoryVideoBuffer>( + QByteArray(jpgData.constData(), jpgData.size()), -1); + QVideoFrame frame = QVideoFramePrivate::createFrame( + std::move(buffer), QVideoFrameFormat(size, QVideoFrameFormat::Format_Jpeg)); QMetaObject::invokeMethod(this, "imageAvailable", Qt::QueuedConnection, Q_ARG(int, request.captureId), Q_ARG(QVideoFrame, frame)); diff --git a/src/plugins/multimedia/darwin/camera/avfimagecapture_p.h b/src/plugins/multimedia/darwin/camera/avfimagecapture_p.h index ef11bd70f..0714fa3cc 100644 --- a/src/plugins/multimedia/darwin/camera/avfimagecapture_p.h +++ b/src/plugins/multimedia/darwin/camera/avfimagecapture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFCAMERAIMAGECAPTURE_H #define AVFCAMERAIMAGECAPTURE_H diff --git a/src/plugins/multimedia/darwin/camera/avfmediaassetwriter.mm b/src/plugins/multimedia/darwin/camera/avfmediaassetwriter.mm index 1cd9d4cf0..37fc69926 100644 --- a/src/plugins/multimedia/darwin/camera/avfmediaassetwriter.mm +++ b/src/plugins/multimedia/darwin/camera/avfmediaassetwriter.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfmediaencoder_p.h" #include "avfcamerarenderer_p.h" diff --git a/src/plugins/multimedia/darwin/camera/avfmediaassetwriter_p.h b/src/plugins/multimedia/darwin/camera/avfmediaassetwriter_p.h index 6691b66a1..8fe3e8522 100644 --- a/src/plugins/multimedia/darwin/camera/avfmediaassetwriter_p.h +++ b/src/plugins/multimedia/darwin/camera/avfmediaassetwriter_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFMEDIAASSETWRITER_H #define AVFMEDIAASSETWRITER_H diff --git a/src/plugins/multimedia/darwin/camera/avfmediaencoder.mm b/src/plugins/multimedia/darwin/camera/avfmediaencoder.mm index ba40393e8..3fbc57995 100644 --- a/src/plugins/multimedia/darwin/camera/avfmediaencoder.mm +++ b/src/plugins/multimedia/darwin/camera/avfmediaencoder.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfmediaencoder_p.h" @@ -52,11 +16,14 @@ #include "private/qmediarecorder_p.h" #include "qdarwinformatsinfo_p.h" #include "private/qplatformaudiooutput_p.h" +#include <private/qplatformaudioinput_p.h> #include <QtCore/qmath.h> #include <QtCore/qdebug.h> #include <QtCore/qmimetype.h> +#include <private/qcoreaudioutils_p.h> + QT_USE_NAMESPACE namespace { @@ -135,7 +102,7 @@ void AVFMediaEncoder::updateDuration(qint64 duration) durationChanged(m_duration); } -static NSDictionary *avfAudioSettings(const QMediaEncoderSettings &encoderSettings) +static NSDictionary *avfAudioSettings(const QMediaEncoderSettings &encoderSettings, const QAudioFormat &format) { NSMutableDictionary *settings = [NSMutableDictionary dictionary]; @@ -219,22 +186,36 @@ static NSDictionary *avfAudioSettings(const QMediaEncoderSettings &encoderSettin } } } + + // if channel count is provided and it's bigger than 2 + // provide a supported channel layout + if (isChannelCountSupported && channelCount > 2) { + AudioChannelLayout channelLayout; + memset(&channelLayout, 0, sizeof(AudioChannelLayout)); + auto channelLayoutTags = qt_supported_channel_layout_tags_for_format(codecId, channelCount); + if (channelLayoutTags.size()) { + channelLayout.mChannelLayoutTag = channelLayoutTags.first(); + [settings setObject:[NSData dataWithBytes: &channelLayout length: sizeof(channelLayout)] forKey:AVChannelLayoutKey]; + } else { + isChannelCountSupported = false; + } + } + + if (isChannelCountSupported) + [settings setObject:[NSNumber numberWithInt:channelCount] forKey:AVNumberOfChannelsKey]; } - if (isChannelCountSupported && channelCount > 2) { - AudioChannelLayout channelLayout; - memset(&channelLayout, 0, sizeof(AudioChannelLayout)); - auto channelLayoutTags = qt_supported_channel_layout_tags_for_format(codecId, channelCount); - if (channelLayoutTags.size()) { - channelLayout.mChannelLayoutTag = channelLayoutTags.first(); - [settings setObject:[NSData dataWithBytes: &channelLayout length: sizeof(channelLayout)] forKey:AVChannelLayoutKey]; + if (!isChannelCountSupported) { + // fallback to providing channel layout if channel count is not specified or supported + UInt32 size = 0; + if (format.isValid()) { + auto layout = CoreAudioUtils::toAudioChannelLayout(format, &size); + [settings setObject:[NSData dataWithBytes:layout.get() length:sizeof(AudioChannelLayout)] forKey:AVChannelLayoutKey]; } else { - isChannelCountSupported = false; + // finally default to setting channel count to 1 + [settings setObject:[NSNumber numberWithInt:1] forKey:AVNumberOfChannelsKey]; } } - if (!isChannelCountSupported) - channelCount = 2; - [settings setObject:[NSNumber numberWithInt:channelCount] forKey:AVNumberOfChannelsKey]; if (codecId == kAudioFormatAppleLossless) [settings setObject:[NSNumber numberWithInt:24] forKey:AVEncoderBitDepthHintKey]; @@ -421,7 +402,9 @@ void AVFMediaEncoder::applySettings(QMediaEncoderSettings &settings) AVFCameraSession *session = m_service->session(); // audio settings - m_audioSettings = avfAudioSettings(settings); + const auto audioInput = m_service->audioInput(); + const QAudioFormat audioFormat = audioInput ? audioInput->device.preferredFormat() : QAudioFormat(); + m_audioSettings = avfAudioSettings(settings, audioFormat); if (m_audioSettings) [m_audioSettings retain]; @@ -496,7 +479,7 @@ void AVFMediaEncoder::record(QMediaEncoderSettings &settings) if (!cameraControl && !audioInput) { qWarning() << Q_FUNC_INFO << "Cannot record without any inputs"; - Q_EMIT error(QMediaRecorder::ResourceError, tr("No inputs specified")); + updateError(QMediaRecorder::ResourceError, tr("No inputs specified")); return; } @@ -508,8 +491,8 @@ void AVFMediaEncoder::record(QMediaEncoderSettings &settings) if (!audioOnly) { if (!cameraControl || !cameraControl->isActive()) { qCDebug(qLcCamera) << Q_FUNC_INFO << "can not start record while camera is not active"; - Q_EMIT error(QMediaRecorder::ResourceError, - QMediaRecorderPrivate::msgFailedStartRecording()); + updateError(QMediaRecorder::ResourceError, + QMediaRecorderPrivate::msgFailedStartRecording()); return; } } @@ -523,13 +506,13 @@ void AVFMediaEncoder::record(QMediaEncoderSettings &settings) NSURL *nsFileURL = fileURL.toNSURL(); if (!nsFileURL) { qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL; - Q_EMIT error(QMediaRecorder::ResourceError, tr("Invalid output file URL")); + updateError(QMediaRecorder::ResourceError, tr("Invalid output file URL")); return; } if (!qt_is_writable_file_URL(nsFileURL)) { qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL << "(the location is not writable)"; - Q_EMIT error(QMediaRecorder::ResourceError, tr("Non-writeable file location")); + updateError(QMediaRecorder::ResourceError, tr("Non-writeable file location")); return; } if (qt_file_exists(nsFileURL)) { @@ -537,7 +520,7 @@ void AVFMediaEncoder::record(QMediaEncoderSettings &settings) // Objective-C exception, which is not good at all. qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL << "(file already exists)"; - Q_EMIT error(QMediaRecorder::ResourceError, tr("File already exists")); + updateError(QMediaRecorder::ResourceError, tr("File already exists")); return; } @@ -572,8 +555,7 @@ void AVFMediaEncoder::record(QMediaEncoderSettings &settings) [m_writer start]; } else { [session startRunning]; - Q_EMIT error(QMediaRecorder::FormatError, - QMediaRecorderPrivate::msgFailedStartRecording()); + updateError(QMediaRecorder::FormatError, QMediaRecorderPrivate::msgFailedStartRecording()); } } @@ -638,7 +620,8 @@ void AVFMediaEncoder::assetWriterFinished() if (session->audioPreviewDelegate()) { [session->audioPreviewDelegate() resetAudioPreviewDelegate]; } - [session->captureSession() startRunning]; + if (session->videoOutput() || session->audioPreviewDelegate()) + [session->captureSession() startRunning]; } m_state = QMediaRecorder::StoppedState; @@ -648,7 +631,7 @@ void AVFMediaEncoder::assetWriterFinished() void AVFMediaEncoder::assetWriterError(QString err) { - Q_EMIT error(QMediaRecorder::FormatError, err); + updateError(QMediaRecorder::FormatError, err); if (m_state != QMediaRecorder::StoppedState) stopWriter(); } diff --git a/src/plugins/multimedia/darwin/camera/avfmediaencoder_p.h b/src/plugins/multimedia/darwin/camera/avfmediaencoder_p.h index eac5a0282..23aced325 100644 --- a/src/plugins/multimedia/darwin/camera/avfmediaencoder_p.h +++ b/src/plugins/multimedia/darwin/camera/avfmediaencoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFMEDIAENCODER_H #define AVFMEDIAENCODER_H diff --git a/src/plugins/multimedia/darwin/camera/qavfcamerabase.mm b/src/plugins/multimedia/darwin/camera/qavfcamerabase.mm index db7f4f8d3..11dfa99a8 100644 --- a/src/plugins/multimedia/darwin/camera/qavfcamerabase.mm +++ b/src/plugins/multimedia/darwin/camera/qavfcamerabase.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfcameradebug_p.h" #include "qavfcamerabase_p.h" @@ -43,6 +7,8 @@ #include <private/qcameradevice_p.h> #include "qavfhelpers_p.h" #include <private/qplatformmediaintegration_p.h> +#include <QtCore/qset.h> +#include <QtCore/qsystemdetection.h> QT_USE_NAMESPACE @@ -54,29 +20,6 @@ namespace { // Misc. helpers to check values/ranges: -bool qt_check_ISO_conversion(float isoValue) -{ - if (isoValue >= std::numeric_limits<int>::max()) - return false; - if (isoValue <= std::numeric_limits<int>::min()) - return false; - return true; -} - -bool qt_check_ISO_range(AVCaptureDeviceFormat *format) -{ - // Qt is using int for ISO, AVFoundation - float. It looks like the ISO range - // at the moment can be represented by int (it's max - min > 100, etc.). - Q_ASSERT(format); - if (format.maxISO - format.minISO < 1.) { - // ISO is in some strange units? - return false; - } - - return qt_check_ISO_conversion(format.minISO) - && qt_check_ISO_conversion(format.maxISO); -} - bool qt_check_exposure_duration(AVCaptureDevice *captureDevice, CMTime duration) { Q_ASSERT(captureDevice); @@ -195,17 +138,64 @@ void QAVFVideoDevices::updateCameraDevices() QList<QCameraDevice> cameras; - AVCaptureDevice *defaultDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; - NSArray *videoDevices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]; + // List of all capture device types that we want to discover. Seems that this is the + // only way to discover all types. This filter is mandatory and has no "unspecified" + // option like AVCaptureDevicePosition(Unspecified) has. Order of the list is important + // because discovered devices will be in the same order and we want the first one found + // to be our default device. + NSArray *discoveryDevices = @[ +#ifdef Q_OS_IOS + AVCaptureDeviceTypeBuiltInTripleCamera, // We always prefer triple camera. + AVCaptureDeviceTypeBuiltInDualCamera, // If triple is not available, we prefer + // dual with wide + tele lens. + AVCaptureDeviceTypeBuiltInDualWideCamera, // Dual with wide and ultrawide is still + // better than single. +#endif + AVCaptureDeviceTypeBuiltInWideAngleCamera, // This is the most common single camera type. + // We prefer that over tele and ultra-wide. +#ifdef Q_OS_IOS + AVCaptureDeviceTypeBuiltInTelephotoCamera, // Cannot imagine how, but if only tele and + // ultrawide are available, we prefer tele. + AVCaptureDeviceTypeBuiltInUltraWideCamera, +#endif + ]; + +#if QT_DARWIN_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_14_0, __IPHONE_17_0, __TVOS_NA, __WATCHOS_NA) + if (@available(macOS 14, iOS 17, *)) { + discoveryDevices = [discoveryDevices arrayByAddingObjectsFromArray: @[ + AVCaptureDeviceTypeExternal, + AVCaptureDeviceTypeContinuityCamera + ]]; + } else +#endif + { +#ifdef Q_OS_MACOS + QT_WARNING_PUSH + QT_WARNING_DISABLE_DEPRECATED + discoveryDevices = [discoveryDevices arrayByAddingObjectsFromArray: @[ + AVCaptureDeviceTypeExternalUnknown + ]]; + QT_WARNING_POP +#endif + } + // Create discovery session to discover all possible camera types of the system. + // Both "hard" and "soft" types. + AVCaptureDeviceDiscoverySession *discoverySession = [AVCaptureDeviceDiscoverySession + discoverySessionWithDeviceTypes:discoveryDevices + mediaType:AVMediaTypeVideo + position:AVCaptureDevicePositionUnspecified]; + NSArray<AVCaptureDevice *> *videoDevices = discoverySession.devices; for (AVCaptureDevice *device in videoDevices) { - - QCameraDevicePrivate *info = new QCameraDevicePrivate; - if (defaultDevice && [defaultDevice.uniqueID isEqualToString:device.uniqueID]) + auto info = std::make_unique<QCameraDevicePrivate>(); + if ([videoDevices[0].uniqueID isEqualToString:device.uniqueID]) info->isDefault = true; info->id = QByteArray([[device uniqueID] UTF8String]); info->description = QString::fromNSString([device localizedName]); + qCDebug(qLcCamera) << "Handling camera info" << info->description + << (info->isDefault ? "(default)" : ""); + QSet<QSize> photoResolutions; QList<QCameraFormat> videoFormats; @@ -222,9 +212,13 @@ void QAVFVideoDevices::updateCameraDevices() auto encoding = CMVideoFormatDescriptionGetCodecType(format.formatDescription); auto pixelFormat = QAVFHelpers::fromCVPixelFormat(encoding); + auto colorRange = QAVFHelpers::colorRangeForCVPixelFormat(encoding); // Ignore pixel formats we can't handle - if (pixelFormat == QVideoFrameFormat::Format_Invalid) + if (pixelFormat == QVideoFrameFormat::Format_Invalid) { + qCDebug(qLcCamera) << "ignore camera CV format" << encoding + << "as no matching video format found"; continue; + } for (AVFrameRateRange *frameRateRange in format.videoSupportedFrameRateRanges) { if (frameRateRange.minFrameRate < minFrameRate) @@ -244,29 +238,30 @@ void QAVFVideoDevices::updateCameraDevices() photoResolutions.insert(hrRes); #endif - auto *f = new QCameraFormatPrivate{ - QSharedData(), - pixelFormat, - resolution, - minFrameRate, - maxFrameRate - }; + qCDebug(qLcCamera) << "Add camera format. pixelFormat:" << pixelFormat + << "colorRange:" << colorRange << "cvPixelFormat" << encoding + << "resolution:" << resolution << "frameRate: [" << minFrameRate + << maxFrameRate << "]"; + + auto *f = new QCameraFormatPrivate{ QSharedData(), pixelFormat, resolution, + minFrameRate, maxFrameRate, colorRange }; videoFormats << f->create(); } if (videoFormats.isEmpty()) { // skip broken cameras without valid formats - delete info; + qCWarning(qLcCamera()) + << "Skip camera" << info->description << "without supported formats"; continue; } info->videoFormats = videoFormats; info->photoResolutions = photoResolutions.values(); - cameras.append(info->create()); + cameras.append(info.release()->create()); } if (cameras != m_cameraDevices) { m_cameraDevices = cameras; - videoInputsChanged(); + emit videoInputsChanged(); } } @@ -515,7 +510,7 @@ void QAVFCameraBase::updateCameraConfiguration() } minimumZoomFactorChanged(captureDevice.minAvailableVideoZoomFactor); - maximumZoomFactorChanged(captureDevice.maxAvailableVideoZoomFactor); + maximumZoomFactorChanged(captureDevice.activeFormat.videoMaxZoomFactor); captureDevice.videoZoomFactor = zoomFactor(); @@ -629,7 +624,8 @@ void QAVFCameraBase::zoomTo(float factor, float rate) if (!captureDevice || !captureDevice.activeFormat) return; - factor = qBound(captureDevice.minAvailableVideoZoomFactor, factor, captureDevice.maxAvailableVideoZoomFactor); + factor = qBound(captureDevice.minAvailableVideoZoomFactor, factor, + captureDevice.activeFormat.videoMaxZoomFactor); const AVFConfigurationLock lock(captureDevice); if (!lock) { @@ -637,10 +633,10 @@ void QAVFCameraBase::zoomTo(float factor, float rate) return; } - if (rate < 0) + if (rate <= 0) captureDevice.videoZoomFactor = factor; - else - [AVCaptureDevice rampToVideoZoomFactor:factor withRate:rate]; + else + [captureDevice rampToVideoZoomFactor:factor withRate:rate]; #endif } @@ -687,14 +683,10 @@ bool QAVFCameraBase::isFlashReady() const if (!isFlashModeSupported(flashMode())) return false; -#ifdef Q_OS_IOS // AVCaptureDevice's docs: // "The flash may become unavailable if, for example, // the device overheats and needs to cool off." return [captureDevice isFlashAvailable]; -#endif - - return true; } void QAVFCameraBase::setTorchMode(QCamera::TorchMode mode) @@ -766,12 +758,8 @@ bool QAVFCameraBase::isExposureModeSupported(QCamera::ExposureMode mode) const if (mode != QCamera::ExposureManual) return false; - if (@available(macOS 10.15, *)) { - AVCaptureDevice *captureDevice = device(); - return captureDevice && [captureDevice isExposureModeSupported:AVCaptureExposureModeCustom]; - } - - return false; + AVCaptureDevice *captureDevice = device(); + return captureDevice && [captureDevice isExposureModeSupported:AVCaptureExposureModeCustom]; } void QAVFCameraBase::applyFlashSettings() @@ -784,42 +772,55 @@ void QAVFCameraBase::applyFlashSettings() return; } - const AVFConfigurationLock lock(captureDevice); if (captureDevice.hasFlash) { - auto mode = flashMode(); + const auto mode = flashMode(); + + auto setAvFlashModeSafe = [&captureDevice](AVCaptureFlashMode avFlashMode) { + // Note, in some cases captureDevice.hasFlash == false even though + // no there're no supported flash modes. + if ([captureDevice isFlashModeSupported:avFlashMode]) + captureDevice.flashMode = avFlashMode; + else + qCDebug(qLcCamera) << "Attempt to setup unsupported flash mode " << avFlashMode; + }; + if (mode == QCamera::FlashOff) { - captureDevice.flashMode = AVCaptureFlashModeOff; + setAvFlashModeSafe(AVCaptureFlashModeOff); } else { -#ifdef Q_OS_IOS - if (![captureDevice isFlashAvailable]) { + if ([captureDevice isFlashAvailable]) { + if (mode == QCamera::FlashOn) + setAvFlashModeSafe(AVCaptureFlashModeOn); + else if (mode == QCamera::FlashAuto) + setAvFlashModeSafe(AVCaptureFlashModeAuto); + } else { qCDebug(qLcCamera) << Q_FUNC_INFO << "flash is not available at the moment"; - return; } -#endif - if (mode == QCamera::FlashOn) - captureDevice.flashMode = AVCaptureFlashModeOn; - else if (mode == QCamera::FlashAuto) - captureDevice.flashMode = AVCaptureFlashModeAuto; } } if (captureDevice.hasTorch) { - auto mode = torchMode(); + const auto mode = torchMode(); + + auto setAvTorchModeSafe = [&captureDevice](AVCaptureTorchMode avTorchMode) { + if ([captureDevice isTorchModeSupported:avTorchMode]) + captureDevice.torchMode = avTorchMode; + else + qCDebug(qLcCamera) << "Attempt to setup unsupported torch mode " << avTorchMode; + }; + if (mode == QCamera::TorchOff) { - captureDevice.torchMode = AVCaptureTorchModeOff; + setAvTorchModeSafe(AVCaptureTorchModeOff); } else { -#ifdef Q_OS_IOS - if (![captureDevice isTorchAvailable]) { + if ([captureDevice isTorchAvailable]) { + if (mode == QCamera::TorchOn) + setAvTorchModeSafe(AVCaptureTorchModeOn); + else if (mode == QCamera::TorchAuto) + setAvTorchModeSafe(AVCaptureTorchModeAuto); + } else { qCDebug(qLcCamera) << Q_FUNC_INFO << "torch is not available at the moment"; - return; } -#endif - if (mode == QCamera::TorchOn) - captureDevice.torchMode = AVCaptureTorchModeOn; - else if (mode == QCamera::TorchAuto) - captureDevice.torchMode = AVCaptureTorchModeAuto; } } } diff --git a/src/plugins/multimedia/darwin/camera/qavfcamerabase_p.h b/src/plugins/multimedia/darwin/camera/qavfcamerabase_p.h index 3b68cd778..1ad3ba250 100644 --- a/src/plugins/multimedia/darwin/camera/qavfcamerabase_p.h +++ b/src/plugins/multimedia/darwin/camera/qavfcamerabase_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QAVFCAMERABASE_H #define QAVFCAMERABASE_H diff --git a/src/plugins/multimedia/darwin/common/avfmetadata.mm b/src/plugins/multimedia/darwin/common/avfmetadata.mm index 070756527..994ef9e42 100644 --- a/src/plugins/multimedia/darwin/common/avfmetadata.mm +++ b/src/plugins/multimedia/darwin/common/avfmetadata.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfmetadata_p.h" #include <qdarwinformatsinfo_p.h> @@ -49,7 +13,7 @@ #include <QImage> #include <QtMultimedia/qvideoframe.h> -#if QT_HAS_INCLUDE(<AppKit/AppKit.h>) +#if __has_include(<AppKit/AppKit.h>) #include <AppKit/AppKit.h> #endif @@ -146,12 +110,14 @@ const AVMetadataIDs keyToAVMetaDataID[] = { // Orientation { nil, nil, AVMetadataIdentifierQuickTimeMetadataVideoOrientation, nil, nil, nil }, // Resolution + { nil, nil, nil, nil, nil, nil }, + // HasHdrContent { nil, nil, nil, nil, nil, nil } }; static AVMetadataIdentifier toIdentifier(QMediaMetaData::Key key, AVMetadataKeySpace keySpace) { - static_assert(sizeof(keyToAVMetaDataID)/sizeof(AVMetadataIDs) == QMediaMetaData::Key::Resolution + 1); + static_assert(sizeof(keyToAVMetaDataID) / sizeof(AVMetadataIDs) == QMediaMetaData::NumMetaData); AVMetadataIdentifier identifier = nil; if ([keySpace isEqualToString:AVMetadataKeySpaceiTunes]) { @@ -168,7 +134,7 @@ static AVMetadataIdentifier toIdentifier(QMediaMetaData::Key key, AVMetadataKeyS static std::optional<QMediaMetaData::Key> toKey(AVMetadataItem *item) { - static_assert(sizeof(keyToAVMetaDataID)/sizeof(AVMetadataIDs) == QMediaMetaData::Key::Resolution + 1); + static_assert(sizeof(keyToAVMetaDataID) / sizeof(AVMetadataIDs) == QMediaMetaData::NumMetaData); // The item identifier may be different than the ones we support, // so check by common key first, as it will get the metadata @@ -217,7 +183,7 @@ static std::optional<QMediaMetaData::Key> toKey(AVMetadataItem *item) itemKeySpace = ID3; } - for (int key = 0; key < QMediaMetaData::Resolution + 1; key++) { + for (int key = 0; key < QMediaMetaData::NumMetaData; key++) { AVMetadataIdentifier idForKey = nil; switch (itemKeySpace) { case iTunes: @@ -297,12 +263,29 @@ QMediaMetaData AVFMetaData::fromAssetTrack(AVAssetTrack *asset) if ([asset.mediaType isEqualToString:AVMediaTypeVideo]) { // add orientation if (metadata.value(QMediaMetaData::Orientation).isNull()) { - QVideoFrame::RotationAngle angle = QVideoFrame::Rotation0; + QtVideo::Rotation angle = QtVideo::Rotation::None; bool mirrored; AVFMediaPlayer::videoOrientationForAssetTrack(asset, angle, mirrored); Q_UNUSED(mirrored); metadata.insert(QMediaMetaData::Orientation, int(angle)); } + + // add HDR content + if (metadata.value(QMediaMetaData::HasHdrContent).isNull()) { + auto hasHdrContent = false; + + NSArray *formatDescriptions = [asset formatDescriptions]; + for (id formatDescription in formatDescriptions) { + NSDictionary *extensions = (__bridge NSDictionary *)CMFormatDescriptionGetExtensions((CMFormatDescriptionRef)formatDescription); + NSString *transferFunction = extensions[(__bridge NSString *)kCMFormatDescriptionExtension_TransferFunction]; + if ([transferFunction isEqualToString:(__bridge NSString *)kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ]) { + hasHdrContent = true; + break; + } + } + + metadata.insert(QMediaMetaData::HasHdrContent, hasHdrContent); + } } return metadata; } diff --git a/src/plugins/multimedia/darwin/common/avfmetadata_p.h b/src/plugins/multimedia/darwin/common/avfmetadata_p.h index e983be531..d1cb2e7e8 100644 --- a/src/plugins/multimedia/darwin/common/avfmetadata_p.h +++ b/src/plugins/multimedia/darwin/common/avfmetadata_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFMEDIAPLAYERMETADATACONTROL_H #define AVFMEDIAPLAYERMETADATACONTROL_H diff --git a/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink.mm b/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink.mm index 64b625f0e..8c6561f37 100644 --- a/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink.mm +++ b/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfdisplaylink_p.h" #include <QtCore/qcoreapplication.h> @@ -239,3 +203,5 @@ bool AVFDisplayLink::event(QEvent *event) } return QObject::event(event); } + +#include "moc_avfdisplaylink_p.cpp" diff --git a/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink_p.h b/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink_p.h index 6b95e1e07..c4eb504a5 100644 --- a/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink_p.h +++ b/src/plugins/multimedia/darwin/mediaplayer/avfdisplaylink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFDISPLAYLINK_H #define AVFDISPLAYLINK_H diff --git a/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer.mm b/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer.mm index 86b7ce0a2..694cc0e3d 100644 --- a/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer.mm +++ b/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfmediaplayer_p.h" #include "avfvideorenderercontrol_p.h" @@ -48,6 +12,7 @@ #include <qpointer.h> #include <QFileInfo> #include <QtCore/qmath.h> +#include <QtCore/qmutex.h> #import <AVFoundation/AVFoundation.h> @@ -95,6 +60,12 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM - (BOOL) resourceLoader:(AVAssetResourceLoader *)resourceLoader shouldWaitForLoadingOfRequestedResource:(AVAssetResourceLoadingRequest *)loadingRequest; @end +#ifdef Q_OS_IOS +// Alas, no such thing as 'class variable', hence globals: +static unsigned sessionActivationCount; +static QMutex sessionMutex; +#endif // Q_OS_IOS + @implementation AVFMediaPlayerObserver { @private @@ -106,10 +77,39 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM BOOL m_bufferIsLikelyToKeepUp; NSData *m_data; NSString *m_mimeType; +#ifdef Q_OS_IOS + BOOL m_activated; +#endif } @synthesize m_player, m_playerItem, m_playerLayer, m_session; +#ifdef Q_OS_IOS +- (void)setSessionActive:(BOOL)active +{ + const QMutexLocker lock(&sessionMutex); + if (active) { + // Don't count the same player twice if already activated, + // unless it tried to deactivate first: + if (m_activated) + return; + if (!sessionActivationCount) + [AVAudioSession.sharedInstance setActive:YES error:nil]; + ++sessionActivationCount; + m_activated = YES; + } else { + if (!sessionActivationCount || !m_activated) { + qWarning("Unbalanced audio session deactivation, ignoring."); + return; + } + --sessionActivationCount; + m_activated = NO; + if (!sessionActivationCount) + [AVAudioSession.sharedInstance setActive:NO error:nil]; + } +} +#endif // Q_OS_IOS + - (AVFMediaPlayerObserver *) initWithMediaPlayerSession:(AVFMediaPlayer *)session { if (!(self = [super init])) @@ -127,6 +127,9 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM - (void) setURL:(NSURL *)url mimeType:(NSString *)mimeType { + if (!m_session) + return; + [m_mimeType release]; m_mimeType = [mimeType retain]; @@ -140,23 +143,29 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM // use __block to avoid maintaining strong references on variables captured by the // following block callback +#if defined(Q_OS_IOS) + BOOL isAccessing = [m_URL startAccessingSecurityScopedResource]; +#endif __block AVURLAsset *asset = [[AVURLAsset URLAssetWithURL:m_URL options:nil] retain]; [asset.resourceLoader setDelegate:self queue:dispatch_get_main_queue()]; __block NSArray *requestedKeys = [[NSArray arrayWithObjects:AVF_TRACKS_KEY, AVF_PLAYABLE_KEY, nil] retain]; - __block AVFMediaPlayerObserver *blockSelf = self; - QPointer<AVFMediaPlayer> session(m_session); + __block AVFMediaPlayerObserver *blockSelf = [self retain]; // Tells the asset to load the values of any of the specified keys that are not already loaded. [asset loadValuesAsynchronouslyForKeys:requestedKeys completionHandler: ^{ dispatch_async( dispatch_get_main_queue(), ^{ - if (session) - [blockSelf prepareToPlayAsset:asset withKeys:requestedKeys]; +#if defined(Q_OS_IOS) + if (isAccessing) + [m_URL stopAccessingSecurityScopedResource]; +#endif + [blockSelf prepareToPlayAsset:asset withKeys:requestedKeys]; [asset release]; [requestedKeys release]; + [blockSelf release]; }); }]; } @@ -186,13 +195,16 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM if (m_playerLayer) m_playerLayer.player = nil; #if defined(Q_OS_IOS) - [[AVAudioSession sharedInstance] setActive:NO error:nil]; + [self setSessionActive:NO]; #endif } - (void) prepareToPlayAsset:(AVURLAsset *)asset withKeys:(NSArray *)requestedKeys { + if (!m_session) + return; + //Make sure that the value of each key has loaded successfully. for (NSString *thisKey in requestedKeys) { @@ -213,10 +225,23 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM qDebug() << Q_FUNC_INFO << "isPlayable: " << [asset isPlayable]; #endif if (!asset.playable) + qWarning() << "Asset reported to be not playable. Playback of this asset may not be possible."; + + //At this point we're ready to set up for playback of the asset. + //Stop observing our prior AVPlayerItem, if we have one. + if (m_playerItem) { + //Remove existing player item key value observers and notifications. + [self unloadMedia]; + } + + //Create a new instance of AVPlayerItem from the now successfully loaded AVAsset. + m_playerItem = [AVPlayerItem playerItemWithAsset:asset]; + if (!m_playerItem) { + qWarning() << "Failed to create player item"; //Generate an error describing the failure. NSString *localizedDescription = NSLocalizedString(@"Item cannot be played", @"Item cannot be played description"); - NSString *localizedFailureReason = NSLocalizedString(@"The assets tracks were loaded, but could not be made playable.", @"Item cannot be played failure reason"); + NSString *localizedFailureReason = NSLocalizedString(@"The assets tracks were loaded, but couldn't create player item.", @"Item cannot be played failure reason"); NSDictionary *errorDict = [NSDictionary dictionaryWithObjectsAndKeys: localizedDescription, NSLocalizedDescriptionKey, localizedFailureReason, NSLocalizedFailureReasonErrorKey, @@ -224,21 +249,9 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM NSError *assetCannotBePlayedError = [NSError errorWithDomain:@"StitchedStreamPlayer" code:0 userInfo:errorDict]; [self assetFailedToPrepareForPlayback:assetCannotBePlayedError]; - return; } - //At this point we're ready to set up for playback of the asset. - //Stop observing our prior AVPlayerItem, if we have one. - if (m_playerItem) - { - //Remove existing player item key value observers and notifications. - [self unloadMedia]; - } - - //Create a new instance of AVPlayerItem from the now successfully loaded AVAsset. - m_playerItem = [AVPlayerItem playerItemWithAsset:asset]; - //Observe the player item "status" key to determine when it is ready to play. [m_playerItem addObserver:self forKeyPath:AVF_STATUS_KEY @@ -271,11 +284,12 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM m_player = [AVPlayer playerWithPlayerItem:m_playerItem]; [m_player retain]; - //Set the initial volume on new player object + //Set the initial audio ouptut settings on new player object if (self.session) { auto *audioOutput = m_session->m_audioOutput; m_player.volume = (audioOutput ? audioOutput->volume : 1.); m_player.muted = (audioOutput ? audioOutput->muted : true); + m_session->updateAudioOutputDevice(); } //Assign the output layer to the new player @@ -302,7 +316,7 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM context:AVFMediaPlayerObserverCurrentItemDurationObservationContext]; #if defined(Q_OS_IOS) [[AVAudioSession sharedInstance] setCategory:AVAudioSessionCategoryPlayback withOptions:AVAudioSessionCategoryOptionMixWithOthers error:nil]; - [[AVAudioSession sharedInstance] setActive:YES error:nil]; + [self setSessionActive:YES]; #endif } @@ -472,19 +486,18 @@ static void *AVFMediaPlayerObserverCurrentItemDurationObservationContext = &AVFM @end AVFMediaPlayer::AVFMediaPlayer(QMediaPlayer *player) - : QObject(player) - , QPlatformMediaPlayer(player) - , m_state(QMediaPlayer::StoppedState) - , m_mediaStatus(QMediaPlayer::NoMedia) - , m_mediaStream(nullptr) - , m_tryingAsync(false) - , m_rate(1.0) - , m_requestedPosition(-1) - , m_duration(0) - , m_bufferProgress(0) - , m_videoAvailable(false) - , m_audioAvailable(false) - , m_seekable(false) + : QObject(player), + QPlatformMediaPlayer(player), + m_state(QMediaPlayer::StoppedState), + m_mediaStatus(QMediaPlayer::NoMedia), + m_mediaStream(nullptr), + m_rate(1.0), + m_requestedPosition(-1), + m_duration(0), + m_bufferProgress(0), + m_videoAvailable(false), + m_audioAvailable(false), + m_seekable(false) { m_observer = [[AVFMediaPlayerObserver alloc] initWithMediaPlayerSession:this]; connect(&m_playbackTimer, &QTimer::timeout, this, &AVFMediaPlayer::processPositionChange); @@ -583,7 +596,7 @@ void AVFMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) setVideoAvailable(false); setSeekable(false); m_requestedPosition = -1; - orientationChanged(QVideoFrame::Rotation0, false); + orientationChanged(QtVideo::Rotation::None, false); Q_EMIT positionChanged(position()); if (m_duration != 0) { m_duration = 0; @@ -738,12 +751,12 @@ void AVFMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) m_audioOutput->q->disconnect(this); m_audioOutput = output; if (m_audioOutput) { - connect(m_audioOutput->q, &QAudioOutput::deviceChanged, this, &AVFMediaPlayer::audioOutputChanged); + connect(m_audioOutput->q, &QAudioOutput::deviceChanged, this, &AVFMediaPlayer::updateAudioOutputDevice); connect(m_audioOutput->q, &QAudioOutput::volumeChanged, this, &AVFMediaPlayer::setVolume); connect(m_audioOutput->q, &QAudioOutput::mutedChanged, this, &AVFMediaPlayer::setMuted); //connect(m_audioOutput->q, &QAudioOutput::audioRoleChanged, this, &AVFMediaPlayer::setAudioRole); } - audioOutputChanged(); + updateAudioOutputDevice(); setMuted(m_audioOutput ? m_audioOutput->muted : true); setVolume(m_audioOutput ? m_audioOutput->volume : 1.); } @@ -924,7 +937,7 @@ void AVFMediaPlayer::setMuted(bool muted) player.muted = muted; } -void AVFMediaPlayer::audioOutputChanged() +void AVFMediaPlayer::updateAudioOutputDevice() { #ifdef Q_OS_MACOS AVPlayer *player = [static_cast<AVFMediaPlayerObserver*>(m_observer) player]; @@ -1120,7 +1133,7 @@ void AVFMediaPlayer::updateTracks() if (m_observer.videoTrack != track) { m_observer.videoTrack = track; bool isMirrored = false; - QVideoFrame::RotationAngle orientation = QVideoFrame::Rotation0; + QtVideo::Rotation orientation = QtVideo::Rotation::None; videoOrientationForAssetTrack(assetTrack, orientation, isMirrored); orientationChanged(orientation, isMirrored); } @@ -1210,7 +1223,7 @@ void AVFMediaPlayer::nativeSizeChanged(QSize size) m_videoSink->setNativeSize(size); } -void AVFMediaPlayer::orientationChanged(QVideoFrame::RotationAngle rotation, bool mirrored) +void AVFMediaPlayer::orientationChanged(QtVideo::Rotation rotation, bool mirrored) { if (!m_videoOutput) return; @@ -1220,10 +1233,10 @@ void AVFMediaPlayer::orientationChanged(QVideoFrame::RotationAngle rotation, boo } void AVFMediaPlayer::videoOrientationForAssetTrack(AVAssetTrack *videoTrack, - QVideoFrame::RotationAngle &angle, + QtVideo::Rotation &angle, bool &mirrored) { - angle = QVideoFrame::Rotation0; + angle = QtVideo::Rotation::None; if (videoTrack) { CGAffineTransform transform = videoTrack.preferredTransform; if (CGAffineTransformIsIdentity(transform)) @@ -1246,11 +1259,13 @@ void AVFMediaPlayer::videoOrientationForAssetTrack(AVAssetTrack *videoTrack, } if (qFuzzyCompare(degrees, qreal(90)) || qFuzzyCompare(degrees, qreal(-270))) { - angle = QVideoFrame::Rotation90; + angle = QtVideo::Rotation::Clockwise90; } else if (qFuzzyCompare(degrees, qreal(-90)) || qFuzzyCompare(degrees, qreal(270))) { - angle = QVideoFrame::Rotation270; + angle = QtVideo::Rotation::Clockwise270; } else if (qFuzzyCompare(degrees, qreal(180)) || qFuzzyCompare(degrees, qreal(-180))) { - angle = QVideoFrame::Rotation180; + angle = QtVideo::Rotation::Clockwise180; } } } + +#include "moc_avfmediaplayer_p.cpp" diff --git a/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer_p.h b/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer_p.h index 6993a28d7..6ac3aef46 100644 --- a/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer_p.h +++ b/src/plugins/multimedia/darwin/mediaplayer/avfmediaplayer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFMEDIAPLAYER_H #define AVFMEDIAPLAYER_H @@ -110,7 +74,7 @@ public: QMediaMetaData metaData() const override; static void videoOrientationForAssetTrack(AVAssetTrack *track, - QVideoFrame::RotationAngle &angle, + QtVideo::Rotation &angle, bool &mirrored); public Q_SLOTS: @@ -125,7 +89,7 @@ public Q_SLOTS: void setVolume(float volume); void setMuted(bool muted); - void audioOutputChanged(); + void updateAudioOutputDevice(); void processEOS(); void processLoadStateChange(QMediaPlayer::PlaybackState newState); @@ -157,7 +121,7 @@ private: void setSeekable(bool seekable); void resetStream(QIODevice *stream = nullptr); - void orientationChanged(QVideoFrame::RotationAngle rotation, bool mirrored); + void orientationChanged(QtVideo::Rotation rotation, bool mirrored); AVFVideoRendererControl *m_videoOutput = nullptr; AVFVideoSink *m_videoSink = nullptr; @@ -168,7 +132,6 @@ private: QUrl m_resources; QMediaMetaData m_metaData; - bool m_tryingAsync; qreal m_rate; qint64 m_requestedPosition; diff --git a/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm b/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm index 0a59708f2..66687c931 100644 --- a/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm +++ b/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol.mm @@ -1,51 +1,16 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "avfvideorenderercontrol_p.h" #include "avfdisplaylink_p.h" #include <avfvideobuffer_p.h> #include "qavfhelpers_p.h" +#include "private/qvideoframe_p.h" #include <QtMultimedia/qvideoframeformat.h> #include <avfvideosink_p.h> -#include <QtGui/private/qrhi_p.h> +#include <rhi/qrhi.h> #include <QtCore/qdebug.h> @@ -154,7 +119,7 @@ void AVFVideoRendererControl::setLayer(CALayer *layer) AVFVideoSinkInterface::setLayer(layer); } -void AVFVideoRendererControl::setVideoRotation(QVideoFrame::RotationAngle rotation) +void AVFVideoRendererControl::setVideoRotation(QtVideo::Rotation rotation) { m_rotation = rotation; } @@ -184,34 +149,17 @@ void AVFVideoRendererControl::updateVideoFrame(const CVTimeStamp &ts) CVPixelBufferRef pixelBuffer = copyPixelBufferFromLayer(width, height); if (!pixelBuffer) return; - AVFVideoBuffer *buffer = new AVFVideoBuffer(this, pixelBuffer); -// qDebug() << "Got pixelbuffer with format" << fmt << Qt::hex << CVPixelBufferGetPixelFormatType(pixelBuffer); + auto buffer = std::make_unique<AVFVideoBuffer>(this, pixelBuffer); + // qDebug() << "Got pixelbuffer with format" << fmt << Qt::hex << + // CVPixelBufferGetPixelFormatType(pixelBuffer); CVPixelBufferRelease(pixelBuffer); - frame = QVideoFrame(buffer, buffer->videoFormat()); - frame.setRotationAngle(m_rotation); + frame = QVideoFramePrivate::createFrame(std::move(buffer), buffer->videoFormat()); + frame.setRotation(m_rotation); frame.setMirrored(m_mirrored); m_sink->setVideoFrame(frame); } -static NSDictionary* const AVF_OUTPUT_SETTINGS = @{ - (NSString *)kCVPixelBufferPixelFormatTypeKey: @[ - @(kCVPixelFormatType_32BGRA), - @(kCVPixelFormatType_32RGBA), - @(kCVPixelFormatType_422YpCbCr8), - @(kCVPixelFormatType_422YpCbCr8_yuvs), - @(kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange), - @(kCVPixelFormatType_420YpCbCr8BiPlanarFullRange), - @(kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange), - @(kCVPixelFormatType_420YpCbCr10BiPlanarFullRange), - @(kCVPixelFormatType_OneComponent8), - @(q_kCVPixelFormatType_OneComponent16), - @(kCVPixelFormatType_420YpCbCr8Planar), - @(kCVPixelFormatType_420YpCbCr8PlanarFullRange) - ], - (NSString *)kCVPixelBufferMetalCompatibilityKey: @true -}; - CVPixelBufferRef AVFVideoRendererControl::copyPixelBufferFromLayer(size_t& width, size_t& height) { AVPlayerLayer *layer = playerLayer(); @@ -227,7 +175,7 @@ CVPixelBufferRef AVFVideoRendererControl::copyPixelBufferFromLayer(size_t& width if (!m_videoOutput) { if (!m_outputSettings) - m_outputSettings = AVF_OUTPUT_SETTINGS; + setOutputSettings(); m_videoOutput = [[AVPlayerItemVideoOutput alloc] initWithPixelBufferAttributes:m_outputSettings]; [m_videoOutput setDelegate:nil queue:nil]; } diff --git a/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol_p.h b/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol_p.h index ea04bbde7..177114127 100644 --- a/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol_p.h +++ b/src/plugins/multimedia/darwin/mediaplayer/avfvideorenderercontrol_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef AVFVIDEORENDERERCONTROL_H #define AVFVIDEORENDERERCONTROL_H @@ -80,7 +44,7 @@ public: void reconfigure() override; void setLayer(CALayer *layer) override; - void setVideoRotation(QVideoFrame::RotationAngle); + void setVideoRotation(QtVideo::Rotation); void setVideoMirrored(bool mirrored); void setSubtitleText(const QString &subtitle) @@ -99,7 +63,7 @@ private: AVPlayerItemVideoOutput *m_videoOutput = nullptr; AVPlayerItemLegibleOutput *m_subtitleOutput = nullptr; SubtitleDelegate *m_subtitleDelegate = nullptr; - QVideoFrame::RotationAngle m_rotation = QVideoFrame::Rotation0; + QtVideo::Rotation m_rotation = QtVideo::Rotation::None; bool m_mirrored = false; }; diff --git a/src/plugins/multimedia/darwin/qavfhelpers.mm b/src/plugins/multimedia/darwin/qavfhelpers.mm index 52215e883..6921309ed 100644 --- a/src/plugins/multimedia/darwin/qavfhelpers.mm +++ b/src/plugins/multimedia/darwin/qavfhelpers.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <qavfhelpers_p.h> #include <CoreMedia/CMFormatDescription.h> #include <CoreVideo/CoreVideo.h> @@ -43,86 +7,73 @@ #import <CoreVideo/CoreVideo.h> -QVideoFrameFormat::PixelFormat QAVFHelpers::fromCVPixelFormat(unsigned avPixelFormat) +namespace { + +using PixelFormat = QVideoFrameFormat::PixelFormat; +using ColorRange = QVideoFrameFormat::ColorRange; + +// clang-format off +constexpr std::tuple<CvPixelFormat, PixelFormat, ColorRange> PixelFormatMap[] = { + { kCVPixelFormatType_32ARGB, PixelFormat::Format_ARGB8888, ColorRange::ColorRange_Unknown }, + { kCVPixelFormatType_32BGRA, PixelFormat::Format_BGRA8888, ColorRange::ColorRange_Unknown }, + { kCVPixelFormatType_420YpCbCr8Planar, PixelFormat::Format_YUV420P, ColorRange::ColorRange_Unknown }, + { kCVPixelFormatType_420YpCbCr8PlanarFullRange, PixelFormat::Format_YUV420P, ColorRange::ColorRange_Full }, + { kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, PixelFormat::Format_NV12, ColorRange::ColorRange_Video }, + { kCVPixelFormatType_420YpCbCr8BiPlanarFullRange, PixelFormat::Format_NV12, ColorRange::ColorRange_Full }, + { kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange, PixelFormat::Format_P010, ColorRange::ColorRange_Video }, + { kCVPixelFormatType_420YpCbCr10BiPlanarFullRange, PixelFormat::Format_P010, ColorRange::ColorRange_Full }, + { kCVPixelFormatType_422YpCbCr8, PixelFormat::Format_UYVY, ColorRange::ColorRange_Video }, + { kCVPixelFormatType_422YpCbCr8_yuvs, PixelFormat::Format_YUYV, ColorRange::ColorRange_Video }, + { kCVPixelFormatType_OneComponent8, PixelFormat::Format_Y8, ColorRange::ColorRange_Unknown }, + { kCVPixelFormatType_OneComponent16, PixelFormat::Format_Y16, ColorRange::ColorRange_Unknown }, + + // The cases with kCMVideoCodecType_JPEG/kCMVideoCodecType_JPEG_OpenDML as cv pixel format should be investigated. + // Matching kCMVideoCodecType_JPEG_OpenDML to ColorRange_Full is a little hack to distinguish between + // kCMVideoCodecType_JPEG and kCMVideoCodecType_JPEG_OpenDML. + { kCMVideoCodecType_JPEG, PixelFormat::Format_Jpeg, ColorRange::ColorRange_Unknown }, + { kCMVideoCodecType_JPEG_OpenDML, PixelFormat::Format_Jpeg, ColorRange::ColorRange_Full } +}; +// clang-format on + +template<typename Type, typename... Args> +Type findInPixelFormatMap(Type defaultValue, Args... args) { - switch (avPixelFormat) { - case kCVPixelFormatType_32ARGB: - return QVideoFrameFormat::Format_ARGB8888; - case kCVPixelFormatType_32BGRA: - return QVideoFrameFormat::Format_BGRA8888; - case kCVPixelFormatType_420YpCbCr8Planar: - case kCVPixelFormatType_420YpCbCr8PlanarFullRange: - return QVideoFrameFormat::Format_YUV420P; - case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange: - case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange: - return QVideoFrameFormat::Format_NV12; - case kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange: - case kCVPixelFormatType_420YpCbCr10BiPlanarFullRange: - return QVideoFrameFormat::Format_P010; - case kCVPixelFormatType_422YpCbCr8: - return QVideoFrameFormat::Format_UYVY; - case kCVPixelFormatType_422YpCbCr8_yuvs: - return QVideoFrameFormat::Format_YUYV; - case kCVPixelFormatType_OneComponent8: - return QVideoFrameFormat::Format_Y8; - case q_kCVPixelFormatType_OneComponent16: - return QVideoFrameFormat::Format_Y16; - - case kCMVideoCodecType_JPEG: - case kCMVideoCodecType_JPEG_OpenDML: - return QVideoFrameFormat::Format_Jpeg; - default: - return QVideoFrameFormat::Format_Invalid; - } + auto checkElement = [&](const auto &element) { + return ((args == std::get<Args>(element)) && ...); + }; + + auto found = std::find_if(std::begin(PixelFormatMap), std::end(PixelFormatMap), checkElement); + return found == std::end(PixelFormatMap) ? defaultValue : std::get<Type>(*found); +} + } -bool QAVFHelpers::toCVPixelFormat(QVideoFrameFormat::PixelFormat qtFormat, unsigned &conv) +ColorRange QAVFHelpers::colorRangeForCVPixelFormat(CvPixelFormat cvPixelFormat) { - switch (qtFormat) { - case QVideoFrameFormat::Format_ARGB8888: - conv = kCVPixelFormatType_32ARGB; - break; - case QVideoFrameFormat::Format_BGRA8888: - conv = kCVPixelFormatType_32BGRA; - break; - case QVideoFrameFormat::Format_YUV420P: - conv = kCVPixelFormatType_420YpCbCr8PlanarFullRange; - break; - case QVideoFrameFormat::Format_NV12: - conv = kCVPixelFormatType_420YpCbCr8BiPlanarFullRange; - break; - case QVideoFrameFormat::Format_P010: - conv = kCVPixelFormatType_420YpCbCr10BiPlanarFullRange; - break; - case QVideoFrameFormat::Format_UYVY: - conv = kCVPixelFormatType_422YpCbCr8; - break; - case QVideoFrameFormat::Format_YUYV: - conv = kCVPixelFormatType_422YpCbCr8_yuvs; - break; - case QVideoFrameFormat::Format_Y8: - conv = kCVPixelFormatType_OneComponent8; - break; - case QVideoFrameFormat::Format_Y16: - conv = q_kCVPixelFormatType_OneComponent16; - break; - default: - return false; - } + return findInPixelFormatMap(ColorRange::ColorRange_Unknown, cvPixelFormat); +} + +PixelFormat QAVFHelpers::fromCVPixelFormat(CvPixelFormat cvPixelFormat) +{ + return findInPixelFormatMap(PixelFormat::Format_Invalid, cvPixelFormat); +} - return true; +CvPixelFormat QAVFHelpers::toCVPixelFormat(PixelFormat pixelFmt, ColorRange colorRange) +{ + return findInPixelFormatMap(CvPixelFormatInvalid, pixelFmt, colorRange); } QVideoFrameFormat QAVFHelpers::videoFormatForImageBuffer(CVImageBufferRef buffer, bool openGL) { - auto avPixelFormat = CVPixelBufferGetPixelFormatType(buffer); + auto cvPixelFormat = CVPixelBufferGetPixelFormatType(buffer); + auto pixelFormat = fromCVPixelFormat(cvPixelFormat); if (openGL) { - if (avPixelFormat == kCVPixelFormatType_32BGRA) - avPixelFormat = QVideoFrameFormat::Format_SamplerRect; + if (cvPixelFormat == kCVPixelFormatType_32BGRA) + pixelFormat = QVideoFrameFormat::Format_SamplerRect; else - qWarning() << "Accelerated macOS OpenGL video supports BGRA only, got CV pixel format" << avPixelFormat; + qWarning() << "Accelerated macOS OpenGL video supports BGRA only, got CV pixel format" + << cvPixelFormat; } - auto pixelFormat = fromCVPixelFormat(avPixelFormat); size_t width = CVPixelBufferGetWidth(buffer); size_t height = CVPixelBufferGetHeight(buffer); @@ -132,57 +83,53 @@ QVideoFrameFormat QAVFHelpers::videoFormatForImageBuffer(CVImageBufferRef buffer auto colorSpace = QVideoFrameFormat::ColorSpace_Undefined; auto colorTransfer = QVideoFrameFormat::ColorTransfer_Unknown; - CFStringRef cSpace = reinterpret_cast<CFStringRef>( - CVBufferGetAttachment(buffer, kCVImageBufferYCbCrMatrixKey, nullptr)); - if (CFEqual(cSpace, kCVImageBufferYCbCrMatrix_ITU_R_709_2)) { - colorSpace = QVideoFrameFormat::ColorSpace_BT709; - } else if (CFEqual(cSpace, kCVImageBufferYCbCrMatrix_ITU_R_601_4) || - CFEqual(cSpace, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995)) { - colorSpace = QVideoFrameFormat::ColorSpace_BT601; - } else if (@available(macOS 10.11, iOS 9.0, *)) { - if (CFEqual(cSpace, kCVImageBufferYCbCrMatrix_ITU_R_2020)) { + if (CFStringRef cSpace = reinterpret_cast<CFStringRef>( + CVBufferGetAttachment(buffer, kCVImageBufferYCbCrMatrixKey, nullptr))) { + if (CFEqual(cSpace, kCVImageBufferYCbCrMatrix_ITU_R_709_2)) { + colorSpace = QVideoFrameFormat::ColorSpace_BT709; + } else if (CFEqual(cSpace, kCVImageBufferYCbCrMatrix_ITU_R_601_4) + || CFEqual(cSpace, kCVImageBufferYCbCrMatrix_SMPTE_240M_1995)) { + colorSpace = QVideoFrameFormat::ColorSpace_BT601; + } else if (CFEqual(cSpace, kCVImageBufferYCbCrMatrix_ITU_R_2020)) { colorSpace = QVideoFrameFormat::ColorSpace_BT2020; } } - CFStringRef cTransfer = reinterpret_cast<CFStringRef>( - CVBufferGetAttachment(buffer, kCVImageBufferTransferFunctionKey, nullptr)); - - if (CFEqual(cTransfer, kCVImageBufferTransferFunction_ITU_R_709_2)) { - colorTransfer = QVideoFrameFormat::ColorTransfer_BT709; - } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_SMPTE_240M_1995)) { - colorTransfer = QVideoFrameFormat::ColorTransfer_BT601; - } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_sRGB)) { - colorTransfer = QVideoFrameFormat::ColorTransfer_Gamma22; - } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_UseGamma)) { - auto gamma = reinterpret_cast<CFNumberRef>( - CVBufferGetAttachment(buffer, kCVImageBufferGammaLevelKey, nullptr)); - double g; - CFNumberGetValue(gamma, kCFNumberFloat32Type, &g); - // These are best fit values given what we have in our enum - if (g < 0.8) - ; // unknown - else if (g < 1.5) - colorTransfer = QVideoFrameFormat::ColorTransfer_Linear; - else if (g < 2.1) + if (CFStringRef cTransfer = reinterpret_cast<CFStringRef>( + CVBufferGetAttachment(buffer, kCVImageBufferTransferFunctionKey, nullptr))) { + + if (CFEqual(cTransfer, kCVImageBufferTransferFunction_ITU_R_709_2)) { colorTransfer = QVideoFrameFormat::ColorTransfer_BT709; - else if (g < 2.5) + } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_SMPTE_240M_1995)) { + colorTransfer = QVideoFrameFormat::ColorTransfer_BT601; + } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_sRGB)) { colorTransfer = QVideoFrameFormat::ColorTransfer_Gamma22; - else if (g < 3.2) - colorTransfer = QVideoFrameFormat::ColorTransfer_Gamma28; - } - if (@available(macOS 10.12, iOS 11.0, *)) { - if (CFEqual(cTransfer, kCVImageBufferTransferFunction_ITU_R_2020)) + } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_UseGamma)) { + auto gamma = reinterpret_cast<CFNumberRef>( + CVBufferGetAttachment(buffer, kCVImageBufferGammaLevelKey, nullptr)); + double g; + CFNumberGetValue(gamma, kCFNumberFloat32Type, &g); + // These are best fit values given what we have in our enum + if (g < 0.8) + ; // unknown + else if (g < 1.5) + colorTransfer = QVideoFrameFormat::ColorTransfer_Linear; + else if (g < 2.1) + colorTransfer = QVideoFrameFormat::ColorTransfer_BT709; + else if (g < 2.5) + colorTransfer = QVideoFrameFormat::ColorTransfer_Gamma22; + else if (g < 3.2) + colorTransfer = QVideoFrameFormat::ColorTransfer_Gamma28; + } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_ITU_R_2020)) { colorTransfer = QVideoFrameFormat::ColorTransfer_BT709; - } - if (@available(macOS 10.12, iOS 11.0, *)) { - if (CFEqual(cTransfer, kCVImageBufferTransferFunction_ITU_R_2100_HLG)) { + } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_ITU_R_2100_HLG)) { colorTransfer = QVideoFrameFormat::ColorTransfer_STD_B67; } else if (CFEqual(cTransfer, kCVImageBufferTransferFunction_SMPTE_ST_2084_PQ)) { colorTransfer = QVideoFrameFormat::ColorTransfer_ST2084; } } + format.setColorRange(colorRangeForCVPixelFormat(cvPixelFormat)); format.setColorSpace(colorSpace); format.setColorTransfer(colorTransfer); return format; diff --git a/src/plugins/multimedia/darwin/qavfhelpers_p.h b/src/plugins/multimedia/darwin/qavfhelpers_p.h index abe9b613e..8133d5500 100644 --- a/src/plugins/multimedia/darwin/qavfhelpers_p.h +++ b/src/plugins/multimedia/darwin/qavfhelpers_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QAVFHELPERS_H #define QAVFHELPERS_H @@ -57,19 +21,19 @@ #include <CoreVideo/CVPixelBuffer.h> #include <CoreVideo/CVImageBuffer.h> -enum { - // macOS 10.14 doesn't define this pixel format yet - q_kCVPixelFormatType_OneComponent16 = 'L016' -}; - QT_BEGIN_NAMESPACE +using CvPixelFormat = unsigned; +constexpr CvPixelFormat CvPixelFormatInvalid = 0; + namespace QAVFHelpers { - QVideoFrameFormat::PixelFormat fromCVPixelFormat(unsigned avPixelFormat); - bool toCVPixelFormat(QVideoFrameFormat::PixelFormat qtFormat, unsigned &conv); +QVideoFrameFormat::ColorRange colorRangeForCVPixelFormat(CvPixelFormat cvPixelFormat); +QVideoFrameFormat::PixelFormat fromCVPixelFormat(CvPixelFormat cvPixelFormat); +CvPixelFormat toCVPixelFormat(QVideoFrameFormat::PixelFormat pixFmt, + QVideoFrameFormat::ColorRange colorRange); - QVideoFrameFormat videoFormatForImageBuffer(CVImageBufferRef buffer, bool openGL = false); +QVideoFrameFormat videoFormatForImageBuffer(CVImageBufferRef buffer, bool openGL = false); }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/darwin/qdarwinformatsinfo.mm b/src/plugins/multimedia/darwin/qdarwinformatsinfo.mm index 4a8364bbb..582060a6c 100644 --- a/src/plugins/multimedia/darwin/qdarwinformatsinfo.mm +++ b/src/plugins/multimedia/darwin/qdarwinformatsinfo.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qdarwinformatsinfo_p.h" #include <AVFoundation/AVFoundation.h> diff --git a/src/plugins/multimedia/darwin/qdarwinformatsinfo_p.h b/src/plugins/multimedia/darwin/qdarwinformatsinfo_p.h index 79e33b6f4..e01486286 100644 --- a/src/plugins/multimedia/darwin/qdarwinformatsinfo_p.h +++ b/src/plugins/multimedia/darwin/qdarwinformatsinfo_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QDARWINFORMATINFO_H #define QDARWINFORMATINFO_H diff --git a/src/plugins/multimedia/darwin/qdarwinintegration.mm b/src/plugins/multimedia/darwin/qdarwinintegration.mm index 0f2ff14e7..0e880447e 100644 --- a/src/plugins/multimedia/darwin/qdarwinintegration.mm +++ b/src/plugins/multimedia/darwin/qdarwinintegration.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qdarwinintegration_p.h" #include <avfmediaplayer_p.h> @@ -65,65 +29,61 @@ public: QPlatformMediaIntegration* create(const QString &name) override { - if (name == QLatin1String("darwin")) + if (name == u"darwin") return new QDarwinIntegration; return nullptr; } }; - -QDarwinIntegration::QDarwinIntegration() +QDarwinIntegration::QDarwinIntegration() : QPlatformMediaIntegration(QLatin1String("darwin")) { #if defined(Q_OS_MACOS) && QT_MACOS_PLATFORM_SDK_EQUAL_OR_ABOVE(__MAC_11_0) if (__builtin_available(macOS 11.0, *)) VTRegisterSupplementalVideoDecoderIfAvailable(kCMVideoCodecType_VP9); #endif - m_videoDevices = new QAVFVideoDevices(this); } -QDarwinIntegration::~QDarwinIntegration() +QPlatformMediaFormatInfo *QDarwinIntegration::createFormatInfo() { - delete m_formatInfo; + return new QDarwinFormatInfo(); } -QPlatformMediaFormatInfo *QDarwinIntegration::formatInfo() +QPlatformVideoDevices *QDarwinIntegration::createVideoDevices() { - if (!m_formatInfo) - m_formatInfo = new QDarwinFormatInfo(); - return m_formatInfo; + return new QAVFVideoDevices(this); } -QPlatformAudioDecoder *QDarwinIntegration::createAudioDecoder(QAudioDecoder *decoder) +QMaybe<QPlatformAudioDecoder *> QDarwinIntegration::createAudioDecoder(QAudioDecoder *decoder) { return new AVFAudioDecoder(decoder); } -QPlatformMediaCaptureSession *QDarwinIntegration::createCaptureSession() +QMaybe<QPlatformMediaCaptureSession *> QDarwinIntegration::createCaptureSession() { return new AVFCameraService; } -QPlatformMediaPlayer *QDarwinIntegration::createPlayer(QMediaPlayer *player) +QMaybe<QPlatformMediaPlayer *> QDarwinIntegration::createPlayer(QMediaPlayer *player) { return new AVFMediaPlayer(player); } -QPlatformCamera *QDarwinIntegration::createCamera(QCamera *camera) +QMaybe<QPlatformCamera *> QDarwinIntegration::createCamera(QCamera *camera) { return new AVFCamera(camera); } -QPlatformMediaRecorder *QDarwinIntegration::createRecorder(QMediaRecorder *recorder) +QMaybe<QPlatformMediaRecorder *> QDarwinIntegration::createRecorder(QMediaRecorder *recorder) { return new AVFMediaEncoder(recorder); } -QPlatformImageCapture *QDarwinIntegration::createImageCapture(QImageCapture *imageCapture) +QMaybe<QPlatformImageCapture *> QDarwinIntegration::createImageCapture(QImageCapture *imageCapture) { return new AVFImageCapture(imageCapture); } -QPlatformVideoSink *QDarwinIntegration::createVideoSink(QVideoSink *sink) +QMaybe<QPlatformVideoSink *> QDarwinIntegration::createVideoSink(QVideoSink *sink) { return new AVFVideoSink(sink); } diff --git a/src/plugins/multimedia/darwin/qdarwinintegration_p.h b/src/plugins/multimedia/darwin/qdarwinintegration_p.h index 39e3434be..8333de4ec 100644 --- a/src/plugins/multimedia/darwin/qdarwinintegration_p.h +++ b/src/plugins/multimedia/darwin/qdarwinintegration_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QDARWININTEGRATION_H #define QDARWININTEGRATION_H @@ -61,20 +25,19 @@ class QDarwinIntegration : public QPlatformMediaIntegration { public: QDarwinIntegration(); - ~QDarwinIntegration(); - QPlatformMediaFormatInfo *formatInfo() override; + QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *) override; + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *player) override; + QMaybe<QPlatformCamera *> createCamera(QCamera *camera) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *) override; - QPlatformAudioDecoder *createAudioDecoder(QAudioDecoder *) override; - QPlatformMediaCaptureSession *createCaptureSession() override; - QPlatformMediaPlayer *createPlayer(QMediaPlayer *player) override; - QPlatformCamera *createCamera(QCamera *camera) override; - QPlatformMediaRecorder *createRecorder(QMediaRecorder *) override; - QPlatformImageCapture *createImageCapture(QImageCapture *) override; + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *) override; - QPlatformVideoSink *createVideoSink(QVideoSink *) override; - - QPlatformMediaFormatInfo *m_formatInfo = nullptr; +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; + QPlatformVideoDevices *createVideoDevices() override; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/CMakeLists.txt b/src/plugins/multimedia/ffmpeg/CMakeLists.txt index 402a3b0aa..c6ab93273 100644 --- a/src/plugins/multimedia/ffmpeg/CMakeLists.txt +++ b/src/plugins/multimedia/ffmpeg/CMakeLists.txt @@ -1,23 +1,28 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + qt_find_package(EGL) -qt_find_package(VAAPI COMPONENTS VA DRM PROVIDED_TARGETS VAAPI::VA VAAPI::DRM MODULE_NAME multimedia QMAKE_LIB vaapi) +qt_find_package(VAAPI COMPONENTS VA DRM PROVIDED_TARGETS VAAPI::VAAPI MODULE_NAME multimedia QMAKE_LIB vaapi) -qt_internal_find_apple_system_framework(FWCoreMedia CoreMedia) # special case -qt_internal_find_apple_system_framework(FWCoreAudio CoreAudio) # special case -qt_internal_find_apple_system_framework(FWAudioUnit AudioUnit) # special case -qt_internal_find_apple_system_framework(FWVideoToolbox VideoToolbox) # special case -qt_internal_find_apple_system_framework(FWAVFoundation AVFoundation) # special case +qt_internal_find_apple_system_framework(FWCoreMedia CoreMedia) +qt_internal_find_apple_system_framework(FWCoreAudio CoreAudio) +qt_internal_find_apple_system_framework(FWAudioUnit AudioUnit) +qt_internal_find_apple_system_framework(FWVideoToolbox VideoToolbox) +qt_internal_find_apple_system_framework(FWAVFoundation AVFoundation) +qt_internal_find_apple_system_framework(FWSecurity Security) qt_internal_add_plugin(QFFmpegMediaPlugin - OUTPUT_NAME ffmegmediaplugin + OUTPUT_NAME ffmpegmediaplugin PLUGIN_TYPE multimedia SOURCES - qffmpeg_p.h + qffmpeg.cpp qffmpeg_p.h + qffmpegdefs_p.h + qffmpegioutils.cpp qffmpegioutils_p.h + qffmpegavaudioformat.cpp qffmpegavaudioformat_p.h qffmpegaudiodecoder.cpp qffmpegaudiodecoder_p.h qffmpegaudioinput.cpp qffmpegaudioinput_p.h - qffmpegclock.cpp qffmpegclock_p.h - qffmpegdecoder.cpp qffmpegdecoder_p.h + qffmpegconverter.cpp qffmpegconverter_p.h qffmpeghwaccel.cpp qffmpeghwaccel_p.h - qffmpegencoderoptions.cpp qffmpegencoderoptions_p.h qffmpegmediametadata.cpp qffmpegmediametadata_p.h qffmpegmediaplayer.cpp qffmpegmediaplayer_p.h qffmpegvideosink.cpp qffmpegvideosink_p.h @@ -27,25 +32,91 @@ qt_internal_add_plugin(QFFmpegMediaPlugin qffmpegimagecapture.cpp qffmpegimagecapture_p.h qffmpegmediacapturesession.cpp qffmpegmediacapturesession_p.h qffmpegmediarecorder.cpp qffmpegmediarecorder_p.h - qffmpegencoder.cpp qffmpegencoder_p.h qffmpegthread.cpp qffmpegthread_p.h qffmpegresampler.cpp qffmpegresampler_p.h - qffmpegvideoframeencoder.cpp qffmpegvideoframeencoder_p.h + qffmpegencodingformatcontext.cpp qffmpegencodingformatcontext_p.h + qgrabwindowsurfacecapture.cpp qgrabwindowsurfacecapture_p.h + qffmpegsurfacecapturegrabber.cpp qffmpegsurfacecapturegrabber_p.h + + qffmpegplaybackengine.cpp qffmpegplaybackengine_p.h + playbackengine/qffmpegplaybackenginedefs_p.h + playbackengine/qffmpegplaybackengineobject.cpp playbackengine/qffmpegplaybackengineobject_p.h + playbackengine/qffmpegdemuxer.cpp playbackengine/qffmpegdemuxer_p.h + playbackengine/qffmpegstreamdecoder.cpp playbackengine/qffmpegstreamdecoder_p.h + playbackengine/qffmpegrenderer.cpp playbackengine/qffmpegrenderer_p.h + playbackengine/qffmpegaudiorenderer.cpp playbackengine/qffmpegaudiorenderer_p.h + playbackengine/qffmpegvideorenderer.cpp playbackengine/qffmpegvideorenderer_p.h + playbackengine/qffmpegsubtitlerenderer.cpp playbackengine/qffmpegsubtitlerenderer_p.h + playbackengine/qffmpegtimecontroller.cpp playbackengine/qffmpegtimecontroller_p.h + playbackengine/qffmpegmediadataholder.cpp playbackengine/qffmpegmediadataholder_p.h + playbackengine/qffmpegcodec.cpp playbackengine/qffmpegcodec_p.h + playbackengine/qffmpegpacket_p.h + playbackengine/qffmpegframe_p.h + playbackengine/qffmpegpositionwithoffset_p.h + + recordingengine/qffmpegaudioencoder_p.h + recordingengine/qffmpegaudioencoder.cpp + recordingengine/qffmpegaudioencoderutils_p.h + recordingengine/qffmpegaudioencoderutils.cpp + recordingengine/qffmpegencoderthread_p.h + recordingengine/qffmpegencoderthread.cpp + recordingengine/qffmpegencoderoptions_p.h + recordingengine/qffmpegencoderoptions.cpp + recordingengine/qffmpegmuxer_p.h + recordingengine/qffmpegmuxer.cpp + recordingengine/qffmpegrecordingengine_p.h + recordingengine/qffmpegrecordingengine.cpp + recordingengine/qffmpegencodinginitializer_p.h + recordingengine/qffmpegencodinginitializer.cpp + recordingengine/qffmpegrecordingengineutils_p.h + recordingengine/qffmpegrecordingengineutils.cpp + recordingengine/qffmpegvideoencoder_p.h + recordingengine/qffmpegvideoencoder.cpp + recordingengine/qffmpegvideoencoderutils_p.h + recordingengine/qffmpegvideoencoderutils.cpp + recordingengine/qffmpegvideoframeencoder_p.h + recordingengine/qffmpegvideoframeencoder.cpp + DEFINES QT_COMPILING_FFMPEG LIBRARIES Qt::MultimediaPrivate Qt::CorePrivate - FFmpeg::avformat FFmpeg::avcodec FFmpeg::swresample FFmpeg::swscale FFmpeg::avutil ) -qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_ffmpeg AND QT_FEATURE_vaapi - SOURCES - qffmpeghwaccel_vaapi.cpp qffmpeghwaccel_vaapi_p.h - LIBRARIES - VAAPI::VAAPI - EGL::EGL -) +if (LINUX OR ANDROID) + # We have 2 options: link shared stubs to QFFmpegMediaPlugin vs + # static compilation of the needed stubs to the FFmpeg plugin. + # Currently, we chose the second option so that user could trivially + # remove the FFmpeg libs we ship. + # Set QT_LINK_STUBS_TO_FFMPEG_PLUGIN = TRUE to change the behavior. + + # set(QT_LINK_STUBS_TO_FFMPEG_PLUGIN TRUE) + + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/QtAddFFmpegStubs.cmake") + qt_internal_multimedia_add_ffmpeg_stubs() +endif() + + +if (QT_FEATURE_vaapi) + qt_internal_extend_target(QFFmpegMediaPlugin + SOURCES + qffmpeghwaccel_vaapi.cpp qffmpeghwaccel_vaapi_p.h + NO_UNITY_BUILD_SOURCES + # Conflicts with macros defined in X11.h, and Xlib.h + qffmpeghwaccel_vaapi.cpp + LIBRARIES + EGL::EGL + ) + + list(FIND FFMPEG_STUBS "va" va_stub_index) + if (NOT QT_LINK_STUBS_TO_FFMPEG_PLUGIN AND (FFMPEG_SHARED_LIBRARIES OR ${va_stub_index} EQUAL -1)) + target_compile_definitions(QFFmpegMediaPlugin PRIVATE Q_FFMPEG_PLUGIN_STUBS_ONLY) + qt_internal_multimedia_find_vaapi_soversion() + qt_internal_multimedia_add_private_stub_to_plugin("va") + endif() +endif() + qt_internal_extend_target(QFFmpegMediaPlugin CONDITION APPLE SOURCES @@ -54,6 +125,11 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION APPLE ../darwin/camera/avfcamerautility_p.h ../darwin/camera/avfcamerautility.mm qffmpeghwaccel_videotoolbox.mm qffmpeghwaccel_videotoolbox_p.h qavfcamera.mm qavfcamera_p.h + qavfsamplebufferdelegate.mm qavfsamplebufferdelegate_p.h + + NO_UNITY_BUILD_SOURCES + qffmpeghwaccel_videotoolbox.mm # AVMediaType clash between libavformat and AVFoundation + INCLUDE_DIRECTORIES ../darwin ../darwin/camera @@ -64,39 +140,121 @@ qt_internal_extend_target(QFFmpegMediaPlugin CONDITION APPLE ${FWCoreMedia} ${FWCoreVideo} ${FWVideoToolbox} + ${FWSecurity} AVFoundation::AVFoundation ) +qt_internal_extend_target(QFFmpegMediaPlugin CONDITION MACOS + SOURCES + qavfscreencapture.mm qavfscreencapture_p.h + qcgwindowcapture.mm qcgwindowcapture_p.h + qcgcapturablewindows.mm qcgcapturablewindows_p.h +) + qt_internal_extend_target(QFFmpegMediaPlugin CONDITION WIN32 SOURCES ../windows/qwindowsvideodevices.cpp ../windows/qwindowsvideodevices_p.h qwindowscamera.cpp qwindowscamera_p.h qffmpeghwaccel_d3d11.cpp qffmpeghwaccel_d3d11_p.h + qgdiwindowcapture.cpp qgdiwindowcapture_p.h + qffmpegscreencapture_dxgi.cpp qffmpegscreencapture_dxgi_p.h + qwincapturablewindows.cpp qwincapturablewindows_p.h INCLUDE_DIRECTORIES ../windows LIBRARIES + Qt::MultimediaPrivate WMF::WMF mfreadwrite ) +qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_cpp_winrt + SOURCES + qffmpegwindowcapture_uwp.cpp qffmpegwindowcapture_uwp_p.h + INCLUDE_DIRECTORIES + ../windows + LIBRARIES + Dwmapi + Dxva2 + windowsapp +) + +qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_xlib + SOURCES + qx11surfacecapture.cpp qx11surfacecapture_p.h + qx11capturablewindows.cpp qx11capturablewindows_p.h + LIBRARIES + X11 + Xrandr + Xext +) + +qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_eglfs + SOURCES + qeglfsscreencapture.cpp qeglfsscreencapture_p.h + qopenglvideobuffer.cpp qopenglvideobuffer_p.h + LIBRARIES + Qt::OpenGLPrivate + Qt::Quick +) + +set_source_files_properties(qx11surfacecapture.cpp qx11capturablewindows.cpp # X headers + PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON) + qt_internal_extend_target(QFFmpegMediaPlugin CONDITION QT_FEATURE_linux_v4l SOURCES qv4l2camera.cpp qv4l2camera_p.h + qv4l2filedescriptor.cpp qv4l2filedescriptor_p.h + qv4l2memorytransfer.cpp qv4l2memorytransfer_p.h + qv4l2cameradevices.cpp qv4l2cameradevices_p.h ) if (ANDROID) qt_internal_extend_target(QFFmpegMediaPlugin + SOURCES + qffmpeghwaccel_mediacodec.cpp qffmpeghwaccel_mediacodec_p.h + qandroidcamera_p.h qandroidcamera.cpp + qandroidvideodevices.cpp qandroidvideodevices_p.h + qandroidcameraframe_p.h qandroidcameraframe.cpp + qandroidimagecapture_p.h qandroidimagecapture.cpp + ../android/wrappers/jni/androidsurfacetexture_p.h + ../android/wrappers/jni/androidsurfacetexture.cpp + NO_UNITY_BUILD_SOURCES + # Duplicate declration of JNI Classes using `Q_DECLARE_JNI_CLASS` + qandroidcamera.cpp + qandroidvideodevices.cpp + qandroidcameraframe.cpp INCLUDE_DIRECTORIES ${FFMPEG_DIR}/include + ../android/wrappers/jni/ + LIBRARIES + OpenSLES + mediandk + android ) set_property(TARGET QFFmpegMediaPlugin APPEND PROPERTY QT_ANDROID_LIB_DEPENDENCIES - plugins/multimedia/libplugins_multimedia_ffmegmediaplugin.so + ${INSTALL_PLUGINSDIR}/multimedia/libplugins_multimedia_ffmpegmediaplugin.so ) set_property(TARGET QFFmpegMediaPlugin APPEND PROPERTY QT_ANDROID_PERMISSIONS - android.permission.CAMERA android.permission.RECORD_AUDIO + android.permission.CAMERA + android.permission.RECORD_AUDIO android.permission.BLUETOOTH android.permission.MODIFY_AUDIO_SETTINGS ) endif() + +# TODO: get libs from FindFFmpeg.cmake +set(ffmpeg_libs FFmpeg::avformat FFmpeg::avcodec FFmpeg::swresample FFmpeg::swscale FFmpeg::avutil) + +if (QT_DEPLOY_FFMPEG AND NOT BUILD_SHARED_LIBS AND NOT UIKIT) + message(FATAL_ERROR "QT_DEPLOY_FFMPEG is not implemented yet for static builds") +endif() + +if (QT_DEPLOY_FFMPEG AND FFMPEG_SHARED_LIBRARIES AND (BUILD_SHARED_LIBS OR UIKIT)) + include("${CMAKE_CURRENT_SOURCE_DIR}/cmake/QtDeployFFmpeg.cmake") + qt_internal_multimedia_copy_or_install_ffmpeg() +endif() + +qt_internal_extend_target(QFFmpegMediaPlugin LIBRARIES ${ffmpeg_libs}) + diff --git a/src/plugins/multimedia/ffmpeg/cmake/QtAddFFmpegStubs.cmake b/src/plugins/multimedia/ffmpeg/cmake/QtAddFFmpegStubs.cmake new file mode 100644 index 000000000..5778ae4d2 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/cmake/QtAddFFmpegStubs.cmake @@ -0,0 +1,199 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +# Utilities + +function(qt_internal_multimedia_find_ffmpeg_stubs) + foreach (stub ${FFMPEG_STUBS}) + if (${stub} MATCHES ${vaapi_regex}) + set(ffmpeg_has_vaapi TRUE PARENT_SCOPE) + elseif (${stub} MATCHES ${openssl_regex}) + set(ffmpeg_has_openssl TRUE PARENT_SCOPE) + else() + set(unknown_ffmpeg_stubs + ${unknown_ffmpeg_stubs} ${stub} PARENT_SCOPE) + endif() + endforeach() +endfunction() + +function(qt_internal_multimedia_check_ffmpeg_stubs_configuration) + if (NOT LINUX AND NOT ANDROID) + message(FATAL_ERROR "Currently, stubs are supported on Linux and Android") + endif() + + if (unknown_ffmpeg_stubs) + message(FATAL_ERROR "Unknown ffmpeg stubs: ${unknown_ffmpeg_stubs}") + endif() + + if (BUILD_SHARED_LIBS AND FFMPEG_SHARED_LIBRARIES AND FFMPEG_STUBS AND NOT QT_DEPLOY_FFMPEG) + message(FATAL_ERROR + "FFmpeg stubs have been found but QT_DEPLOY_FFMPEG is not specified. " + "Set -DQT_DEPLOY_FFMPEG=TRUE to continue.") + endif() + + if (ffmpeg_has_vaapi AND NOT QT_FEATURE_vaapi) + message(FATAL_ERROR + "QT_FEATURE_vaapi is OFF but FFmpeg includes VAAPI.") + elseif (NOT ffmpeg_has_vaapi AND QT_FEATURE_vaapi) + message(WARNING + "QT_FEATURE_vaapi is ON " + "but FFmpeg includes VAAPI and dynamic symbols resolve is enabled.") + elseif(ffmpeg_has_vaapi AND NOT VAAPI_SUFFIX) + message(FATAL_ERROR "Cannot find VAAPI_SUFFIX, fix FindVAAPI.cmake") + elseif (ffmpeg_has_vaapi AND "${VAAPI_SUFFIX}" MATCHES "^1\\.32.*") + # drop the ancient vaapi version to avoid ABI problems + message(FATAL_ERROR "VAAPI ${VAAPI_SUFFIX} is not supported") + endif() + + if (ffmpeg_has_openssl AND NOT QT_FEATURE_openssl) + message(FATAL_ERROR + "QT_FEATURE_openssl is OFF but FFmpeg includes OpenSSL.") + endif() +endfunction() + +macro(qt_internal_multimedia_find_vaapi_soversion) + string(REGEX MATCH "^[0-9]+" va_soversion "${VAAPI_SUFFIX}") + + set(va-drm_soversion "${va_soversion}") + set(va-x11_soversion "${va_soversion}") +endmacro() + +macro(qt_internal_multimedia_find_openssl_soversion) + # Update OpenSSL variables since OPENSSL_SSL_LIBRARY is not propagated to this place in some cases. + qt_find_package(OpenSSL) + + if (NOT OPENSSL_INCLUDE_DIR AND OPENSSL_ROOT_DIR) + set(OPENSSL_INCLUDE_DIR "${OPENSSL_ROOT_DIR}/include") + endif() + + if (LINUX) + if (NOT OPENSSL_SSL_LIBRARY) + message(FATAL_ERROR "OPENSSL_SSL_LIBRARY is not found") + endif() + + get_filename_component(ssl_lib_realpath "${OPENSSL_SSL_LIBRARY}" REALPATH) + string(REGEX MATCH "[0-9]+(\\.[0-9]+)*$" ssl_soversion "${ssl_lib_realpath}") + string(REGEX REPLACE "^3(\\..*|$)" "3" ssl_soversion "${ssl_soversion}") + endif() + + #TODO: enhance finding openssl version and throw an error if it's not found. + + set(crypto_soversion "${ssl_soversion}") +endmacro() + +function(qt_internal_multimedia_set_stub_version_script stub stub_target) + if ("${stub}" MATCHES "${openssl_regex}") + if ("${ssl_soversion}" STREQUAL "3" OR + (NOT ssl_soversion AND "${OPENSSL_VERSION}" MATCHES "^3\\..*")) + # Symbols in OpenSSL 1.* are not versioned. + set(file_name "openssl3.ver") + endif() + elseif("${stub}" STREQUAL "va") + set(file_name "va.ver") + endif() + + if (file_name) + set(version_script "${CMAKE_CURRENT_SOURCE_DIR}/symbolstubs/${file_name}") + set_property(TARGET ${stub_target} APPEND_STRING + PROPERTY LINK_FLAGS " -Wl,--version-script=${version_script}") + set_target_properties(${stub_target} PROPERTIES LINK_DEPENDS ${version_script}) + source_group("Stubs Version Scripts" FILES ${version_script}) + endif() +endfunction() + +function(qt_internal_multimedia_set_stub_output stub stub_target) + set(output_dir "${QT_BUILD_DIR}/${INSTALL_LIBDIR}") + + set_target_properties(${stub_target} PROPERTIES + RUNTIME_OUTPUT_DIRECTORY "${output_dir}" + LIBRARY_OUTPUT_DIRECTORY "${output_dir}" + ) + + if (${stub}_soversion) + set_target_properties(${stub_target} PROPERTIES + VERSION "${${stub}_soversion}" + SOVERSION "${${stub}_soversion}") + endif() + + qt_apply_rpaths(TARGET ${stub_target} INSTALL_PATH "${INSTALL_LIBDIR}" RELATIVE_RPATH) +endfunction() + +function(qt_internal_multimedia_set_stub_include_directories stub target) + qt_internal_extend_target(${target} + CONDITION ${stub} MATCHES "${openssl_regex}" + INCLUDE_DIRECTORIES "${OPENSSL_INCLUDE_DIR}") + + qt_internal_extend_target(${target} + CONDITION ${stub} MATCHES "${vaapi_regex}" + INCLUDE_DIRECTORIES "${VAAPI_INCLUDE_DIR}") +endfunction() + +function(qt_internal_multimedia_set_stub_symbols_visibility stub stub_target) + set_target_properties(${stub_target} PROPERTIES + C_VISIBILITY_PRESET hidden + CXX_VISIBILITY_PRESET hidden) + target_compile_definitions(${stub_target} PRIVATE Q_EXPORT_STUB_SYMBOLS) +endfunction() + +function(qt_internal_multimedia_set_stub_libraries stub stub_target) + qt_internal_extend_target(${stub_target} LIBRARIES Qt::Core Qt::MultimediaPrivate) + + if (LINK_STUBS_TO_FFMPEG_PLUGIN AND ${stub} STREQUAL "va") + qt_internal_extend_target(QFFmpegMediaPlugin LIBRARIES ${stub_target}) + endif() +endfunction() + +function(qt_internal_multimedia_define_stub_needed_version stub target) + string(TOUPPER ${stub} prefix) + string(REPLACE "-" "_" prefix ${prefix}) + + target_compile_definitions(${target} PRIVATE + "${prefix}_NEEDED_SOVERSION=\"${${stub}_soversion}\"") +endfunction() + +function(qt_internal_multimedia_add_shared_stub stub) + set(stub_target "Qt${PROJECT_VERSION_MAJOR}FFmpegStub-${stub}") + + qt_add_library(${stub_target} SHARED "symbolstubs/qffmpegsymbols-${stub}.cpp") + + qt_internal_multimedia_set_stub_include_directories(${stub} ${stub_target}) + qt_internal_multimedia_set_stub_output(${stub} ${stub_target}) + qt_internal_multimedia_set_stub_symbols_visibility(${stub} ${stub_target}) + qt_internal_multimedia_set_stub_version_script(${stub} ${stub_target}) + qt_internal_multimedia_define_stub_needed_version(${stub} ${stub_target}) + qt_internal_multimedia_set_stub_libraries(${stub} ${stub_target}) + + qt_install(TARGETS ${stub_target} LIBRARY NAMELINK_SKIP) +endfunction() + +function(qt_internal_multimedia_add_private_stub_to_plugin stub) + qt_internal_multimedia_set_stub_include_directories(${stub} QFFmpegMediaPlugin) + qt_internal_multimedia_define_stub_needed_version(${stub} QFFmpegMediaPlugin) + qt_internal_extend_target(QFFmpegMediaPlugin SOURCES "symbolstubs/qffmpegsymbols-${stub}.cpp") +endfunction() + +# Main function + +set(vaapi_regex "^(va|va-drm|va-x11)$") +set(openssl_regex "^(ssl|crypto)$") + +function(qt_internal_multimedia_add_ffmpeg_stubs) + qt_internal_multimedia_find_ffmpeg_stubs() + qt_internal_multimedia_check_ffmpeg_stubs_configuration() + + if (ffmpeg_has_vaapi) + qt_internal_multimedia_find_vaapi_soversion() + endif() + + if (ffmpeg_has_openssl) + qt_internal_multimedia_find_openssl_soversion() + endif() + + foreach (stub ${FFMPEG_STUBS}) + if (FFMPEG_SHARED_LIBRARIES) + qt_internal_multimedia_add_shared_stub("${stub}") + else() + qt_internal_multimedia_add_private_stub_to_plugin("${stub}") + endif() + endforeach() +endfunction() diff --git a/src/plugins/multimedia/ffmpeg/cmake/QtDeployFFmpeg.cmake b/src/plugins/multimedia/ffmpeg/cmake/QtDeployFFmpeg.cmake new file mode 100644 index 000000000..5e7d4b552 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/cmake/QtDeployFFmpeg.cmake @@ -0,0 +1,43 @@ +# Copyright (C) 2023 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +function(qt_internal_multimedia_set_ffmpeg_link_directory directory) + foreach (lib ${ffmpeg_libs} FFmpeg) + set_target_properties(${lib} PROPERTIES INTERFACE_LINK_DIRECTORIES ${directory}) + endforeach() +endfunction() + +function(qt_internal_multimedia_copy_or_install_ffmpeg) + if (WIN32) + set(install_dir ${INSTALL_BINDIR}) + else() + set(install_dir ${INSTALL_LIBDIR}) + endif() + + if (QT_WILL_INSTALL) + qt_install(FILES "${FFMPEG_SHARED_LIBRARIES}" DESTINATION ${install_dir}) + else() + # elseif(NOT WIN32) actually we can just drop the coping for unix platforms + # However, it makes sense to copy anyway for consistency: + # in order to have the same configuration for developer builds. + + set(ffmpeg_output_dir "${QT_BUILD_DIR}/${install_dir}") + file(MAKE_DIRECTORY ${ffmpeg_output_dir}) + + foreach(lib_path ${FFMPEG_SHARED_LIBRARIES}) + get_filename_component(lib_name ${lib_path} NAME) + if(NOT EXISTS "${ffmpeg_output_dir}/${lib_name}") + file(COPY ${lib_path} DESTINATION ${ffmpeg_output_dir}) + endif() + endforeach() + + # On Windows, shared linking goes through 'integration' static libs, + # otherwise we should link the directory with copied libs + if (NOT WIN32) + qt_internal_multimedia_set_ffmpeg_link_directory(${ffmpeg_output_dir}) + endif() + endif() + + # Should we set the compile definition for the plugin or for the QtMM module? + # target_compile_definitions(QFFmpegMediaPlugin PRIVATE FFMPEG_DEPLOY_FOLDER="${FFMPEG_DEPLOY_FOLDER}") +endfunction() diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp new file mode 100644 index 000000000..773e573e5 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer.cpp @@ -0,0 +1,407 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegaudiorenderer_p.h" +#include "qaudiosink.h" +#include "qaudiooutput.h" +#include "qaudiobufferoutput.h" +#include "private/qplatformaudiooutput_p.h" +#include <QtCore/qloggingcategory.h> + +#include "qffmpegresampler_p.h" +#include "qffmpegmediaformatinfo_p.h" + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcAudioRenderer, "qt.multimedia.ffmpeg.audiorenderer"); + +namespace QFFmpeg { + +using namespace std::chrono_literals; +using namespace std::chrono; + +namespace { +constexpr auto DesiredBufferTime = 110000us; +constexpr auto MinDesiredBufferTime = 22000us; +constexpr auto MaxDesiredBufferTime = 64000us; +constexpr auto MinDesiredFreeBufferTime = 10000us; + +// It might be changed with #ifdef, as on Linux, QPulseAudioSink has quite unstable timings, +// and it needs much more time to make sure that the buffer is overloaded. +constexpr auto BufferLoadingMeasureTime = 400ms; + +constexpr auto DurationBias = 2ms; // avoids extra timer events + +qreal sampleRateFactor() { + // Test purposes: + // + // The env var describes a factor for the sample rate of + // audio data that we feed to the audio sink. + // + // In some cases audio sink might consume data slightly slower or faster than expected; + // even though the synchronization in the audio renderer is supposed to handle it, + // it makes sense to experiment with QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR != 1. + // + // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR > 1 (e.g. 1.01 - 1.1) to test high buffer loading + // or try compensating too fast data consumption by the audio sink. + // Set QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR < 1 to test low buffer loading + // or try compensating too slow data consumption by the audio sink. + + + static const qreal result = []() { + const auto sampleRateFactorStr = qEnvironmentVariable("QT_MEDIA_PLAYER_AUDIO_SAMPLE_RATE_FACTOR"); + bool ok = false; + const auto result = sampleRateFactorStr.toDouble(&ok); + return ok ? result : 1.; + }(); + + return result; +} + +QAudioFormat audioFormatFromFrame(const Frame &frame) +{ + return QFFmpegMediaFormatInfo::audioFormatFromCodecParameters( + frame.codec()->stream()->codecpar); +} + +std::unique_ptr<QFFmpegResampler> createResampler(const Frame &frame, + const QAudioFormat &outputFormat) +{ + return std::make_unique<QFFmpegResampler>(frame.codec(), outputFormat, frame.pts()); +} + +} // namespace + +AudioRenderer::AudioRenderer(const TimeController &tc, QAudioOutput *output, + QAudioBufferOutput *bufferOutput) + : Renderer(tc), m_output(output), m_bufferOutput(bufferOutput) +{ + if (output) { + // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294 + connect(output, &QAudioOutput::deviceChanged, this, &AudioRenderer::onDeviceChanged); + connect(output, &QAudioOutput::volumeChanged, this, &AudioRenderer::updateVolume); + connect(output, &QAudioOutput::mutedChanged, this, &AudioRenderer::updateVolume); + } +} + +void AudioRenderer::setOutput(QAudioOutput *output) +{ + setOutputInternal(m_output, output, [this](QAudioOutput *) { onDeviceChanged(); }); +} + +void AudioRenderer::setOutput(QAudioBufferOutput *bufferOutput) +{ + setOutputInternal(m_bufferOutput, bufferOutput, + [this](QAudioBufferOutput *) { m_bufferOutputChanged = true; }); +} + +AudioRenderer::~AudioRenderer() +{ + freeOutput(); +} + +void AudioRenderer::updateVolume() +{ + if (m_sink) + m_sink->setVolume(m_output->isMuted() ? 0.f : m_output->volume()); +} + +void AudioRenderer::onDeviceChanged() +{ + m_deviceChanged = true; +} + +Renderer::RenderingResult AudioRenderer::renderInternal(Frame frame) +{ + if (frame.isValid()) + updateOutputs(frame); + + // push to sink first in order not to waste time on resampling + // for QAudioBufferOutput + const RenderingResult result = pushFrameToOutput(frame); + + if (m_lastFramePushDone) + pushFrameToBufferOutput(frame); + // else // skip pushing the same data to QAudioBufferOutput + + m_lastFramePushDone = result.done; + + return result; +} + +AudioRenderer::RenderingResult AudioRenderer::pushFrameToOutput(const Frame &frame) +{ + if (!m_ioDevice || !m_resampler) + return {}; + + Q_ASSERT(m_sink); + + auto firstFrameFlagGuard = qScopeGuard([&]() { m_firstFrameToSink = false; }); + + const SynchronizationStamp syncStamp{ m_sink->state(), m_sink->bytesFree(), + m_bufferedData.offset, Clock::now() }; + + if (!m_bufferedData.isValid()) { + if (!frame.isValid()) { + if (std::exchange(m_drained, true)) + return {}; + + const auto time = bufferLoadingTime(syncStamp); + + qCDebug(qLcAudioRenderer) << "Draining AudioRenderer, time:" << time; + + return { time.count() == 0, time }; + } + + m_bufferedData = { m_resampler->resample(frame.avFrame()) }; + } + + if (m_bufferedData.isValid()) { + // synchronize after "QIODevice::write" to deliver audio data to the sink ASAP. + auto syncGuard = qScopeGuard([&]() { updateSynchronization(syncStamp, frame); }); + + const auto bytesWritten = m_ioDevice->write(m_bufferedData.data(), m_bufferedData.size()); + + m_bufferedData.offset += bytesWritten; + + if (m_bufferedData.size() <= 0) { + m_bufferedData = {}; + + return {}; + } + + const auto remainingDuration = durationForBytes(m_bufferedData.size()); + + return { false, + std::min(remainingDuration + DurationBias, m_timings.actualBufferDuration / 2) }; + } + + return {}; +} + +void AudioRenderer::pushFrameToBufferOutput(const Frame &frame) +{ + if (!m_bufferOutput) + return; + + Q_ASSERT(m_bufferOutputResampler); + + if (frame.isValid()) { + // TODO: get buffer from m_bufferedData if resample formats are equal + QAudioBuffer buffer = m_resampler->resample(frame.avFrame()); + emit m_bufferOutput->audioBufferReceived(buffer); + } else { + emit m_bufferOutput->audioBufferReceived({}); + } +} + +void AudioRenderer::onPlaybackRateChanged() +{ + m_resampler.reset(); +} + +int AudioRenderer::timerInterval() const +{ + constexpr auto MaxFixableInterval = 50; // ms + + const auto interval = Renderer::timerInterval(); + + if (m_firstFrameToSink || !m_sink || m_sink->state() != QAudio::IdleState + || interval > MaxFixableInterval) + return interval; + + return 0; +} + +void AudioRenderer::onPauseChanged() +{ + m_firstFrameToSink = true; + Renderer::onPauseChanged(); +} + +void AudioRenderer::initResempler(const Frame &frame) +{ + // We recreate resampler whenever format is changed + + auto resamplerFormat = m_sinkFormat; + resamplerFormat.setSampleRate( + qRound(m_sinkFormat.sampleRate() / playbackRate() * sampleRateFactor())); + m_resampler = createResampler(frame, resamplerFormat); +} + +void AudioRenderer::freeOutput() +{ + qCDebug(qLcAudioRenderer) << "Free audio output"; + if (m_sink) { + m_sink->reset(); + + // TODO: inestigate if it's enough to reset the sink without deleting + m_sink.reset(); + } + + m_ioDevice = nullptr; + + m_bufferedData = {}; + m_deviceChanged = false; + m_sinkFormat = {}; + m_timings = {}; + m_bufferLoadingInfo = {}; +} + +void AudioRenderer::updateOutputs(const Frame &frame) +{ + if (m_deviceChanged) { + freeOutput(); + m_resampler.reset(); + } + + if (m_bufferOutput) { + if (m_bufferOutputChanged) { + m_bufferOutputChanged = false; + m_bufferOutputResampler.reset(); + } + + if (!m_bufferOutputResampler) { + QAudioFormat outputFormat = m_bufferOutput->format(); + if (!outputFormat.isValid()) + outputFormat = audioFormatFromFrame(frame); + m_bufferOutputResampler = createResampler(frame, outputFormat); + } + } + + if (!m_output) + return; + + if (!m_sinkFormat.isValid()) { + m_sinkFormat = audioFormatFromFrame(frame); + m_sinkFormat.setChannelConfig(m_output->device().channelConfiguration()); + } + + if (!m_sink) { + // Insert a delay here to test time offset synchronization, e.g. QThread::sleep(1) + m_sink = std::make_unique<QAudioSink>(m_output->device(), m_sinkFormat); + updateVolume(); + m_sink->setBufferSize(m_sinkFormat.bytesForDuration(DesiredBufferTime.count())); + m_ioDevice = m_sink->start(); + m_firstFrameToSink = true; + + connect(m_sink.get(), &QAudioSink::stateChanged, this, + &AudioRenderer::onAudioSinkStateChanged); + + m_timings.actualBufferDuration = durationForBytes(m_sink->bufferSize()); + m_timings.maxSoundDelay = qMin(MaxDesiredBufferTime, + m_timings.actualBufferDuration - MinDesiredFreeBufferTime); + m_timings.minSoundDelay = MinDesiredBufferTime; + + Q_ASSERT(DurationBias < m_timings.minSoundDelay + && m_timings.maxSoundDelay < m_timings.actualBufferDuration); + } + + if (!m_resampler) + initResempler(frame); +} + +void AudioRenderer::updateSynchronization(const SynchronizationStamp &stamp, const Frame &frame) +{ + if (!frame.isValid()) + return; + + Q_ASSERT(m_sink); + + const auto bufferLoadingTime = this->bufferLoadingTime(stamp); + const auto currentFrameDelay = frameDelay(frame, stamp.timePoint); + const auto writtenTime = durationForBytes(stamp.bufferBytesWritten); + const auto soundDelay = currentFrameDelay + bufferLoadingTime - writtenTime; + + auto synchronize = [&](microseconds fixedDelay, microseconds targetSoundDelay) { + // TODO: investigate if we need sample compensation here + + changeRendererTime(fixedDelay - targetSoundDelay); + if (qLcAudioRenderer().isDebugEnabled()) { + // clang-format off + qCDebug(qLcAudioRenderer) + << "Change rendering time:" + << "\n First frame:" << m_firstFrameToSink + << "\n Delay (frame+buffer-written):" << currentFrameDelay << "+" + << bufferLoadingTime << "-" + << writtenTime << "=" + << soundDelay + << "\n Fixed delay:" << fixedDelay + << "\n Target delay:" << targetSoundDelay + << "\n Buffer durations (min/max/limit):" << m_timings.minSoundDelay + << m_timings.maxSoundDelay + << m_timings.actualBufferDuration + << "\n Audio sink state:" << stamp.audioSinkState; + // clang-format on + } + }; + + const auto loadingType = soundDelay > m_timings.maxSoundDelay ? BufferLoadingInfo::High + : soundDelay < m_timings.minSoundDelay ? BufferLoadingInfo::Low + : BufferLoadingInfo::Moderate; + + if (loadingType != m_bufferLoadingInfo.type) { + // qCDebug(qLcAudioRenderer) << "Change buffer loading type:" << + // m_bufferLoadingInfo.type + // << "->" << loadingType << "soundDelay:" << soundDelay; + m_bufferLoadingInfo = { loadingType, stamp.timePoint, soundDelay }; + } + + if (loadingType != BufferLoadingInfo::Moderate) { + const auto isHigh = loadingType == BufferLoadingInfo::High; + const auto shouldHandleIdle = stamp.audioSinkState == QAudio::IdleState && !isHigh; + + auto &fixedDelay = m_bufferLoadingInfo.delay; + + fixedDelay = shouldHandleIdle ? soundDelay + : isHigh ? qMin(soundDelay, fixedDelay) + : qMax(soundDelay, fixedDelay); + + if (stamp.timePoint - m_bufferLoadingInfo.timePoint > BufferLoadingMeasureTime + || (m_firstFrameToSink && isHigh) || shouldHandleIdle) { + const auto targetDelay = isHigh + ? (m_timings.maxSoundDelay + m_timings.minSoundDelay) / 2 + : m_timings.minSoundDelay + DurationBias; + + synchronize(fixedDelay, targetDelay); + m_bufferLoadingInfo = { BufferLoadingInfo::Moderate, stamp.timePoint, targetDelay }; + } + } +} + +microseconds AudioRenderer::bufferLoadingTime(const SynchronizationStamp &syncStamp) const +{ + Q_ASSERT(m_sink); + + if (syncStamp.audioSinkState == QAudio::IdleState) + return microseconds(0); + + const auto bytes = qMax(m_sink->bufferSize() - syncStamp.audioSinkBytesFree, 0); + +#ifdef Q_OS_ANDROID + // The hack has been added due to QAndroidAudioSink issues (QTBUG-118609). + // The method QAndroidAudioSink::bytesFree returns 0 or bufferSize, intermediate values are not + // available now; to be fixed. + if (bytes == 0) + return m_timings.minSoundDelay + MinDesiredBufferTime; +#endif + + return durationForBytes(bytes); +} + +void AudioRenderer::onAudioSinkStateChanged(QAudio::State state) +{ + if (state == QAudio::IdleState && !m_firstFrameToSink) + scheduleNextStep(); +} + +microseconds AudioRenderer::durationForBytes(qsizetype bytes) const +{ + return microseconds(m_sinkFormat.durationForBytes(static_cast<qint32>(bytes))); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegaudiorenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h new file mode 100644 index 000000000..9a22a8a48 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegaudiorenderer_p.h @@ -0,0 +1,132 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGAUDIORENDERER_P_H +#define QFFMPEGAUDIORENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegrenderer_p.h" + +#include "qaudiobuffer.h" + +QT_BEGIN_NAMESPACE + +class QAudioOutput; +class QAudioBufferOutput; +class QAudioSink; +class QFFmpegResampler; + +namespace QFFmpeg { + +class AudioRenderer : public Renderer +{ + Q_OBJECT +public: + AudioRenderer(const TimeController &tc, QAudioOutput *output, QAudioBufferOutput *bufferOutput); + + void setOutput(QAudioOutput *output); + + void setOutput(QAudioBufferOutput *bufferOutput); + + ~AudioRenderer() override; + +protected: + using Microseconds = std::chrono::microseconds; + struct SynchronizationStamp + { + QAudio::State audioSinkState = QAudio::IdleState; + qsizetype audioSinkBytesFree = 0; + qsizetype bufferBytesWritten = 0; + TimePoint timePoint = TimePoint::max(); + }; + + struct BufferLoadingInfo + { + enum Type { Low, Moderate, High }; + Type type = Moderate; + TimePoint timePoint = TimePoint::max(); + Microseconds delay = Microseconds(0); + }; + + struct AudioTimings + { + Microseconds actualBufferDuration = Microseconds(0); + Microseconds maxSoundDelay = Microseconds(0); + Microseconds minSoundDelay = Microseconds(0); + }; + + struct BufferedDataWithOffset + { + QAudioBuffer buffer; + qsizetype offset = 0; + + bool isValid() const { return buffer.isValid(); } + qsizetype size() const { return buffer.byteCount() - offset; } + const char *data() const { return buffer.constData<char>() + offset; } + }; + + RenderingResult renderInternal(Frame frame) override; + + RenderingResult pushFrameToOutput(const Frame &frame); + + void pushFrameToBufferOutput(const Frame &frame); + + void onPlaybackRateChanged() override; + + int timerInterval() const override; + + void onPauseChanged() override; + + void freeOutput(); + + void updateOutputs(const Frame &frame); + + void initResempler(const Frame &frame); + + void onDeviceChanged(); + + void updateVolume(); + + void updateSynchronization(const SynchronizationStamp &stamp, const Frame &frame); + + Microseconds bufferLoadingTime(const SynchronizationStamp &syncStamp) const; + + void onAudioSinkStateChanged(QAudio::State state); + + Microseconds durationForBytes(qsizetype bytes) const; + +private: + QPointer<QAudioOutput> m_output; + QPointer<QAudioBufferOutput> m_bufferOutput; + std::unique_ptr<QAudioSink> m_sink; + AudioTimings m_timings; + BufferLoadingInfo m_bufferLoadingInfo; + std::unique_ptr<QFFmpegResampler> m_resampler; + std::unique_ptr<QFFmpegResampler> m_bufferOutputResampler; + QAudioFormat m_sinkFormat; + + BufferedDataWithOffset m_bufferedData; + QIODevice *m_ioDevice = nullptr; + + bool m_lastFramePushDone = true; + + bool m_deviceChanged = false; + bool m_bufferOutputChanged = false; + bool m_drained = false; + bool m_firstFrameToSink = true; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGAUDIORENDERER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp new file mode 100644 index 000000000..96cfc1bfe --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec.cpp @@ -0,0 +1,105 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegcodec_p.h" +#include "qloggingcategory.h" + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcPlaybackEngineCodec, "qt.multimedia.playbackengine.codec"); + +namespace QFFmpeg { + +Codec::Data::Data(AVCodecContextUPtr context, AVStream *stream, AVFormatContext *formatContext, + std::unique_ptr<QFFmpeg::HWAccel> hwAccel) + : context(std::move(context)), stream(stream), hwAccel(std::move(hwAccel)) +{ + if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) + pixelAspectRatio = av_guess_sample_aspect_ratio(formatContext, stream, nullptr); +} + +QMaybe<Codec> Codec::create(AVStream *stream, AVFormatContext *formatContext) +{ + if (!stream) + return { "Invalid stream" }; + + if (stream->codecpar->codec_type == AVMEDIA_TYPE_VIDEO) { + auto hwCodec = create(stream, formatContext, Hw); + if (hwCodec) + return hwCodec; + + qCInfo(qLcPlaybackEngineCodec) << hwCodec.error(); + } + + auto codec = create(stream, formatContext, Sw); + if (!codec) + qCWarning(qLcPlaybackEngineCodec) << codec.error(); + + return codec; +} + +AVRational Codec::pixelAspectRatio(AVFrame *frame) const +{ + // does the same as av_guess_sample_aspect_ratio, but more efficient + return d->pixelAspectRatio.num && d->pixelAspectRatio.den ? d->pixelAspectRatio + : frame->sample_aspect_ratio; +} + +QMaybe<Codec> Codec::create(AVStream *stream, AVFormatContext *formatContext, + VideoCodecCreationPolicy videoCodecPolicy) +{ + Q_ASSERT(stream); + + if (videoCodecPolicy == Hw && stream->codecpar->codec_type != AVMEDIA_TYPE_VIDEO) + Q_ASSERT(!"Codec::create has been called with Hw policy on a non-video stream"); + + const AVCodec *decoder = nullptr; + std::unique_ptr<QFFmpeg::HWAccel> hwAccel; + + if (videoCodecPolicy == Hw) + std::tie(decoder, hwAccel) = HWAccel::findDecoderWithHwAccel(stream->codecpar->codec_id); + else + decoder = QFFmpeg::findAVDecoder(stream->codecpar->codec_id); + + if (!decoder) + return { QString("No %1 decoder found").arg(videoCodecPolicy == Hw ? "HW" : "SW") }; + + qCDebug(qLcPlaybackEngineCodec) << "found decoder" << decoder->name << "for id" << decoder->id; + + AVCodecContextUPtr context(avcodec_alloc_context3(decoder)); + if (!context) + return { "Failed to allocate a FFmpeg codec context" }; + + if (hwAccel) + context->hw_device_ctx = av_buffer_ref(hwAccel->hwDeviceContextAsBuffer()); + + if (context->codec_type != AVMEDIA_TYPE_AUDIO && context->codec_type != AVMEDIA_TYPE_VIDEO + && context->codec_type != AVMEDIA_TYPE_SUBTITLE) { + return { "Unknown codec type" }; + } + + int ret = avcodec_parameters_to_context(context.get(), stream->codecpar); + if (ret < 0) + return QStringLiteral("Failed to set FFmpeg codec parameters: %1").arg(err2str(ret)); + + // ### This still gives errors about wrong HW formats (as we accept all of them) + // But it would be good to get so we can filter out pixel format we don't support natively + context->get_format = QFFmpeg::getFormat; + + /* Init the decoder, with reference counting and threading */ + AVDictionaryHolder opts; + av_dict_set(opts, "refcounted_frames", "1", 0); + av_dict_set(opts, "threads", "auto", 0); + applyExperimentalCodecOptions(decoder, opts); + + ret = avcodec_open2(context.get(), decoder, opts); + + if (ret < 0) + return QStringLiteral("Failed to open FFmpeg codec context: %1").arg(err2str(ret)); + + return Codec(new Data(std::move(context), stream, formatContext, std::move(hwAccel))); +} + +QT_END_NAMESPACE + +} // namespace QFFmpeg diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h new file mode 100644 index 000000000..b6866ed6b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegcodec_p.h @@ -0,0 +1,69 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGCODEC_P_H +#define QFFMPEGCODEC_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qshareddata.h" +#include "qqueue.h" +#include "private/qmultimediautils_p.h" +#include "qffmpeg_p.h" +#include "qffmpeghwaccel_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class Codec +{ + struct Data + { + Data(AVCodecContextUPtr context, AVStream *stream, AVFormatContext *formatContext, + std::unique_ptr<QFFmpeg::HWAccel> hwAccel); + QAtomicInt ref; + AVCodecContextUPtr context; + AVStream *stream = nullptr; + AVRational pixelAspectRatio = { 0, 1 }; + std::unique_ptr<QFFmpeg::HWAccel> hwAccel; + }; + +public: + static QMaybe<Codec> create(AVStream *stream, AVFormatContext *formatContext); + + AVRational pixelAspectRatio(AVFrame *frame) const; + + AVCodecContext *context() const { return d->context.get(); } + AVStream *stream() const { return d->stream; } + uint streamIndex() const { return d->stream->index; } + HWAccel *hwAccel() const { return d->hwAccel.get(); } + qint64 toMs(qint64 ts) const { return timeStampMs(ts, d->stream->time_base).value_or(0); } + qint64 toUs(qint64 ts) const { return timeStampUs(ts, d->stream->time_base).value_or(0); } + +private: + enum VideoCodecCreationPolicy { + Hw, + Sw, + }; + + static QMaybe<Codec> create(AVStream *stream, AVFormatContext *formatContext, + VideoCodecCreationPolicy videoCodecPolicy); + Codec(Data *data) : d(data) { } + QExplicitlySharedDataPointer<Data> d; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGCODEC_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp new file mode 100644 index 000000000..f11d3e811 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer.cpp @@ -0,0 +1,228 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegdemuxer_p.h" +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +// 4 sec for buffering. TODO: maybe move to env var customization +static constexpr qint64 MaxBufferedDurationUs = 4'000'000; + +// around 4 sec of hdr video +static constexpr qint64 MaxBufferedSize = 32 * 1024 * 1024; + +namespace QFFmpeg { + +Q_STATIC_LOGGING_CATEGORY(qLcDemuxer, "qt.multimedia.ffmpeg.demuxer"); + +static qint64 streamTimeToUs(const AVStream *stream, qint64 time) +{ + Q_ASSERT(stream); + + const auto res = mul(time * 1000000, stream->time_base); + return res ? *res : time; +} + +static qint64 packetEndPos(const AVStream *stream, const Packet &packet) +{ + return packet.loopOffset().pos + + streamTimeToUs(stream, packet.avPacket()->pts + packet.avPacket()->duration); +} + +Demuxer::Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops) + : m_context(context), m_posWithOffset(posWithOffset), m_loops(loops) +{ + qCDebug(qLcDemuxer) << "Create demuxer." + << "pos:" << posWithOffset.pos << "loop offset:" << posWithOffset.offset.pos + << "loop index:" << posWithOffset.offset.index << "loops:" << loops; + + Q_ASSERT(m_context); + + for (auto i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) { + if (streamIndexes[i] >= 0) { + const auto trackType = static_cast<QPlatformMediaPlayer::TrackType>(i); + qCDebug(qLcDemuxer) << "Activate demuxing stream" << i << ", trackType:" << trackType; + m_streams[streamIndexes[i]] = { trackType }; + } + } +} + +void Demuxer::doNextStep() +{ + ensureSeeked(); + + Packet packet(m_posWithOffset.offset, AVPacketUPtr{ av_packet_alloc() }, id()); + if (av_read_frame(m_context, packet.avPacket()) < 0) { + ++m_posWithOffset.offset.index; + + const auto loops = m_loops.loadAcquire(); + if (loops >= 0 && m_posWithOffset.offset.index >= loops) { + qCDebug(qLcDemuxer) << "finish demuxing"; + + if (!std::exchange(m_buffered, true)) + emit packetsBuffered(); + + setAtEnd(true); + } else { + m_seeked = false; + m_posWithOffset.pos = 0; + m_posWithOffset.offset.pos = m_maxPacketsEndPos; + m_maxPacketsEndPos = 0; + + ensureSeeked(); + + qCDebug(qLcDemuxer) << "Demuxer loops changed. Index:" << m_posWithOffset.offset.index + << "Offset:" << m_posWithOffset.offset.pos; + + scheduleNextStep(false); + } + + return; + } + + auto &avPacket = *packet.avPacket(); + + const auto streamIndex = avPacket.stream_index; + const auto stream = m_context->streams[streamIndex]; + + auto it = m_streams.find(streamIndex); + if (it != m_streams.end()) { + auto &streamData = it->second; + + const auto endPos = packetEndPos(stream, packet); + m_maxPacketsEndPos = qMax(m_maxPacketsEndPos, endPos); + + // Increase buffered metrics as the packet has been processed. + + streamData.bufferedDuration += streamTimeToUs(stream, avPacket.duration); + streamData.bufferedSize += avPacket.size; + streamData.maxSentPacketsPos = qMax(streamData.maxSentPacketsPos, endPos); + updateStreamDataLimitFlag(streamData); + + if (!m_buffered && streamData.isDataLimitReached) { + m_buffered = true; + emit packetsBuffered(); + } + + if (!m_firstPacketFound) { + m_firstPacketFound = true; + const auto pos = streamTimeToUs(stream, avPacket.pts); + emit firstPacketFound(std::chrono::steady_clock::now(), pos); + } + + auto signal = signalByTrackType(it->second.trackType); + emit (this->*signal)(packet); + } + + scheduleNextStep(false); +} + +void Demuxer::onPacketProcessed(Packet packet) +{ + Q_ASSERT(packet.isValid()); + + if (packet.sourceId() != id()) + return; + + auto &avPacket = *packet.avPacket(); + + const auto streamIndex = avPacket.stream_index; + const auto stream = m_context->streams[streamIndex]; + auto it = m_streams.find(streamIndex); + + if (it != m_streams.end()) { + auto &streamData = it->second; + + // Decrease buffered metrics as new data (the packet) has been received (buffered) + + streamData.bufferedDuration -= streamTimeToUs(stream, avPacket.duration); + streamData.bufferedSize -= avPacket.size; + streamData.maxProcessedPacketPos = + qMax(streamData.maxProcessedPacketPos, packetEndPos(stream, packet)); + + Q_ASSERT(it->second.bufferedDuration >= 0); + Q_ASSERT(it->second.bufferedSize >= 0); + + updateStreamDataLimitFlag(streamData); + } + + scheduleNextStep(); +} + +bool Demuxer::canDoNextStep() const +{ + auto isDataLimitReached = [](const auto &streamIndexToData) { + return streamIndexToData.second.isDataLimitReached; + }; + + // Demuxer waits: + // - if it's paused + // - if the end has been reached + // - if streams are empty (probably, should be handled on the initialization) + // - if at least one of the streams has reached the data limit (duration or size) + + return PlaybackEngineObject::canDoNextStep() && !isAtEnd() && !m_streams.empty() + && std::none_of(m_streams.begin(), m_streams.end(), isDataLimitReached); +} + +void Demuxer::ensureSeeked() +{ + if (std::exchange(m_seeked, true)) + return; + + if ((m_context->ctx_flags & AVFMTCTX_UNSEEKABLE) == 0) { + const qint64 seekPos = m_posWithOffset.pos * AV_TIME_BASE / 1000000; + auto err = av_seek_frame(m_context, -1, seekPos, AVSEEK_FLAG_BACKWARD); + + if (err < 0) { + qCWarning(qLcDemuxer) << "Failed to seek, pos" << seekPos; + + // Drop an error of seeking to initial position of streams with undefined duration. + // This needs improvements. + if (seekPos != 0 || m_context->duration > 0) + emit error(QMediaPlayer::ResourceError, + QLatin1StringView("Failed to seek: ") + err2str(err)); + } + } + + setAtEnd(false); +} + +Demuxer::RequestingSignal Demuxer::signalByTrackType(QPlatformMediaPlayer::TrackType trackType) +{ + switch (trackType) { + case QPlatformMediaPlayer::TrackType::VideoStream: + return &Demuxer::requestProcessVideoPacket; + case QPlatformMediaPlayer::TrackType::AudioStream: + return &Demuxer::requestProcessAudioPacket; + case QPlatformMediaPlayer::TrackType::SubtitleStream: + return &Demuxer::requestProcessSubtitlePacket; + default: + Q_ASSERT(!"Unknown track type"); + } + + return nullptr; +} + +void Demuxer::setLoops(int loopsCount) +{ + qCDebug(qLcDemuxer) << "setLoops to demuxer" << loopsCount; + m_loops.storeRelease(loopsCount); +} + +void Demuxer::updateStreamDataLimitFlag(StreamData &streamData) +{ + const auto packetsPosDiff = streamData.maxSentPacketsPos - streamData.maxProcessedPacketPos; + streamData.isDataLimitReached = + streamData.bufferedDuration >= MaxBufferedDurationUs + || (streamData.bufferedDuration == 0 && packetsPosDiff >= MaxBufferedDurationUs) + || streamData.bufferedSize >= MaxBufferedSize; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegdemuxer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h new file mode 100644 index 000000000..b72056185 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegdemuxer_p.h @@ -0,0 +1,87 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGDEMUXER_P_H +#define QFFMPEGDEMUXER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegplaybackengineobject_p.h" +#include "private/qplatformmediaplayer_p.h" +#include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" + +#include <unordered_map> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class Demuxer : public PlaybackEngineObject +{ + Q_OBJECT +public: + Demuxer(AVFormatContext *context, const PositionWithOffset &posWithOffset, + const StreamIndexes &streamIndexes, int loops); + + using RequestingSignal = void (Demuxer::*)(Packet); + static RequestingSignal signalByTrackType(QPlatformMediaPlayer::TrackType trackType); + + void setLoops(int loopsCount); + +public slots: + void onPacketProcessed(Packet); + +signals: + void requestProcessAudioPacket(Packet); + void requestProcessVideoPacket(Packet); + void requestProcessSubtitlePacket(Packet); + void firstPacketFound(TimePoint tp, qint64 trackPos); + void packetsBuffered(); + +private: + bool canDoNextStep() const override; + + void doNextStep() override; + + void ensureSeeked(); + +private: + struct StreamData + { + QPlatformMediaPlayer::TrackType trackType = QPlatformMediaPlayer::TrackType::NTrackTypes; + qint64 bufferedDuration = 0; + qint64 bufferedSize = 0; + + qint64 maxSentPacketsPos = 0; + qint64 maxProcessedPacketPos = 0; + + bool isDataLimitReached = false; + }; + + void updateStreamDataLimitFlag(StreamData &streamData); + +private: + AVFormatContext *m_context = nullptr; + bool m_seeked = false; + bool m_firstPacketFound = false; + std::unordered_map<int, StreamData> m_streams; + PositionWithOffset m_posWithOffset; + qint64 m_maxPacketsEndPos = 0; + QAtomicInt m_loops = QMediaPlayer::Once; + bool m_buffered = false; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE // QFFMPEGDEMUXER_P_H + +#endif diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h new file mode 100644 index 000000000..84fe2fead --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegframe_p.h @@ -0,0 +1,109 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGFRAME_P_H +#define QFFMPEGFRAME_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeg_p.h" +#include "playbackengine/qffmpegcodec_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" +#include "QtCore/qsharedpointer.h" +#include "qpointer.h" +#include "qobject.h" + +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct Frame +{ + struct Data + { + Data(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64, quint64 sourceId) + : loopOffset(offset), codec(codec), frame(std::move(f)), sourceId(sourceId) + { + Q_ASSERT(frame); + if (frame->pts != AV_NOPTS_VALUE) + pts = codec.toUs(frame->pts); + else + pts = codec.toUs(frame->best_effort_timestamp); + + if (auto frameDuration = getAVFrameDuration(*frame)) { + duration = codec.toUs(frameDuration); + } else { + const auto &avgFrameRate = codec.stream()->avg_frame_rate; + duration = mul(qint64(1000000), { avgFrameRate.den, avgFrameRate.num }).value_or(0); + } + } + Data(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + quint64 sourceId) + : loopOffset(offset), text(text), pts(pts), duration(duration), sourceId(sourceId) + { + } + + QAtomicInt ref; + LoopOffset loopOffset; + std::optional<Codec> codec; + AVFrameUPtr frame; + QString text; + qint64 pts = -1; + qint64 duration = -1; + quint64 sourceId = 0; + }; + Frame() = default; + + Frame(const LoopOffset &offset, AVFrameUPtr f, const Codec &codec, qint64 pts, + quint64 sourceIndex) + : d(new Data(offset, std::move(f), codec, pts, sourceIndex)) + { + } + Frame(const LoopOffset &offset, const QString &text, qint64 pts, qint64 duration, + quint64 sourceIndex) + : d(new Data(offset, text, pts, duration, sourceIndex)) + { + } + bool isValid() const { return !!d; } + + AVFrame *avFrame() const { return data().frame.get(); } + AVFrameUPtr takeAVFrame() { return std::move(data().frame); } + const Codec *codec() const { return data().codec ? &data().codec.value() : nullptr; } + qint64 pts() const { return data().pts; } + qint64 duration() const { return data().duration; } + qint64 end() const { return data().pts + data().duration; } + QString text() const { return data().text; } + quint64 sourceId() const { return data().sourceId; }; + const LoopOffset &loopOffset() const { return data().loopOffset; }; + qint64 absolutePts() const { return pts() + loopOffset().pos; } + qint64 absoluteEnd() const { return end() + loopOffset().pos; } + +private: + Data &data() const + { + Q_ASSERT(d); + return *d; + } + +private: + QExplicitlySharedDataPointer<Data> d; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +Q_DECLARE_METATYPE(QFFmpeg::Frame); + +#endif // QFFMPEGFRAME_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder.cpp new file mode 100644 index 000000000..3bb9aad16 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder.cpp @@ -0,0 +1,390 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegmediadataholder_p.h" + +#include "qffmpegmediametadata_p.h" +#include "qffmpegmediaformatinfo_p.h" +#include "qffmpegioutils_p.h" +#include "qiodevice.h" +#include "qdatetime.h" +#include "qloggingcategory.h" + +#include <math.h> +#include <optional> + +extern "C" { +#include "libavutil/display.h" +} + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcMediaDataHolder, "qt.multimedia.ffmpeg.mediadataholder") + +namespace QFFmpeg { + +static std::optional<qint64> streamDuration(const AVStream &stream) +{ + const auto &factor = stream.time_base; + + if (stream.duration > 0 && factor.num > 0 && factor.den > 0) { + return qint64(1000000) * stream.duration * factor.num / factor.den; + } + + // In some cases ffmpeg reports negative duration that is definitely invalid. + // However, the correct duration may be read from the metadata. + + if (stream.duration < 0) { + qCWarning(qLcMediaDataHolder) << "AVStream duration" << stream.duration + << "is invalid. Taking it from the metadata"; + } + + if (const auto duration = av_dict_get(stream.metadata, "DURATION", nullptr, 0)) { + const auto time = QTime::fromString(QString::fromUtf8(duration->value)); + return qint64(1000) * time.msecsSinceStartOfDay(); + } + + return {}; +} + +static int streamOrientation(const AVStream *stream) +{ + Q_ASSERT(stream); + + using SideDataSize = decltype(AVPacketSideData::size); + constexpr SideDataSize displayMatrixSize = sizeof(int32_t) * 9; + const auto *sideData = streamSideData(stream, AV_PKT_DATA_DISPLAYMATRIX); + if (!sideData || sideData->size < displayMatrixSize) + return 0; + + auto displayMatrix = reinterpret_cast<const int32_t *>(sideData->data); + auto rotation = static_cast<int>(std::round(av_display_rotation_get(displayMatrix))); + // Convert counterclockwise rotation angle to clockwise, restricted to 0, 90, 180 and 270 + if (rotation % 90 != 0) + return 0; + return rotation < 0 ? -rotation % 360 : -rotation % 360 + 360; +} + + +static bool colorTransferSupportsHdr(const AVStream *stream) +{ + if (!stream) + return false; + + const AVCodecParameters *codecPar = stream->codecpar; + if (!codecPar) + return false; + + const QVideoFrameFormat::ColorTransfer colorTransfer = fromAvColorTransfer(codecPar->color_trc); + + // Assume that content is using HDR if the color transfer supports high + // dynamic range. The video may still not utilize the extended range, + // but we can't determine the actual range without decoding frames. + return colorTransfer == QVideoFrameFormat::ColorTransfer_ST2084 + || colorTransfer == QVideoFrameFormat::ColorTransfer_STD_B67; +} + +QtVideo::Rotation MediaDataHolder::rotation() const +{ + int orientation = m_metaData.value(QMediaMetaData::Orientation).toInt(); + return static_cast<QtVideo::Rotation>(orientation); +} + +AVFormatContext *MediaDataHolder::avContext() +{ + return m_context.get(); +} + +int MediaDataHolder::currentStreamIndex(QPlatformMediaPlayer::TrackType trackType) const +{ + return m_currentAVStreamIndex[trackType]; +} + +static void insertMediaData(QMediaMetaData &metaData, QPlatformMediaPlayer::TrackType trackType, + const AVStream *stream) +{ + Q_ASSERT(stream); + const auto *codecPar = stream->codecpar; + + switch (trackType) { + case QPlatformMediaPlayer::VideoStream: + metaData.insert(QMediaMetaData::VideoBitRate, (int)codecPar->bit_rate); + metaData.insert(QMediaMetaData::VideoCodec, + QVariant::fromValue(QFFmpegMediaFormatInfo::videoCodecForAVCodecId( + codecPar->codec_id))); + metaData.insert(QMediaMetaData::Resolution, QSize(codecPar->width, codecPar->height)); + metaData.insert(QMediaMetaData::VideoFrameRate, + qreal(stream->avg_frame_rate.num) / qreal(stream->avg_frame_rate.den)); + metaData.insert(QMediaMetaData::Orientation, QVariant::fromValue(streamOrientation(stream))); + metaData.insert(QMediaMetaData::HasHdrContent, colorTransferSupportsHdr(stream)); + break; + case QPlatformMediaPlayer::AudioStream: + metaData.insert(QMediaMetaData::AudioBitRate, (int)codecPar->bit_rate); + metaData.insert(QMediaMetaData::AudioCodec, + QVariant::fromValue(QFFmpegMediaFormatInfo::audioCodecForAVCodecId( + codecPar->codec_id))); + break; + default: + break; + } +}; + +QPlatformMediaPlayer::TrackType MediaDataHolder::trackTypeFromMediaType(int mediaType) +{ + switch (mediaType) { + case AVMEDIA_TYPE_AUDIO: + return QPlatformMediaPlayer::AudioStream; + case AVMEDIA_TYPE_VIDEO: + return QPlatformMediaPlayer::VideoStream; + case AVMEDIA_TYPE_SUBTITLE: + return QPlatformMediaPlayer::SubtitleStream; + default: + return QPlatformMediaPlayer::NTrackTypes; + } +} + +namespace { +QMaybe<AVFormatContextUPtr, MediaDataHolder::ContextError> +loadMedia(const QUrl &mediaUrl, QIODevice *stream, const std::shared_ptr<ICancelToken> &cancelToken) +{ + const QByteArray url = mediaUrl.toString(QUrl::PreferLocalFile).toUtf8(); + + AVFormatContextUPtr context{ avformat_alloc_context() }; + + if (stream) { + if (!stream->isOpen()) { + if (!stream->open(QIODevice::ReadOnly)) + return MediaDataHolder::ContextError{ + QMediaPlayer::ResourceError, QLatin1String("Could not open source device.") + }; + } + if (!stream->isSequential()) + stream->seek(0); + + constexpr int bufferSize = 32768; + unsigned char *buffer = (unsigned char *)av_malloc(bufferSize); + context->pb = avio_alloc_context(buffer, bufferSize, false, stream, &readQIODevice, nullptr, + &seekQIODevice); + } + + AVDictionaryHolder dict; + constexpr auto NetworkTimeoutUs = "5000000"; + av_dict_set(dict, "timeout", NetworkTimeoutUs, 0); + + const QByteArray protocolWhitelist = qgetenv("QT_FFMPEG_PROTOCOL_WHITELIST"); + if (!protocolWhitelist.isNull()) + av_dict_set(dict, "protocol_whitelist", protocolWhitelist.data(), 0); + + context->interrupt_callback.opaque = cancelToken.get(); + context->interrupt_callback.callback = [](void *opaque) { + const auto *cancelToken = static_cast<const ICancelToken *>(opaque); + if (cancelToken && cancelToken->isCancelled()) + return 1; + return 0; + }; + + int ret = 0; + { + AVFormatContext *contextRaw = context.release(); + ret = avformat_open_input(&contextRaw, url.constData(), nullptr, dict); + context.reset(contextRaw); + } + + if (ret < 0) { + auto code = QMediaPlayer::ResourceError; + if (ret == AVERROR(EACCES)) + code = QMediaPlayer::AccessDeniedError; + else if (ret == AVERROR(EINVAL)) + code = QMediaPlayer::FormatError; + + return MediaDataHolder::ContextError{ code, QMediaPlayer::tr("Could not open file") }; + } + + ret = avformat_find_stream_info(context.get(), nullptr); + if (ret < 0) { + return MediaDataHolder::ContextError{ + QMediaPlayer::FormatError, + QMediaPlayer::tr("Could not find stream information for media file") + }; + } + +#ifndef QT_NO_DEBUG + av_dump_format(context.get(), 0, url.constData(), 0); +#endif + return context; +} + +} // namespace + +MediaDataHolder::Maybe MediaDataHolder::create(const QUrl &url, QIODevice *stream, + const std::shared_ptr<ICancelToken> &cancelToken) +{ + QMaybe context = loadMedia(url, stream, cancelToken); + if (context) { + // MediaDataHolder is wrapped in a shared pointer to interop with signal/slot mechanism + return QSharedPointer<MediaDataHolder>{ new MediaDataHolder{ std::move(context.value()), cancelToken } }; + } + return context.error(); +} + +MediaDataHolder::MediaDataHolder(AVFormatContextUPtr context, + const std::shared_ptr<ICancelToken> &cancelToken) + : m_cancelToken{ cancelToken } +{ + Q_ASSERT(context); + + m_context = std::move(context); + m_isSeekable = !(m_context->ctx_flags & AVFMTCTX_UNSEEKABLE); + + for (unsigned int i = 0; i < m_context->nb_streams; ++i) { + + const auto *stream = m_context->streams[i]; + const auto trackType = trackTypeFromMediaType(stream->codecpar->codec_type); + + if (trackType == QPlatformMediaPlayer::NTrackTypes) + continue; + + if (stream->disposition & AV_DISPOSITION_ATTACHED_PIC) + continue; // Ignore attached picture streams because we treat them as metadata + + auto metaData = QFFmpegMetaData::fromAVMetaData(stream->metadata); + const bool isDefault = stream->disposition & AV_DISPOSITION_DEFAULT; + + if (trackType != QPlatformMediaPlayer::SubtitleStream) { + insertMediaData(metaData, trackType, stream); + + if (isDefault && m_requestedStreams[trackType] < 0) + m_requestedStreams[trackType] = m_streamMap[trackType].size(); + } + + if (auto duration = streamDuration(*stream)) { + m_duration = qMax(m_duration, *duration); + metaData.insert(QMediaMetaData::Duration, *duration / qint64(1000)); + } + + m_streamMap[trackType].append({ (int)i, isDefault, metaData }); + } + + // With some media files, streams may be lacking duration info. Let's + // get it from ffmpeg's duration estimation instead. + if (m_duration == 0 && m_context->duration > 0ll) { + m_duration = m_context->duration; + } + + for (auto trackType : + { QPlatformMediaPlayer::VideoStream, QPlatformMediaPlayer::AudioStream }) { + auto &requestedStream = m_requestedStreams[trackType]; + auto &streamMap = m_streamMap[trackType]; + + if (requestedStream < 0 && !streamMap.empty()) + requestedStream = 0; + + if (requestedStream >= 0) + m_currentAVStreamIndex[trackType] = streamMap[requestedStream].avStreamIndex; + } + + updateMetaData(); +} + +namespace { + +/*! + \internal + + Attempt to find an attached picture from the context's streams. + This will find ID3v2 pictures on audio files, and also pictures + attached to videos. + */ +QImage getAttachedPicture(const AVFormatContext *context) +{ + if (!context) + return {}; + + for (unsigned int i = 0; i < context->nb_streams; ++i) { + const AVStream* stream = context->streams[i]; + if (!stream || !(stream->disposition & AV_DISPOSITION_ATTACHED_PIC)) + continue; + + const AVPacket *compressedImage = &stream->attached_pic; + if (!compressedImage || !compressedImage->data || compressedImage->size <= 0) + continue; + + // Feed raw compressed data to QImage::fromData, which will decompress it + // if it is a recognized format. + QImage image = QImage::fromData({ compressedImage->data, compressedImage->size }); + if (!image.isNull()) + return image; + } + + return {}; +} + +} + +void MediaDataHolder::updateMetaData() +{ + m_metaData = {}; + + if (!m_context) + return; + + m_metaData = QFFmpegMetaData::fromAVMetaData(m_context->metadata); + m_metaData.insert(QMediaMetaData::FileFormat, + QVariant::fromValue(QFFmpegMediaFormatInfo::fileFormatForAVInputFormat( + m_context->iformat))); + m_metaData.insert(QMediaMetaData::Duration, m_duration / qint64(1000)); + + if (!m_cachedThumbnail.has_value()) + m_cachedThumbnail = getAttachedPicture(m_context.get()); + + if (!m_cachedThumbnail->isNull()) + m_metaData.insert(QMediaMetaData::ThumbnailImage, m_cachedThumbnail.value()); + + for (auto trackType : + { QPlatformMediaPlayer::AudioStream, QPlatformMediaPlayer::VideoStream }) { + const auto streamIndex = m_currentAVStreamIndex[trackType]; + if (streamIndex >= 0) + insertMediaData(m_metaData, trackType, m_context->streams[streamIndex]); + } +} + +bool MediaDataHolder::setActiveTrack(QPlatformMediaPlayer::TrackType type, int streamNumber) +{ + if (!m_context) + return false; + + if (streamNumber < 0 || streamNumber >= m_streamMap[type].size()) + streamNumber = -1; + if (m_requestedStreams[type] == streamNumber) + return false; + m_requestedStreams[type] = streamNumber; + const int avStreamIndex = m_streamMap[type].value(streamNumber).avStreamIndex; + + const int oldIndex = m_currentAVStreamIndex[type]; + qCDebug(qLcMediaDataHolder) << ">>>>> change track" << type << "from" << oldIndex << "to" + << avStreamIndex; + + // TODO: maybe add additional verifications + m_currentAVStreamIndex[type] = avStreamIndex; + + updateMetaData(); + + return true; +} + +int MediaDataHolder::activeTrack(QPlatformMediaPlayer::TrackType type) const +{ + return type < QPlatformMediaPlayer::NTrackTypes ? m_requestedStreams[type] : -1; +} + +const QList<MediaDataHolder::StreamInfo> &MediaDataHolder::streamInfo( + QPlatformMediaPlayer::TrackType trackType) const +{ + Q_ASSERT(trackType < QPlatformMediaPlayer::NTrackTypes); + + return m_streamMap[trackType]; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder_p.h new file mode 100644 index 000000000..a55b0766a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegmediadataholder_p.h @@ -0,0 +1,107 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGMEDIADATAHOLDER_P_H +#define QFFMPEGMEDIADATAHOLDER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qmediametadata.h" +#include "private/qplatformmediaplayer_p.h" +#include "qffmpeg_p.h" +#include "qvideoframe.h" +#include <private/qmultimediautils_p.h> + +#include <array> +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct ICancelToken +{ + virtual ~ICancelToken() = default; + virtual bool isCancelled() const = 0; +}; + +using AVFormatContextUPtr = std::unique_ptr<AVFormatContext, AVDeleter<decltype(&avformat_close_input), &avformat_close_input>>; + +class MediaDataHolder +{ +public: + struct StreamInfo + { + int avStreamIndex = -1; + bool isDefault = false; + QMediaMetaData metaData; + }; + + struct ContextError + { + int code = 0; + QString description; + }; + + using StreamsMap = std::array<QList<StreamInfo>, QPlatformMediaPlayer::NTrackTypes>; + using StreamIndexes = std::array<int, QPlatformMediaPlayer::NTrackTypes>; + + MediaDataHolder() = default; + MediaDataHolder(AVFormatContextUPtr context, const std::shared_ptr<ICancelToken> &cancelToken); + + static QPlatformMediaPlayer::TrackType trackTypeFromMediaType(int mediaType); + + int activeTrack(QPlatformMediaPlayer::TrackType type) const; + + const QList<StreamInfo> &streamInfo(QPlatformMediaPlayer::TrackType trackType) const; + + qint64 duration() const { return m_duration; } + + const QMediaMetaData &metaData() const { return m_metaData; } + + bool isSeekable() const { return m_isSeekable; } + + QtVideo::Rotation rotation() const; + + AVFormatContext *avContext(); + + int currentStreamIndex(QPlatformMediaPlayer::TrackType trackType) const; + + using Maybe = QMaybe<QSharedPointer<MediaDataHolder>, ContextError>; + static Maybe create(const QUrl &url, QIODevice *stream, + const std::shared_ptr<ICancelToken> &cancelToken); + + bool setActiveTrack(QPlatformMediaPlayer::TrackType type, int streamNumber); + +private: + void updateMetaData(); + + std::shared_ptr<ICancelToken> m_cancelToken; // NOTE: Cancel token may be accessed by + // AVFormatContext during destruction and + // must outlive the context object + AVFormatContextUPtr m_context; + + bool m_isSeekable = false; + + StreamIndexes m_currentAVStreamIndex = { -1, -1, -1 }; + StreamsMap m_streamMap; + StreamIndexes m_requestedStreams = { -1, -1, -1 }; + qint64 m_duration = 0; + QMediaMetaData m_metaData; + std::optional<QImage> m_cachedThumbnail; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGMEDIADATAHOLDER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h new file mode 100644 index 000000000..5e15bf012 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpacket_p.h @@ -0,0 +1,61 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGPACKET_P_H +#define QFFMPEGPACKET_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeg_p.h" +#include "QtCore/qsharedpointer.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct Packet +{ + struct Data + { + Data(const LoopOffset &offset, AVPacketUPtr p, quint64 sourceId) + : loopOffset(offset), packet(std::move(p)), sourceId(sourceId) + { + } + + QAtomicInt ref; + LoopOffset loopOffset; + AVPacketUPtr packet; + quint64 sourceId; + }; + Packet() = default; + Packet(const LoopOffset &offset, AVPacketUPtr p, quint64 sourceId) + : d(new Data(offset, std::move(p), sourceId)) + { + } + + bool isValid() const { return !!d; } + AVPacket *avPacket() const { return d->packet.get(); } + const LoopOffset &loopOffset() const { return d->loopOffset; } + quint64 sourceId() const { return d->sourceId; } + +private: + QExplicitlySharedDataPointer<Data> d; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +Q_DECLARE_METATYPE(QFFmpeg::Packet) + +#endif // QFFMPEGPACKET_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackenginedefs_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackenginedefs_p.h new file mode 100644 index 000000000..18254ef64 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackenginedefs_p.h @@ -0,0 +1,46 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGPLAYBACKENGINEDEFS_P_H +#define QFFMPEGPLAYBACKENGINEDEFS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// +#include "qobject.h" +#include "qpointer.h" + +#include <memory> +#include <array> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { +class PlaybackEngine; +} + +namespace QFFmpeg { + +using StreamIndexes = std::array<int, 3>; + +class PlaybackEngineObjectsController; +class PlaybackEngineObject; +class Demuxer; +class StreamDecoder; +class Renderer; +class SubtitleRenderer; +class AudioRenderer; +class VideoRenderer; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGPLAYBACKENGINEDEFS_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject.cpp new file mode 100644 index 000000000..2d23802de --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject.cpp @@ -0,0 +1,109 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegplaybackengineobject_p.h" + +#include "qtimer.h" +#include "qdebug.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +static QAtomicInteger<PlaybackEngineObject::Id> PersistentId = 0; + +PlaybackEngineObject::PlaybackEngineObject() : m_id(PersistentId.fetchAndAddRelaxed(1)) { } + +PlaybackEngineObject::~PlaybackEngineObject() +{ + if (thread() != QThread::currentThread()) + qWarning() << "The playback engine object is being removed in an unexpected thread"; +} + +bool PlaybackEngineObject::isPaused() const +{ + return m_paused; +} + +void PlaybackEngineObject::setAtEnd(bool isAtEnd) +{ + if (m_atEnd.testAndSetRelease(!isAtEnd, isAtEnd) && isAtEnd) + emit atEnd(); +} + +bool PlaybackEngineObject::isAtEnd() const +{ + return m_atEnd; +} + +PlaybackEngineObject::Id PlaybackEngineObject::id() const +{ + return m_id; +} + +void PlaybackEngineObject::setPaused(bool isPaused) +{ + if (m_paused.testAndSetRelease(!isPaused, isPaused)) + QMetaObject::invokeMethod(this, &PlaybackEngineObject::onPauseChanged); +} + +void PlaybackEngineObject::kill() +{ + m_deleting.storeRelease(true); + + disconnect(); + deleteLater(); +} + +bool PlaybackEngineObject::canDoNextStep() const +{ + return !m_paused; +} + +QTimer &PlaybackEngineObject::timer() +{ + if (!m_timer) { + m_timer = std::make_unique<QTimer>(); + m_timer->setTimerType(Qt::PreciseTimer); + m_timer->setSingleShot(true); + connect(m_timer.get(), &QTimer::timeout, this, &PlaybackEngineObject::onTimeout); + } + + return *m_timer; +} + +void PlaybackEngineObject::onTimeout() +{ + if (!m_deleting && canDoNextStep()) + doNextStep(); +} + +int PlaybackEngineObject::timerInterval() const +{ + return 0; +} + +void PlaybackEngineObject::onPauseChanged() +{ + scheduleNextStep(); +} + +void PlaybackEngineObject::scheduleNextStep(bool allowDoImmediatelly) +{ + if (!m_deleting && canDoNextStep()) { + const auto interval = timerInterval(); + if (interval == 0 && allowDoImmediatelly) { + timer().stop(); + doNextStep(); + } else { + timer().start(interval); + } + } else { + timer().stop(); + } +} +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegplaybackengineobject_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject_p.h new file mode 100644 index 000000000..02943a55b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegplaybackengineobject_p.h @@ -0,0 +1,84 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGPLAYBACKENGINEOBJECT_P_H +#define QFFMPEGPLAYBACKENGINEOBJECT_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegplaybackenginedefs_p.h" +#include "qthread.h" +#include "qatomic.h" + +QT_BEGIN_NAMESPACE + +class QTimer; + +namespace QFFmpeg { + +class PlaybackEngineObject : public QObject +{ + Q_OBJECT +public: + using TimePoint = std::chrono::steady_clock::time_point; + using TimePointOpt = std::optional<TimePoint>; + using Id = quint64; + + PlaybackEngineObject(); + + ~PlaybackEngineObject(); + + bool isPaused() const; + + bool isAtEnd() const; + + void kill(); + + void setPaused(bool isPaused); + + Id id() const; + +signals: + void atEnd(); + + void error(int code, const QString &errorString); + +protected: + QTimer &timer(); + + void scheduleNextStep(bool allowDoImmediatelly = true); + + virtual void onPauseChanged(); + + virtual bool canDoNextStep() const; + + virtual int timerInterval() const; + + void setAtEnd(bool isAtEnd); + + virtual void doNextStep() { } + +private slots: + void onTimeout(); + +private: + std::unique_ptr<QTimer> m_timer; + + QAtomicInteger<bool> m_paused = true; + QAtomicInteger<bool> m_atEnd = false; + QAtomicInteger<bool> m_deleting = false; + const Id m_id; +}; +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGPLAYBACKENGINEOBJECT_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h new file mode 100644 index 000000000..a30fdc119 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegpositionwithoffset_p.h @@ -0,0 +1,40 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef qffmpegpositionwithoffset_p_H +#define qffmpegpositionwithoffset_p_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtypes.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +struct LoopOffset +{ + qint64 pos = 0; + int index = 0; +}; + +struct PositionWithOffset +{ + qint64 pos = 0; + LoopOffset offset; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // qffmpegpositionwithoffset_p_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp new file mode 100644 index 000000000..5382ff023 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer.cpp @@ -0,0 +1,216 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegrenderer_p.h" +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +Q_STATIC_LOGGING_CATEGORY(qLcRenderer, "qt.multimedia.ffmpeg.renderer"); + +Renderer::Renderer(const TimeController &tc, const std::chrono::microseconds &seekPosTimeOffset) + : m_timeController(tc), + m_lastFrameEnd(tc.currentPosition()), + m_lastPosition(m_lastFrameEnd), + m_seekPos(tc.currentPosition(-seekPosTimeOffset)) +{ +} + +void Renderer::syncSoft(TimePoint tp, qint64 trackTime) +{ + QMetaObject::invokeMethod(this, [this, tp, trackTime]() { + m_timeController.syncSoft(tp, trackTime); + scheduleNextStep(true); + }); +} + +qint64 Renderer::seekPosition() const +{ + return m_seekPos; +} + +qint64 Renderer::lastPosition() const +{ + return m_lastPosition; +} + +void Renderer::setPlaybackRate(float rate) +{ + QMetaObject::invokeMethod(this, [this, rate]() { + m_timeController.setPlaybackRate(rate); + onPlaybackRateChanged(); + scheduleNextStep(); + }); +} + +void Renderer::doForceStep() +{ + if (m_isStepForced.testAndSetOrdered(false, true)) + QMetaObject::invokeMethod(this, [this]() { + // maybe set m_forceStepMaxPos + + if (isAtEnd()) { + setForceStepDone(); + } + else { + m_explicitNextFrameTime = Clock::now(); + scheduleNextStep(); + } + }); +} + +bool Renderer::isStepForced() const +{ + return m_isStepForced; +} + +void Renderer::setInitialPosition(TimePoint tp, qint64 trackPos) +{ + QMetaObject::invokeMethod(this, [this, tp, trackPos]() { + Q_ASSERT(m_loopIndex == 0); + Q_ASSERT(m_frames.empty()); + + m_loopIndex = 0; + m_lastPosition.storeRelease(trackPos); + m_seekPos.storeRelease(trackPos); + + m_timeController.sync(tp, trackPos); + }); +} + +void Renderer::onFinalFrameReceived() +{ + render({}); +} + +void Renderer::render(Frame frame) +{ + const auto isFrameOutdated = frame.isValid() && frame.absoluteEnd() < seekPosition(); + + if (isFrameOutdated) { + qCDebug(qLcRenderer) << "frame outdated! absEnd:" << frame.absoluteEnd() << "absPts" + << frame.absolutePts() << "seekPos:" << seekPosition(); + emit frameProcessed(frame); + return; + } + + m_frames.enqueue(frame); + + if (m_frames.size() == 1) + scheduleNextStep(); +} + +void Renderer::onPauseChanged() +{ + m_timeController.setPaused(isPaused()); + PlaybackEngineObject::onPauseChanged(); +} + +bool Renderer::canDoNextStep() const +{ + return !m_frames.empty() && (m_isStepForced || PlaybackEngineObject::canDoNextStep()); +} + +float Renderer::playbackRate() const +{ + return m_timeController.playbackRate(); +} + +int Renderer::timerInterval() const +{ + if (m_frames.empty()) + return 0; + + auto calculateInterval = [](const TimePoint &nextTime) { + using namespace std::chrono; + + const auto delay = nextTime - Clock::now(); + return std::max(0, static_cast<int>(duration_cast<milliseconds>(delay).count())); + }; + + if (m_explicitNextFrameTime) + return calculateInterval(*m_explicitNextFrameTime); + + if (m_frames.front().isValid()) + return calculateInterval(m_timeController.timeFromPosition(m_frames.front().absolutePts())); + + if (m_lastFrameEnd > 0) + return calculateInterval(m_timeController.timeFromPosition(m_lastFrameEnd)); + + return 0; +} + +bool Renderer::setForceStepDone() +{ + if (!m_isStepForced.testAndSetOrdered(true, false)) + return false; + + m_explicitNextFrameTime.reset(); + emit forceStepDone(); + return true; +} + +void Renderer::doNextStep() +{ + auto frame = m_frames.front(); + + if (setForceStepDone()) { + // if (frame.isValid() && frame.pts() > m_forceStepMaxPos) { + // scheduleNextStep(false); + // return; + // } + } + + const auto result = renderInternal(frame); + + if (result.done) { + m_explicitNextFrameTime.reset(); + m_frames.dequeue(); + + if (frame.isValid()) { + m_lastPosition.storeRelease(std::max(frame.absolutePts(), lastPosition())); + + // TODO: get rid of m_lastFrameEnd or m_seekPos + m_lastFrameEnd = frame.absoluteEnd(); + m_seekPos.storeRelaxed(m_lastFrameEnd); + + const auto loopIndex = frame.loopOffset().index; + if (m_loopIndex < loopIndex) { + m_loopIndex = loopIndex; + emit loopChanged(id(), frame.loopOffset().pos, m_loopIndex); + } + + emit frameProcessed(frame); + } else { + m_lastPosition.storeRelease(std::max(m_lastFrameEnd, lastPosition())); + } + } else { + m_explicitNextFrameTime = Clock::now() + result.recheckInterval; + } + + setAtEnd(result.done && !frame.isValid()); + + scheduleNextStep(false); +} + +std::chrono::microseconds Renderer::frameDelay(const Frame &frame, TimePoint timePoint) const +{ + return std::chrono::duration_cast<std::chrono::microseconds>( + timePoint - m_timeController.timeFromPosition(frame.absolutePts())); +} + +void Renderer::changeRendererTime(std::chrono::microseconds offset) +{ + const auto now = Clock::now(); + const auto pos = m_timeController.positionFromTime(now); + m_timeController.sync(now + offset, pos); + emit synchronized(id(), now + offset, pos); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegrenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h new file mode 100644 index 000000000..99c5ef1b1 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegrenderer_p.h @@ -0,0 +1,125 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGRENDERER_P_H +#define QFFMPEGRENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegplaybackengineobject_p.h" +#include "playbackengine/qffmpegtimecontroller_p.h" +#include "playbackengine/qffmpegframe_p.h" + +#include <QtCore/qpointer.h> + +#include <chrono> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class Renderer : public PlaybackEngineObject +{ + Q_OBJECT +public: + using TimePoint = TimeController::TimePoint; + using Clock = TimeController::Clock; + Renderer(const TimeController &tc, const std::chrono::microseconds &seekPosTimeOffset = {}); + + void syncSoft(TimePoint tp, qint64 trackPos); + + qint64 seekPosition() const; + + qint64 lastPosition() const; + + void setPlaybackRate(float rate); + + void doForceStep(); + + bool isStepForced() const; + +public slots: + void setInitialPosition(TimePoint tp, qint64 trackPos); + + void onFinalFrameReceived(); + + void render(Frame); + +signals: + void frameProcessed(Frame); + + void synchronized(Id id, TimePoint tp, qint64 pos); + + void forceStepDone(); + + void loopChanged(Id id, qint64 offset, int index); + +protected: + bool setForceStepDone(); + + void onPauseChanged() override; + + bool canDoNextStep() const override; + + int timerInterval() const override; + + virtual void onPlaybackRateChanged() { } + + struct RenderingResult + { + bool done = true; + std::chrono::microseconds recheckInterval = std::chrono::microseconds(0); + }; + + virtual RenderingResult renderInternal(Frame frame) = 0; + + float playbackRate() const; + + std::chrono::microseconds frameDelay(const Frame &frame, + TimePoint timePoint = Clock::now()) const; + + void changeRendererTime(std::chrono::microseconds offset); + + template<typename Output, typename ChangeHandler> + void setOutputInternal(QPointer<Output> &actual, Output *desired, ChangeHandler &&changeHandler) + { + const auto connectionType = thread() == QThread::currentThread() + ? Qt::AutoConnection + : Qt::BlockingQueuedConnection; + auto doer = [desired, changeHandler, &actual]() { + const auto prev = std::exchange(actual, desired); + if (prev != desired) + changeHandler(prev); + }; + QMetaObject::invokeMethod(this, doer, connectionType); + } + +private: + void doNextStep() override; + +private: + TimeController m_timeController; + qint64 m_lastFrameEnd = 0; + QAtomicInteger<qint64> m_lastPosition = 0; + QAtomicInteger<qint64> m_seekPos = 0; + + int m_loopIndex = 0; + QQueue<Frame> m_frames; + + QAtomicInteger<bool> m_isStepForced = false; + std::optional<TimePoint> m_explicitNextFrameTime; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGRENDERER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp new file mode 100644 index 000000000..2d9d63b90 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder.cpp @@ -0,0 +1,245 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegstreamdecoder_p.h" +#include "playbackengine/qffmpegmediadataholder_p.h" +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcStreamDecoder, "qt.multimedia.ffmpeg.streamdecoder"); + +namespace QFFmpeg { + +StreamDecoder::StreamDecoder(const Codec &codec, qint64 absSeekPos) + : m_codec(codec), + m_absSeekPos(absSeekPos), + m_trackType(MediaDataHolder::trackTypeFromMediaType(codec.context()->codec_type)) +{ + qCDebug(qLcStreamDecoder) << "Create stream decoder, trackType" << m_trackType + << "absSeekPos:" << absSeekPos; + Q_ASSERT(m_trackType != QPlatformMediaPlayer::NTrackTypes); +} + +StreamDecoder::~StreamDecoder() +{ + avcodec_flush_buffers(m_codec.context()); +} + +void StreamDecoder::onFinalPacketReceived() +{ + decode({}); +} + +void StreamDecoder::setInitialPosition(TimePoint, qint64 trackPos) +{ + m_absSeekPos = trackPos; +} + +void StreamDecoder::decode(Packet packet) +{ + m_packets.enqueue(packet); + + scheduleNextStep(); +} + +void StreamDecoder::doNextStep() +{ + auto packet = m_packets.dequeue(); + + auto decodePacket = [this](Packet packet) { + if (trackType() == QPlatformMediaPlayer::SubtitleStream) + decodeSubtitle(packet); + else + decodeMedia(packet); + }; + + if (packet.isValid() && packet.loopOffset().index != m_offset.index) { + decodePacket({}); + + qCDebug(qLcStreamDecoder) << "flush buffers due to new loop:" << packet.loopOffset().index; + + avcodec_flush_buffers(m_codec.context()); + m_offset = packet.loopOffset(); + } + + decodePacket(packet); + + setAtEnd(!packet.isValid()); + + if (packet.isValid()) + emit packetProcessed(packet); + + scheduleNextStep(false); +} + +QPlatformMediaPlayer::TrackType StreamDecoder::trackType() const +{ + return m_trackType; +} + +qint32 StreamDecoder::maxQueueSize(QPlatformMediaPlayer::TrackType type) +{ + switch (type) { + + case QPlatformMediaPlayer::VideoStream: + return 3; + case QPlatformMediaPlayer::AudioStream: + return 9; + case QPlatformMediaPlayer::SubtitleStream: + return 6; /*main packet and closing packet*/ + default: + Q_UNREACHABLE_RETURN(-1); + } +} + +void StreamDecoder::onFrameProcessed(Frame frame) +{ + if (frame.sourceId() != id()) + return; + + --m_pendingFramesCount; + Q_ASSERT(m_pendingFramesCount >= 0); + + scheduleNextStep(); +} + +bool StreamDecoder::canDoNextStep() const +{ + const qint32 maxCount = maxQueueSize(m_trackType); + + return !m_packets.empty() && m_pendingFramesCount < maxCount + && PlaybackEngineObject::canDoNextStep(); +} + +void StreamDecoder::onFrameFound(Frame frame) +{ + if (frame.isValid() && frame.absoluteEnd() < m_absSeekPos) + return; + + Q_ASSERT(m_pendingFramesCount >= 0); + ++m_pendingFramesCount; + emit requestHandleFrame(frame); +} + +void StreamDecoder::decodeMedia(Packet packet) +{ + auto sendPacketResult = sendAVPacket(packet); + + if (sendPacketResult == AVERROR(EAGAIN)) { + // Doc says: + // AVERROR(EAGAIN): input is not accepted in the current state - user + // must read output with avcodec_receive_frame() (once + // all output is read, the packet should be resent, and + // the call will not fail with EAGAIN). + receiveAVFrames(); + sendPacketResult = sendAVPacket(packet); + + if (sendPacketResult != AVERROR(EAGAIN)) + qWarning() << "Unexpected FFmpeg behavior"; + } + + if (sendPacketResult == 0) + receiveAVFrames(); +} + +int StreamDecoder::sendAVPacket(Packet packet) +{ + return avcodec_send_packet(m_codec.context(), packet.isValid() ? packet.avPacket() : nullptr); +} + +void StreamDecoder::receiveAVFrames() +{ + while (true) { + auto avFrame = makeAVFrame(); + + const auto receiveFrameResult = avcodec_receive_frame(m_codec.context(), avFrame.get()); + + if (receiveFrameResult == AVERROR_EOF || receiveFrameResult == AVERROR(EAGAIN)) + break; + + if (receiveFrameResult < 0) { + emit error(QMediaPlayer::FormatError, err2str(receiveFrameResult)); + break; + } + + + // Avoid starvation on FFmpeg decoders with fixed size frame pool + if (m_trackType == QPlatformMediaPlayer::VideoStream) + avFrame = copyFromHwPool(std::move(avFrame)); + + onFrameFound({ m_offset, std::move(avFrame), m_codec, 0, id() }); + } +} + +void StreamDecoder::decodeSubtitle(Packet packet) +{ + if (!packet.isValid()) + return; + // qCDebug(qLcDecoder) << " decoding subtitle" << "has delay:" << + // (codec->codec->capabilities & AV_CODEC_CAP_DELAY); + AVSubtitle subtitle; + memset(&subtitle, 0, sizeof(subtitle)); + int gotSubtitle = 0; + + const int res = + avcodec_decode_subtitle2(m_codec.context(), &subtitle, &gotSubtitle, packet.avPacket()); + // qCDebug(qLcDecoder) << " subtitle got:" << res << gotSubtitle << subtitle.format << + // Qt::hex << (quint64)subtitle.pts; + if (res < 0 || !gotSubtitle) + return; + + // apparently the timestamps in the AVSubtitle structure are not always filled in + // if they are missing, use the packets pts and duration values instead + qint64 start, end; + if (subtitle.pts == AV_NOPTS_VALUE) { + start = m_codec.toUs(packet.avPacket()->pts); + end = start + m_codec.toUs(packet.avPacket()->duration); + } else { + auto pts = timeStampUs(subtitle.pts, AVRational{ 1, AV_TIME_BASE }); + start = *pts + qint64(subtitle.start_display_time) * 1000; + end = *pts + qint64(subtitle.end_display_time) * 1000; + } + + if (end <= start) { + qWarning() << "Invalid subtitle time"; + return; + } + // qCDebug(qLcDecoder) << " got subtitle (" << start << "--" << end << "):"; + QString text; + for (uint i = 0; i < subtitle.num_rects; ++i) { + const auto *r = subtitle.rects[i]; + // qCDebug(qLcDecoder) << " subtitletext:" << r->text << "/" << r->ass; + if (i) + text += QLatin1Char('\n'); + if (r->text) + text += QString::fromUtf8(r->text); + else { + const char *ass = r->ass; + int nCommas = 0; + while (*ass) { + if (nCommas == 8) + break; + if (*ass == ',') + ++nCommas; + ++ass; + } + text += QString::fromUtf8(ass); + } + } + text.replace(QLatin1String("\\N"), QLatin1String("\n")); + text.replace(QLatin1String("\\n"), QLatin1String("\n")); + text.replace(QLatin1String("\r\n"), QLatin1String("\n")); + if (text.endsWith(QLatin1Char('\n'))) + text.chop(1); + + onFrameFound({ m_offset, text, start, end - start, id() }); + + // TODO: maybe optimize + onFrameFound({ m_offset, QString(), end, 0, id() }); +} +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegstreamdecoder_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h new file mode 100644 index 000000000..1acc07983 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegstreamdecoder_p.h @@ -0,0 +1,87 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGSTREAMDECODER_P_H +#define QFFMPEGSTREAMDECODER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// +#include "playbackengine/qffmpegplaybackengineobject_p.h" +#include "playbackengine/qffmpegframe_p.h" +#include "playbackengine/qffmpegpacket_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" +#include "private/qplatformmediaplayer_p.h" + +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class StreamDecoder : public PlaybackEngineObject +{ + Q_OBJECT +public: + StreamDecoder(const Codec &codec, qint64 absSeekPos); + + ~StreamDecoder(); + + QPlatformMediaPlayer::TrackType trackType() const; + + // Maximum number of frames that we are allowed to keep in render queue + static qint32 maxQueueSize(QPlatformMediaPlayer::TrackType type); + +public slots: + void setInitialPosition(TimePoint tp, qint64 trackPos); + + void decode(Packet); + + void onFinalPacketReceived(); + + void onFrameProcessed(Frame frame); + +signals: + void requestHandleFrame(Frame frame); + + void packetProcessed(Packet); + +protected: + bool canDoNextStep() const override; + + void doNextStep() override; + +private: + void decodeMedia(Packet); + + void decodeSubtitle(Packet); + + void onFrameFound(Frame frame); + + int sendAVPacket(Packet); + + void receiveAVFrames(); + +private: + Codec m_codec; + qint64 m_absSeekPos = 0; + const QPlatformMediaPlayer::TrackType m_trackType; + + qint32 m_pendingFramesCount = 0; + + LoopOffset m_offset; + + QQueue<Packet> m_packets; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGSTREAMDECODER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer.cpp new file mode 100644 index 000000000..789c9b53b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer.cpp @@ -0,0 +1,44 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegsubtitlerenderer_p.h" + +#include "qvideosink.h" +#include "qdebug.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +SubtitleRenderer::SubtitleRenderer(const TimeController &tc, QVideoSink *sink) + : Renderer(tc), m_sink(sink) +{ +} + +void SubtitleRenderer::setOutput(QVideoSink *sink, bool cleanPrevSink) +{ + setOutputInternal(m_sink, sink, [cleanPrevSink](QVideoSink *prev) { + if (prev && cleanPrevSink) + prev->setSubtitleText({}); + }); +} + +SubtitleRenderer::~SubtitleRenderer() +{ + if (m_sink) + m_sink->setSubtitleText({}); +} + +Renderer::RenderingResult SubtitleRenderer::renderInternal(Frame frame) +{ + if (m_sink) + m_sink->setSubtitleText(frame.isValid() ? frame.text() : QString()); + + return {}; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegsubtitlerenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer_p.h new file mode 100644 index 000000000..805212e83 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegsubtitlerenderer_p.h @@ -0,0 +1,48 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGSUBTITLERENDERER_P_H +#define QFFMPEGSUBTITLERENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegrenderer_p.h" + +#include <QtCore/qpointer.h> + +QT_BEGIN_NAMESPACE + +class QVideoSink; + +namespace QFFmpeg { + +class SubtitleRenderer : public Renderer +{ + Q_OBJECT +public: + SubtitleRenderer(const TimeController &tc, QVideoSink *sink); + + ~SubtitleRenderer() override; + + void setOutput(QVideoSink *sink, bool cleanPrevSink = false); + +protected: + RenderingResult renderInternal(Frame frame) override; + +private: + QPointer<QVideoSink> m_sink; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGSUBTITLERENDERER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller.cpp new file mode 100644 index 000000000..8352384b4 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller.cpp @@ -0,0 +1,165 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegtimecontroller_p.h" + +#include "qglobal.h" +#include "qdebug.h" + +#include <algorithm> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +TimeController::TimeController() +{ + sync(); +} + +TimeController::PlaybackRate TimeController::playbackRate() const +{ + return m_playbackRate; +} + +void TimeController::setPlaybackRate(PlaybackRate playbackRate) +{ + if (playbackRate == m_playbackRate) + return; + + Q_ASSERT(playbackRate > 0.f); + + scrollTimeTillNow(); + m_playbackRate = playbackRate; + + if (m_softSyncData) + m_softSyncData = makeSoftSyncData(m_timePoint, m_position, m_softSyncData->dstTimePoint); +} + +void TimeController::sync(qint64 trackPos) +{ + sync(Clock::now(), trackPos); +} + +void TimeController::sync(const TimePoint &tp, qint64 pos) +{ + m_softSyncData.reset(); + m_position = TrackTime(pos); + m_timePoint = tp; +} + +void TimeController::syncSoft(const TimePoint &tp, qint64 pos, const Clock::duration &fixingTime) +{ + const auto srcTime = Clock::now(); + const auto srcPos = positionFromTime(srcTime, true); + const auto dstTime = srcTime + fixingTime; + + m_position = TrackTime(pos); + m_timePoint = tp; + + m_softSyncData = makeSoftSyncData(srcTime, TrackTime(srcPos), dstTime); +} + +qint64 TimeController::currentPosition(const Clock::duration &offset) const +{ + return positionFromTime(Clock::now() + offset); +} + +void TimeController::setPaused(bool paused) +{ + if (m_paused == paused) + return; + + scrollTimeTillNow(); + m_paused = paused; +} + +qint64 TimeController::positionFromTime(TimePoint tp, bool ignorePause) const +{ + tp = m_paused && !ignorePause ? m_timePoint : tp; + + if (m_softSyncData && tp < m_softSyncData->dstTimePoint) { + const PlaybackRate rate = + tp > m_softSyncData->srcTimePoint ? m_softSyncData->internalRate : m_playbackRate; + + return (m_softSyncData->srcPosition + + toTrackTime((tp - m_softSyncData->srcTimePoint) * rate)) + .count(); + } + + return positionFromTimeInternal(tp).count(); +} + +TimeController::TimePoint TimeController::timeFromPosition(qint64 pos, bool ignorePause) const +{ + auto position = m_paused && !ignorePause ? m_position : TrackTime(pos); + + if (m_softSyncData && position < m_softSyncData->dstPosition) { + const auto rate = position > m_softSyncData->srcPosition ? m_softSyncData->internalRate + : m_playbackRate; + return m_softSyncData->srcTimePoint + + toClockTime((position - m_softSyncData->srcPosition) / rate); + } + + return timeFromPositionInternal(position); +} + +TimeController::SoftSyncData TimeController::makeSoftSyncData(const TimePoint &srcTp, + const TrackTime &srcPos, + const TimePoint &dstTp) const +{ + SoftSyncData result; + result.srcTimePoint = srcTp; + result.srcPosition = srcPos; + result.dstTimePoint = dstTp; + result.srcPosOffest = srcPos - positionFromTimeInternal(srcTp); + result.dstPosition = positionFromTimeInternal(dstTp); + result.internalRate = + static_cast<PlaybackRate>(toClockTime(TrackTime(result.dstPosition - srcPos)).count()) + / (dstTp - srcTp).count(); + + return result; +} + +TimeController::TrackTime TimeController::positionFromTimeInternal(const TimePoint &tp) const +{ + return m_position + toTrackTime((tp - m_timePoint) * m_playbackRate); +} + +TimeController::TimePoint TimeController::timeFromPositionInternal(const TrackTime &pos) const +{ + return m_timePoint + toClockTime(TrackTime(pos - m_position) / m_playbackRate); +} + +void TimeController::scrollTimeTillNow() +{ + const auto now = Clock::now(); + if (!m_paused) { + m_position = positionFromTimeInternal(now); + + // let's forget outdated syncronizations + if (m_softSyncData && m_softSyncData->dstTimePoint <= now) + m_softSyncData.reset(); + } else if (m_softSyncData) { + m_softSyncData->dstTimePoint += now - m_timePoint; + m_softSyncData->srcTimePoint += now - m_timePoint; + } + + m_timePoint = now; +} + +template<typename T> +TimeController::Clock::duration TimeController::toClockTime(const T &t) +{ + return std::chrono::duration_cast<Clock::duration>(t); +} + +template<typename T> +TimeController::TrackTime TimeController::toTrackTime(const T &t) +{ + return std::chrono::duration_cast<TrackTime>(t); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller_p.h new file mode 100644 index 000000000..93ced7e64 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegtimecontroller_p.h @@ -0,0 +1,94 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGTIMECONTROLLER_P_H +#define QFFMPEGTIMECONTROLLER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qglobal.h" + +#include <chrono> +#include <optional> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class TimeController +{ + using TrackTime = std::chrono::microseconds; + +public: + using Clock = std::chrono::steady_clock; + using TimePoint = Clock::time_point; + using PlaybackRate = float; + + TimeController(); + + PlaybackRate playbackRate() const; + + void setPlaybackRate(PlaybackRate playbackRate); + + void sync(qint64 trackPos = 0); + + void sync(const TimePoint &tp, qint64 pos); + + void syncSoft(const TimePoint &tp, qint64 pos, + const Clock::duration &fixingTime = std::chrono::seconds(4)); + + qint64 currentPosition(const Clock::duration &offset = Clock::duration{ 0 }) const; + + void setPaused(bool paused); + + qint64 positionFromTime(TimePoint tp, bool ignorePause = false) const; + + TimePoint timeFromPosition(qint64 pos, bool ignorePause = false) const; + +private: + struct SoftSyncData + { + TimePoint srcTimePoint; + TrackTime srcPosition; + TimePoint dstTimePoint; + TrackTime srcPosOffest; + TrackTime dstPosition; + PlaybackRate internalRate = 1; + }; + + SoftSyncData makeSoftSyncData(const TimePoint &srcTp, const TrackTime &srcPos, + const TimePoint &dstTp) const; + + TrackTime positionFromTimeInternal(const TimePoint &tp) const; + + TimePoint timeFromPositionInternal(const TrackTime &pos) const; + + void scrollTimeTillNow(); + + template<typename T> + static Clock::duration toClockTime(const T &t); + + template<typename T> + static TrackTime toTrackTime(const T &t); + +private: + bool m_paused = true; + PlaybackRate m_playbackRate = 1; + TrackTime m_position; + TimePoint m_timePoint = {}; + std::optional<SoftSyncData> m_softSyncData; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGTIMECONTROLLER_P_H diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp new file mode 100644 index 000000000..dceb00f83 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer.cpp @@ -0,0 +1,79 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "playbackengine/qffmpegvideorenderer_p.h" +#include "qffmpegvideobuffer_p.h" +#include "qvideosink.h" +#include "private/qvideoframe_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +VideoRenderer::VideoRenderer(const TimeController &tc, QVideoSink *sink, QtVideo::Rotation rotation) + : Renderer(tc), m_sink(sink), m_rotation(rotation) +{ +} + +void VideoRenderer::setOutput(QVideoSink *sink, bool cleanPrevSink) +{ + setOutputInternal(m_sink, sink, [cleanPrevSink](QVideoSink *prev) { + if (prev && cleanPrevSink) + prev->setVideoFrame({}); + }); +} + +VideoRenderer::RenderingResult VideoRenderer::renderInternal(Frame frame) +{ + if (!m_sink) + return {}; + + if (!frame.isValid()) { + m_sink->setVideoFrame({}); + return {}; + } + + // qCDebug(qLcVideoRenderer) << "RHI:" << accel.isNull() << accel.rhi() << sink->rhi(); + + const auto codec = frame.codec(); + Q_ASSERT(codec); + +#ifdef Q_OS_ANDROID + // QTBUG-108446 + // In general case, just creation of frames context is not correct since + // frames may require additional specific data for hw contexts, so + // just setting of hw_frames_ctx is not enough. + // TODO: investigate the case in order to remove or fix the code. + if (codec->hwAccel() && !frame.avFrame()->hw_frames_ctx) { + HWAccel *hwaccel = codec->hwAccel(); + AVFrame *avframe = frame.avFrame(); + if (!hwaccel->hwFramesContext()) + hwaccel->createFramesContext(AVPixelFormat(avframe->format), + { avframe->width, avframe->height }); + + if (hwaccel->hwFramesContext()) + avframe->hw_frames_ctx = av_buffer_ref(hwaccel->hwFramesContextAsBuffer()); + } +#endif + + const auto pixelAspectRatio = codec->pixelAspectRatio(frame.avFrame()); + auto buffer = std::make_unique<QFFmpegVideoBuffer>(frame.takeAVFrame(), pixelAspectRatio); + QVideoFrameFormat format(buffer->size(), buffer->pixelFormat()); + format.setColorSpace(buffer->colorSpace()); + format.setColorTransfer(buffer->colorTransfer()); + format.setColorRange(buffer->colorRange()); + format.setMaxLuminance(buffer->maxNits()); + format.setRotation(m_rotation); + QVideoFrame videoFrame = QVideoFramePrivate::createFrame(std::move(buffer), format); + videoFrame.setStartTime(frame.pts()); + videoFrame.setEndTime(frame.end()); + m_sink->setVideoFrame(videoFrame); + + return {}; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegvideorenderer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer_p.h b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer_p.h new file mode 100644 index 000000000..4866420e8 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/playbackengine/qffmpegvideorenderer_p.h @@ -0,0 +1,47 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGVIDEORENDERER_P_H +#define QFFMPEGVIDEORENDERER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "playbackengine/qffmpegrenderer_p.h" + +#include <QtCore/qpointer.h> + +QT_BEGIN_NAMESPACE + +class QVideoSink; + +namespace QFFmpeg { + +class VideoRenderer : public Renderer +{ + Q_OBJECT +public: + VideoRenderer(const TimeController &tc, QVideoSink *sink, QtVideo::Rotation rotation); + + void setOutput(QVideoSink *sink, bool cleanPrevSink = false); + +protected: + RenderingResult renderInternal(Frame frame) override; + +private: + QPointer<QVideoSink> m_sink; + QtVideo::Rotation m_rotation; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGVIDEORENDERER_P_H diff --git a/src/plugins/multimedia/ffmpeg/qandroidcamera.cpp b/src/plugins/multimedia/ffmpeg/qandroidcamera.cpp new file mode 100644 index 000000000..34d18f7a7 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidcamera.cpp @@ -0,0 +1,699 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qandroidcamera_p.h" + +#include <jni.h> +#include <QMediaFormat> +#include <memory> +#include <optional> +#include <qmediadevices.h> +#include <qguiapplication.h> +#include <qscreen.h> +#include <QDebug> +#include <qloggingcategory.h> +#include <QtCore/qcoreapplication.h> +#include <QtCore/qpermissions.h> +#include <QtCore/private/qandroidextras_p.h> +#include <private/qcameradevice_p.h> +#include <QReadWriteLock> +#include <private/qvideoframeconverter_p.h> +#include <private/qvideotexturehelper_p.h> +#include <qffmpegvideobuffer_p.h> + +#include <qandroidcameraframe_p.h> +#include <utility> + +extern "C" { +#include "libavutil/hwcontext.h" +} + +QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLCAndroidCamera, "qt.multimedia.ffmpeg.androidCamera"); + +typedef QMap<QString, QAndroidCamera *> QAndroidCameraMap; +Q_GLOBAL_STATIC(QAndroidCameraMap, g_qcameras) +Q_GLOBAL_STATIC(QReadWriteLock, rwLock) + +namespace { + +QCameraFormat getDefaultCameraFormat(const QCameraDevice & cameraDevice) +{ + // default settings + QCameraFormatPrivate *defaultFormat = new QCameraFormatPrivate{ + .pixelFormat = QVideoFrameFormat::Format_YUV420P, + .resolution = { 1920, 1080 }, + .minFrameRate = 12, + .maxFrameRate = 30, + }; + QCameraFormat format = defaultFormat->create(); + + if (!cameraDevice.videoFormats().empty() && !cameraDevice.videoFormats().contains(format)) + return cameraDevice.videoFormats().first(); + + return format; +} + +bool checkCameraPermission() +{ + QCameraPermission permission; + + const bool granted = qApp->checkPermission(permission) == Qt::PermissionStatus::Granted; + if (!granted) + qCWarning(qLCAndroidCamera) << "Access to camera not granted!"; + + return granted; +} + +int sensorOrientation(QString cameraId) +{ + QJniObject deviceManager(QtJniTypes::Traits<QtJniTypes::QtVideoDeviceManager>::className(), + QNativeInterface::QAndroidApplication::context()); + + if (!deviceManager.isValid()) { + qCWarning(qLCAndroidCamera) << "Failed to connect to Qt Video Device Manager."; + return 0; + } + + return deviceManager.callMethod<jint>("getSensorOrientation", + QJniObject::fromString(cameraId).object<jstring>()); +} +} // namespace + +// QAndroidCamera + +QAndroidCamera::QAndroidCamera(QCamera *camera) : QPlatformCamera(camera) +{ + m_jniCamera = QJniObject(QtJniTypes::Traits<QtJniTypes::QtCamera2>::className(), + QNativeInterface::QAndroidApplication::context()); + + m_hwAccel = QFFmpeg::HWAccel::create(AVHWDeviceType::AV_HWDEVICE_TYPE_MEDIACODEC); + if (camera) { + m_cameraDevice = camera->cameraDevice(); + m_cameraFormat = !camera->cameraFormat().isNull() ? camera->cameraFormat() + : getDefaultCameraFormat(m_cameraDevice); + updateCameraCharacteristics(); + } + + if (qApp) { + connect(qApp, &QGuiApplication::applicationStateChanged, + this, &QAndroidCamera::onApplicationStateChanged); + } +}; + +QAndroidCamera::~QAndroidCamera() +{ + { + QWriteLocker locker(rwLock); + g_qcameras->remove(m_cameraDevice.id()); + + m_jniCamera.callMethod<void>("stopAndClose"); + setState(State::Closed); + } + + m_jniCamera.callMethod<void>("stopBackgroundThread"); +} + +void QAndroidCamera::setCamera(const QCameraDevice &camera) +{ + const bool active = isActive(); + if (active) + setActive(false); + + m_cameraDevice = camera; + updateCameraCharacteristics(); + m_cameraFormat = getDefaultCameraFormat(camera); + + if (active) + setActive(true); +} + +std::optional<int> QAndroidCamera::ffmpegHWPixelFormat() const +{ + return QFFmpegVideoBuffer::toAVPixelFormat(m_androidFramePixelFormat); +} + +QVideoFrameFormat QAndroidCamera::frameFormat() const +{ + QVideoFrameFormat result = QPlatformCamera::frameFormat(); + result.setRotation(rotation()); + return result; +} + +static void deleteFrame(void *opaque, uint8_t *data) +{ + Q_UNUSED(data); + + auto frame = reinterpret_cast<QAndroidCameraFrame *>(opaque); + + if (frame) + delete frame; +} + +void QAndroidCamera::frameAvailable(QJniObject image, bool takePhoto) +{ + if (!(m_state == State::WaitingStart || m_state == State::Started) && !m_waitingForFirstFrame) { + qCWarning(qLCAndroidCamera) << "Received frame when not active... ignoring"; + qCWarning(qLCAndroidCamera) << "state:" << m_state; + image.callMethod<void>("close"); + return; + } + + auto androidFrame = new QAndroidCameraFrame(image); + if (!androidFrame->isParsed()) { + qCWarning(qLCAndroidCamera) << "Failed to parse frame.. dropping frame"; + delete androidFrame; + return; + } + + int timestamp = androidFrame->timestamp(); + m_androidFramePixelFormat = androidFrame->format(); + if (m_waitingForFirstFrame) { + m_waitingForFirstFrame = false; + setState(State::Started); + } + auto avframe = QFFmpeg::makeAVFrame(); + + avframe->width = androidFrame->size().width(); + avframe->height = androidFrame->size().height(); + avframe->format = QFFmpegVideoBuffer::toAVPixelFormat(androidFrame->format()); + + avframe->extended_data = avframe->data; + avframe->pts = androidFrame->timestamp(); + + for (int planeNumber = 0; planeNumber < androidFrame->numberPlanes(); planeNumber++) { + QAndroidCameraFrame::Plane plane = androidFrame->plane(planeNumber); + avframe->linesize[planeNumber] = plane.rowStride; + avframe->data[planeNumber] = plane.data; + } + + avframe->data[3] = nullptr; + avframe->buf[0] = nullptr; + + avframe->opaque_ref = av_buffer_create(NULL, 1, deleteFrame, androidFrame, 0); + avframe->extended_data = avframe->data; + avframe->pts = timestamp; + + QVideoFrameFormat format(androidFrame->size(), androidFrame->format()); + + QVideoFrame videoFrame(new QFFmpegVideoBuffer(std::move(avframe)), format); + + if (lastTimestamp == 0) + lastTimestamp = timestamp; + + videoFrame.setRotation(rotation()); + videoFrame.setMirrored(m_cameraDevice.position() == QCameraDevice::Position::FrontFace); + + videoFrame.setStartTime(lastTimestamp); + videoFrame.setEndTime(timestamp); + + if (!takePhoto) + emit newVideoFrame(videoFrame); + else + emit onCaptured(videoFrame); + + lastTimestamp = timestamp; +} + +QtVideo::Rotation QAndroidCamera::rotation() const +{ + auto screen = QGuiApplication::primaryScreen(); + auto screenOrientation = screen->orientation(); + if (screenOrientation == Qt::PrimaryOrientation) + screenOrientation = screen->primaryOrientation(); + + // Display rotation is the opposite direction of the physical device rotation. We need the + // device rotation, that's why Landscape is 270 and InvertedLandscape is 90 + int deviceOrientation = 0; + switch (screenOrientation) { + case Qt::PrimaryOrientation: + case Qt::PortraitOrientation: + break; + case Qt::LandscapeOrientation: + deviceOrientation = 270; + break; + case Qt::InvertedPortraitOrientation: + deviceOrientation = 180; + break; + case Qt::InvertedLandscapeOrientation: + deviceOrientation = 90; + break; + } + + int sign = (m_cameraDevice.position() == QCameraDevice::Position::FrontFace) ? 1 : -1; + int rotation = (sensorOrientation(m_cameraDevice.id()) - deviceOrientation * sign + 360) % 360; + + return QtVideo::Rotation(rotation); +} + +void QAndroidCamera::setActive(bool active) +{ + if (isActive() == active) + return; + + if (!m_jniCamera.isValid()) { + updateError(QCamera::CameraError, QStringLiteral("No connection to Android Camera2 API")); + return; + } + + if (active && checkCameraPermission()) { + QWriteLocker locker(rwLock); + int width = m_cameraFormat.resolution().width(); + int height = m_cameraFormat.resolution().height(); + + if (width < 0 || height < 0) { + m_cameraFormat = getDefaultCameraFormat(m_cameraDevice); + width = m_cameraFormat.resolution().width(); + height = m_cameraFormat.resolution().height(); + } + + width = FFALIGN(width, 16); + height = FFALIGN(height, 16); + + setState(State::WaitingOpen); + g_qcameras->insert(m_cameraDevice.id(), this); + + // this should use the camera format. + // but there is only 2 fully supported formats on android - JPG and YUV420P + // and JPEG is not supported for encoding in FFmpeg, so it's locked for YUV for now. + const static int imageFormat = + QJniObject::getStaticField<QtJniTypes::AndroidImageFormat, jint>("YUV_420_888"); + m_jniCamera.callMethod<void>("prepareCamera", jint(width), jint(height), + jint(imageFormat), jint(m_cameraFormat.minFrameRate()), + jint(m_cameraFormat.maxFrameRate())); + + bool canOpen = m_jniCamera.callMethod<jboolean>( + "open", QJniObject::fromString(m_cameraDevice.id()).object<jstring>()); + + if (!canOpen) { + g_qcameras->remove(m_cameraDevice.id()); + setState(State::Closed); + updateError(QCamera::CameraError, + QString("Failed to start camera: ").append(m_cameraDevice.description())); + } + } else { + m_jniCamera.callMethod<void>("stopAndClose"); + m_jniCamera.callMethod<void>("clearSurfaces"); + setState(State::Closed); + } +} + +void QAndroidCamera::setState(QAndroidCamera::State newState) +{ + if (newState == m_state) + return; + + bool wasActive = isActive(); + + if (newState == State::Started) + m_state = State::Started; + + if (m_state == State::Started && newState == State::Closed) + m_state = State::Closed; + + if ((m_state == State::WaitingOpen || m_state == State::WaitingStart) + && newState == State::Closed) { + + m_state = State::Closed; + + updateError(QCamera::CameraError, + QString("Failed to start Camera %1").arg(m_cameraDevice.description())); + } + + if (m_state == State::Closed && newState == State::WaitingOpen) + m_state = State::WaitingOpen; + + if (m_state == State::WaitingOpen && newState == State::WaitingStart) + m_state = State::WaitingStart; + + if (wasActive != isActive()) + emit activeChanged(isActive()); +} + +bool QAndroidCamera::setCameraFormat(const QCameraFormat &format) +{ + const auto chosenFormat = format.isNull() ? getDefaultCameraFormat(m_cameraDevice) : format; + + if (chosenFormat == m_cameraFormat) + return true; + if (!m_cameraDevice.videoFormats().contains(chosenFormat)) + return false; + + m_cameraFormat = chosenFormat; + + if (isActive()) { + // Restart the camera to set new camera format + setActive(false); + setActive(true); + } + + return true; +} + +void QAndroidCamera::updateCameraCharacteristics() +{ + if (m_cameraDevice.id().isEmpty()) { + cleanCameraCharacteristics(); + return; + } + + QJniObject deviceManager(QtJniTypes::Traits<QtJniTypes::QtVideoDeviceManager>::className(), + QNativeInterface::QAndroidApplication::context()); + + if (!deviceManager.isValid()) { + qCWarning(qLCAndroidCamera) << "Failed to connect to Qt Video Device Manager."; + cleanCameraCharacteristics(); + return; + } + + const float maxZoom = deviceManager.callMethod<jfloat>( + "getMaxZoom", QJniObject::fromString(m_cameraDevice.id()).object<jstring>()); + maximumZoomFactorChanged(maxZoom); + if (maxZoom < zoomFactor()) { + zoomTo(1.0, -1.0); + } + + m_TorchModeSupported = deviceManager.callMethod<jboolean>( + "isTorchModeSupported", QJniObject::fromString(m_cameraDevice.id()).object<jstring>()); + + m_supportedFlashModes.clear(); + m_supportedFlashModes.append(QCamera::FlashOff); + QJniObject flashModesObj = deviceManager.callMethod<QtJniTypes::StringArray>( + "getSupportedFlashModes", + QJniObject::fromString(m_cameraDevice.id()).object<jstring>()); + QJniEnvironment jniEnv; + jobjectArray flashModes = flashModesObj.object<jobjectArray>(); + int size = jniEnv->GetArrayLength(flashModes); + for (int i = 0; i < size; ++i) { + QJniObject flashModeObj = jniEnv->GetObjectArrayElement(flashModes, i); + QString flashMode = flashModeObj.toString(); + if (flashMode == QLatin1String("auto")) + m_supportedFlashModes.append(QCamera::FlashAuto); + else if (flashMode == QLatin1String("on")) + m_supportedFlashModes.append(QCamera::FlashOn); + } +} + +void QAndroidCamera::cleanCameraCharacteristics() +{ + maximumZoomFactorChanged(1.0); + if (zoomFactor() != 1.0) { + zoomTo(1.0, -1.0); + } + if (torchMode() != QCamera::TorchOff) { + setTorchMode(QCamera::TorchOff); + } + m_TorchModeSupported = false; + + if (flashMode() != QCamera::FlashOff) { + setFlashMode(QCamera::FlashOff); + } + m_supportedFlashModes.clear(); + m_supportedFlashModes.append(QCamera::FlashOff); +} + +void QAndroidCamera::setFlashMode(QCamera::FlashMode mode) +{ + if (!isFlashModeSupported(mode)) + return; + + QString flashMode; + switch (mode) { + case QCamera::FlashAuto: + flashMode = QLatin1String("auto"); + break; + case QCamera::FlashOn: + flashMode = QLatin1String("on"); + break; + case QCamera::FlashOff: + default: + flashMode = QLatin1String("off"); + break; + } + + m_jniCamera.callMethod<void>("setFlashMode", QJniObject::fromString(flashMode).object<jstring>()); + flashModeChanged(mode); +} + +bool QAndroidCamera::isFlashModeSupported(QCamera::FlashMode mode) const +{ + return m_supportedFlashModes.contains(mode); +} + +bool QAndroidCamera::isFlashReady() const +{ + // Android doesn't have an API for that. + // Only check if device supports more flash modes than just FlashOff. + return m_supportedFlashModes.size() > 1; +} + +bool QAndroidCamera::isTorchModeSupported(QCamera::TorchMode mode) const +{ + if (mode == QCamera::TorchOff) + return true; + else if (mode == QCamera::TorchOn) + return m_TorchModeSupported; + + return false; +} + +void QAndroidCamera::setTorchMode(QCamera::TorchMode mode) +{ + bool torchMode; + if (mode == QCamera::TorchOff) { + torchMode = false; + } else if (mode == QCamera::TorchOn) { + torchMode = true; + } else { + qWarning() << "Unknown Torch mode"; + return; + } + m_jniCamera.callMethod<void>("setTorchMode", jboolean(torchMode)); + torchModeChanged(mode); +} + +void QAndroidCamera::zoomTo(float factor, float rate) +{ + Q_UNUSED(rate); + m_jniCamera.callMethod<void>("zoomTo", factor); + zoomFactorChanged(factor); +} + +void QAndroidCamera::onApplicationStateChanged() +{ + switch (QGuiApplication::applicationState()) { + case Qt::ApplicationInactive: + if (isActive()) { + setActive(false); + m_wasActive = true; + } + break; + case Qt::ApplicationActive: + if (m_wasActive) { + setActive(true); + m_wasActive = false; + } + break; + default: + break; + } +} + +void QAndroidCamera::onCaptureSessionConfigured() +{ + bool canStart = m_jniCamera.callMethod<jboolean>("start", 3); + setState(canStart ? State::WaitingStart : State::Closed); +} + +void QAndroidCamera::onCaptureSessionConfigureFailed() +{ + setState(State::Closed); +} + +void QAndroidCamera::onCameraOpened() +{ + bool canStart = m_jniCamera.callMethod<jboolean>("createSession"); + setState(canStart ? State::WaitingStart : State::Closed); +} + +void QAndroidCamera::onCameraDisconnect() +{ + setState(State::Closed); +} + +void QAndroidCamera::onCameraError(int reason) +{ + updateError(QCamera::CameraError, + QString("Capture error with Camera %1. Camera2 Api error code: %2") + .arg(m_cameraDevice.description()) + .arg(reason)); +} + +void QAndroidCamera::onSessionActive() +{ + m_waitingForFirstFrame = true; +} + +void QAndroidCamera::onSessionClosed() +{ + m_waitingForFirstFrame = false; + setState(State::Closed); +} + +void QAndroidCamera::capture() +{ + m_jniCamera.callMethod<void>("takePhoto"); +} + +void QAndroidCamera::updateExif(const QString &filename) +{ + m_jniCamera.callMethod<void>("saveExifToFile", QJniObject::fromString(filename).object<jstring>()); +} + +void QAndroidCamera::onCaptureSessionFailed(int reason, long frameNumber) +{ + Q_UNUSED(frameNumber); + + updateError(QCamera::CameraError, + QStringLiteral("Capture session failure with Camera %1. Camera2 Api error code: %2") + .arg(m_cameraDevice.description()) + .arg(reason)); +} + +// JNI logic + +#define GET_CAMERA(cameraId) \ + QString key = QJniObject(cameraId).toString(); \ + QReadLocker locker(rwLock); \ + if (!g_qcameras->contains(key)) { \ + qCWarning(qLCAndroidCamera) << "Calling back a QtCamera2 after being destroyed."; \ + return; \ + } \ + QAndroidCamera *camera = g_qcameras->find(key).value(); + +static void onFrameAvailable(JNIEnv *env, jobject obj, jstring cameraId, + QtJniTypes::AndroidImage image) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->frameAvailable(QJniObject(image)); +} +Q_DECLARE_JNI_NATIVE_METHOD(onFrameAvailable) + +static void onPhotoAvailable(JNIEnv *env, jobject obj, jstring cameraId, + QtJniTypes::AndroidImage image) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->frameAvailable(QJniObject(image), true); +} +Q_DECLARE_JNI_NATIVE_METHOD(onPhotoAvailable) + + +static void onCameraOpened(JNIEnv *env, jobject obj, jstring cameraId) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onCameraOpened(); +} +Q_DECLARE_JNI_NATIVE_METHOD(onCameraOpened) + +static void onCameraDisconnect(JNIEnv *env, jobject obj, jstring cameraId) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onCameraDisconnect(); +} +Q_DECLARE_JNI_NATIVE_METHOD(onCameraDisconnect) + +static void onCameraError(JNIEnv *env, jobject obj, jstring cameraId, jint error) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onCameraError(error); +} +Q_DECLARE_JNI_NATIVE_METHOD(onCameraError) + +static void onCaptureSessionConfigured(JNIEnv *env, jobject obj, jstring cameraId) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onCaptureSessionConfigured(); +} +Q_DECLARE_JNI_NATIVE_METHOD(onCaptureSessionConfigured) + +static void onCaptureSessionConfigureFailed(JNIEnv *env, jobject obj, jstring cameraId) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onCaptureSessionConfigureFailed(); +} +Q_DECLARE_JNI_NATIVE_METHOD(onCaptureSessionConfigureFailed) + +static void onSessionActive(JNIEnv *env, jobject obj, jstring cameraId) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onSessionActive(); +} +Q_DECLARE_JNI_NATIVE_METHOD(onSessionActive) + +static void onSessionClosed(JNIEnv *env, jobject obj, jstring cameraId) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onSessionClosed(); +} +Q_DECLARE_JNI_NATIVE_METHOD(onSessionClosed) + +static void onCaptureSessionFailed(JNIEnv *env, jobject obj, jstring cameraId, jint reason, + jlong framenumber) +{ + Q_UNUSED(env); + Q_UNUSED(obj); + GET_CAMERA(cameraId); + + camera->onCaptureSessionFailed(reason, framenumber); +} +Q_DECLARE_JNI_NATIVE_METHOD(onCaptureSessionFailed) + +bool QAndroidCamera::registerNativeMethods() +{ + static const bool registered = []() { + return QJniEnvironment().registerNativeMethods( + QtJniTypes::Traits<QtJniTypes::QtCamera2>::className(), + { + Q_JNI_NATIVE_METHOD(onCameraOpened), + Q_JNI_NATIVE_METHOD(onCameraDisconnect), + Q_JNI_NATIVE_METHOD(onCameraError), + Q_JNI_NATIVE_METHOD(onCaptureSessionConfigured), + Q_JNI_NATIVE_METHOD(onCaptureSessionConfigureFailed), + Q_JNI_NATIVE_METHOD(onCaptureSessionFailed), + Q_JNI_NATIVE_METHOD(onFrameAvailable), + Q_JNI_NATIVE_METHOD(onPhotoAvailable), + Q_JNI_NATIVE_METHOD(onSessionActive), + Q_JNI_NATIVE_METHOD(onSessionClosed), + }); + }(); + return registered; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qandroidcamera_p.h b/src/plugins/multimedia/ffmpeg/qandroidcamera_p.h new file mode 100644 index 000000000..55496c766 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidcamera_p.h @@ -0,0 +1,93 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QANDROIDCAMERA_H +#define QANDROIDCAMERA_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeghwaccel_p.h" +#include <private/qplatformcamera_p.h> +#include <QObject> +#include <QJniObject> + +QT_BEGIN_NAMESPACE + +class QVideoFrame; + +class QAndroidCamera : public QPlatformCamera +{ + Q_OBJECT +public: + enum State { Closed, WaitingOpen, WaitingStart, Started }; + explicit QAndroidCamera(QCamera *camera); + ~QAndroidCamera() override; + + bool isActive() const override { return m_state == State::Started; } + bool isFlashModeSupported(QCamera::FlashMode mode) const override; + bool isFlashReady() const override; + bool isTorchModeSupported(QCamera::TorchMode mode) const override; + void setActive(bool active) override; + void setCamera(const QCameraDevice &camera) override; + bool setCameraFormat(const QCameraFormat &format) override; + void setFlashMode(QCamera::FlashMode mode) override; + void setTorchMode(QCamera::TorchMode mode) override; + void zoomTo(float factor, float rate) override; + + std::optional<int> ffmpegHWPixelFormat() const override; + + QVideoFrameFormat frameFormat() const override; + + static bool registerNativeMethods(); + + void capture(); + void updateExif(const QString &filename); +public slots: + void onApplicationStateChanged(); + void onCameraOpened(); + void onCameraDisconnect(); + void onCameraError(int error); + void frameAvailable(QJniObject image, bool takePhoto = false); + void onCaptureSessionConfigured(); + void onCaptureSessionConfigureFailed(); + void onCaptureSessionFailed(int reason, long frameNumber); + void onSessionActive(); + void onSessionClosed(); + +Q_SIGNALS: + void onCaptured(const QVideoFrame&); + +private: + bool isActivating() const { return m_state != State::Closed; } + + void setState(State newState); + QtVideo::Rotation rotation() const; + void updateCameraCharacteristics(); + void cleanCameraCharacteristics(); + + State m_state = State::Closed; + QCameraDevice m_cameraDevice; + long lastTimestamp = 0; + QJniObject m_jniCamera; + + std::unique_ptr<QFFmpeg::HWAccel> m_hwAccel; + + QVideoFrameFormat::PixelFormat m_androidFramePixelFormat; + QList<QCamera::FlashMode> m_supportedFlashModes; + bool m_waitingForFirstFrame = false; + bool m_TorchModeSupported = false; + bool m_wasActive = false; +}; + +QT_END_NAMESPACE + +#endif // QANDROIDCAMERA_H diff --git a/src/plugins/multimedia/ffmpeg/qandroidcameraframe.cpp b/src/plugins/multimedia/ffmpeg/qandroidcameraframe.cpp new file mode 100644 index 000000000..0bdf3e07f --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidcameraframe.cpp @@ -0,0 +1,231 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qandroidcameraframe_p.h" +#include <jni.h> +#include <QDebug> +#include <QtCore/qjnitypes.h> +#include <QtCore/QLoggingCategory> + +QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLCAndroidCameraFrame, "qt.multimedia.ffmpeg.android.camera.frame"); + +namespace { +bool isWorkaroundForEmulatorNeeded() { + const static bool workaroundForEmulator + = QtJniTypes::QtVideoDeviceManager::callStaticMethod<jboolean>("isEmulator"); + return workaroundForEmulator; +} +} + +bool QAndroidCameraFrame::parse(const QJniObject &frame) +{ + QJniEnvironment jniEnv; + + if (!frame.isValid()) + return false; + + auto planes = frame.callMethod<QtJniTypes::AndroidImagePlaneArray>("getPlanes"); + if (!planes.isValid()) + return false; + + int numberPlanes = jniEnv->GetArrayLength(planes.object<jarray>()); + // create and populate temporary array structure + int pixelStrides[numberPlanes]; + int rowStrides[numberPlanes]; + int bufferSize[numberPlanes]; + uint8_t *buffer[numberPlanes]; + + auto resetPlane = [&](int index) { + if (index < 0 || index > numberPlanes) + return; + + rowStrides[index] = 0; + pixelStrides[index] = 0; + bufferSize[index] = 0; + buffer[index] = nullptr; + }; + + for (int index = 0; index < numberPlanes; index++) { + QJniObject plane = jniEnv->GetObjectArrayElement(planes.object<jobjectArray>(), index); + if (jniEnv.checkAndClearExceptions() || !plane.isValid()) { + resetPlane(index); + continue; + } + + rowStrides[index] = plane.callMethod<jint>("getRowStride"); + pixelStrides[index] = plane.callMethod<jint>("getPixelStride"); + + auto byteBuffer = plane.callMethod<QtJniTypes::JavaByteBuffer>("getBuffer"); + if (!byteBuffer.isValid()) { + resetPlane(index); + continue; + } + + // Uses direct access which is garanteed by android to work with + // ImageReader bytebuffer + buffer[index] = static_cast<uint8_t *>(jniEnv->GetDirectBufferAddress(byteBuffer.object())); + bufferSize[index] = byteBuffer.callMethod<jint>("remaining"); + } + + QVideoFrameFormat::PixelFormat calculedPixelFormat = QVideoFrameFormat::Format_Invalid; + + // finding the image format + // the ImageFormats that can happen here are stated here: + // https://developer.android.com/reference/android/media/Image#getFormat() + int format = frame.callMethod<jint>("getFormat"); + AndroidImageFormat imageFormat = AndroidImageFormat(format); + + switch (imageFormat) { + case AndroidImageFormat::JPEG: + calculedPixelFormat = QVideoFrameFormat::Format_Jpeg; + break; + case AndroidImageFormat::YUV_420_888: + if (numberPlanes < 3) { + // something went wrong on parsing. YUV_420_888 format must always have 3 planes + calculedPixelFormat = QVideoFrameFormat::Format_Invalid; + break; + } + if (pixelStrides[1] == 1) + calculedPixelFormat = QVideoFrameFormat::Format_YUV420P; + else if (pixelStrides[1] == 2) { + if (buffer[1] - buffer[2] == -1) // Interleaved UVUV -> NV12 + calculedPixelFormat = QVideoFrameFormat::Format_NV12; + else if (buffer[1] - buffer[2] == 1) // Interleaved VUVU -> NV21 + calculedPixelFormat = QVideoFrameFormat::Format_NV21; + } + break; + case AndroidImageFormat::HEIC: + // QImage cannot parse HEIC + calculedPixelFormat = QVideoFrameFormat::Format_Invalid; + break; + case AndroidImageFormat::RAW_PRIVATE: + case AndroidImageFormat::RAW_SENSOR: + // we cannot know raw formats + calculedPixelFormat = QVideoFrameFormat::Format_Invalid; + break; + case AndroidImageFormat::FLEX_RGBA_8888: + case AndroidImageFormat::FLEX_RGB_888: + // these formats are only returned by Mediacodec.getOutputImage, they are not used as a + // Camera2 Image frame return + calculedPixelFormat = QVideoFrameFormat::Format_Invalid; + break; + case AndroidImageFormat::YUV_422_888: + case AndroidImageFormat::YUV_444_888: + case AndroidImageFormat::YCBCR_P010: + // not dealing with these formats, they require higher API levels than the current Qt min + calculedPixelFormat = QVideoFrameFormat::Format_Invalid; + break; + default: + calculedPixelFormat = QVideoFrameFormat::Format_Invalid; + break; + } + + if (calculedPixelFormat == QVideoFrameFormat::Format_Invalid) { + qCWarning(qLCAndroidCameraFrame) << "Cannot determine image format!"; + return false; + } + + auto copyPlane = [&](int mapIndex, int arrayIndex) { + if (arrayIndex >= numberPlanes) + return; + + m_planes[mapIndex].rowStride = rowStrides[arrayIndex]; + m_planes[mapIndex].size = bufferSize[arrayIndex]; + m_planes[mapIndex].data = buffer[arrayIndex]; + }; + + int width = frame.callMethod<jint>("getWidth"); + int height = frame.callMethod<jint>("getHeight"); + m_size = QSize(width, height); + + switch (calculedPixelFormat) { + case QVideoFrameFormat::Format_YUV420P: + m_numberPlanes = 3; + copyPlane(0, 0); + copyPlane(1, 1); + copyPlane(2, 2); + + if (isWorkaroundForEmulatorNeeded()) { + for (int i = 0; i < 3; ++i) { + const int dataSize = (i == 0) ? width * height : width * height / 4; + m_planes[i].data = new uint8_t[dataSize]; + memcpy(m_planes[i].data, buffer[i], dataSize); + } + } + + m_pixelFormat = QVideoFrameFormat::Format_YUV420P; + break; + case QVideoFrameFormat::Format_NV12: + case QVideoFrameFormat::Format_NV21: + // Y-plane and combined interleaved UV-plane + m_numberPlanes = 2; + copyPlane(0, 0); + + // Android reports U and V planes as planes[1] and planes[2] respectively, regardless of the + // order of interleaved samples. We point to whichever is first in memory. + copyPlane(1, calculedPixelFormat == QVideoFrameFormat::Format_NV21 ? 2 : 1); + + // With interleaved UV plane, Android reports the size of each plane as the smallest size + // that includes all samples of that plane. For example, if the UV plane is [u, v, u, v], + // the size of the U-plane is 3, not 4. With FFmpeg we need to count the total number of + // bytes in the UV-plane, which is 1 more than what Android reports. + m_planes[1].size++; + + m_pixelFormat = calculedPixelFormat; + break; + case QVideoFrameFormat::Format_Jpeg: + qCWarning(qLCAndroidCameraFrame) + << "FFmpeg HW Mediacodec does not encode other than YCbCr formats"; + // we still parse it to preview the frame + m_image = QImage::fromData(buffer[0], bufferSize[0]); + m_planes[0].rowStride = m_image.bytesPerLine(); + m_planes[0].size = m_image.sizeInBytes(); + m_planes[0].data = m_image.bits(); + m_pixelFormat = QVideoFrameFormat::pixelFormatFromImageFormat(m_image.format()); + break; + default: + break; + } + + long timestamp = frame.callMethod<jlong>("getTimestamp"); + m_timestamp = timestamp / 1000; + + return true; +} + +QAndroidCameraFrame::QAndroidCameraFrame(QJniObject frame) + : m_pixelFormat(QVideoFrameFormat::Format_Invalid), m_parsed(parse(frame)) +{ + if (isParsed()) { + // holding the frame java object + QJniEnvironment jniEnv; + m_frame = jniEnv->NewGlobalRef(frame.object()); + jniEnv.checkAndClearExceptions(); + } else if (frame.isValid()) { + frame.callMethod<void>("close"); + } +} + +QAndroidCameraFrame::~QAndroidCameraFrame() +{ + if (!isParsed()) // nothing to clean + return; + + QJniObject qFrame(m_frame); + if (qFrame.isValid()) + qFrame.callMethod<void>("close"); + + QJniEnvironment jniEnv; + if (m_frame) + jniEnv->DeleteGlobalRef(m_frame); + + if (isWorkaroundForEmulatorNeeded()) { + if (m_pixelFormat == QVideoFrameFormat::Format_YUV420P) { + for (int i = 0; i < 3; ++i) + delete[] m_planes[i].data; + } + } +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qandroidcameraframe_p.h b/src/plugins/multimedia/ffmpeg/qandroidcameraframe_p.h new file mode 100644 index 000000000..f55c066ae --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidcameraframe_p.h @@ -0,0 +1,94 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QANDROIDCAMERAFRAME_H +#define QANDROIDCAMERAFRAME_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QVideoFrameFormat> +#include <QJniObject> +#include <QtCore/qjnitypes.h> + +Q_DECLARE_JNI_CLASS(QtCamera2, "org/qtproject/qt/android/multimedia/QtCamera2"); +Q_DECLARE_JNI_CLASS(QtVideoDeviceManager, + "org/qtproject/qt/android/multimedia/QtVideoDeviceManager"); + +Q_DECLARE_JNI_CLASS(AndroidImage, "android/media/Image"); +Q_DECLARE_JNI_CLASS(AndroidImageFormat, "android/graphics/ImageFormat"); +Q_DECLARE_JNI_CLASS(AndroidImagePlane, "android/media/Image$Plane"); +Q_DECLARE_JNI_CLASS(JavaByteBuffer, "java/nio/ByteBuffer"); + +#ifndef QT_DECLARE_JNI_CLASS_STANDARD_TYPES +Q_DECLARE_JNI_CLASS(String, "java/lang/String"); +#endif + +namespace QtJniTypes { +using AndroidImagePlaneArray = QJniArray<AndroidImagePlane>; +using StringArray = QJniArray<String>; +} + +class QAndroidCameraFrame +{ +public: + struct Plane + { + int pixelStride = 0; + int rowStride = 0; + int size = 0; + uint8_t *data; + }; + + QAndroidCameraFrame(QJniObject frame); + ~QAndroidCameraFrame(); + + QVideoFrameFormat::PixelFormat format() const { return m_pixelFormat; } + int numberPlanes() const { return m_numberPlanes; } + Plane plane(int index) const + { + if (index < 0 || index >= numberPlanes()) + return {}; + + return m_planes[index]; + } + QSize size() const { return m_size; } + long timestamp() const { return m_timestamp; } + + bool isParsed() const { return m_parsed; } + +private: + bool parse(const QJniObject &frame); + QVideoFrameFormat::PixelFormat m_pixelFormat; + + QSize m_size = {}; + long m_timestamp = 0; + int m_numberPlanes = 0; + Plane m_planes[3]; // 3 max number planes + jobject m_frame = nullptr; + bool m_parsed = false; + QImage m_image; + + enum AndroidImageFormat { + RAW_SENSOR = 32, + YUV_420_888 = 35, + RAW_PRIVATE = 36, + YUV_422_888 = 39, + YUV_444_888 = 40, + FLEX_RGB_888 = 41, + FLEX_RGBA_8888 = 42, + YCBCR_P010 = 54, + JPEG = 256, + HEIC = 1212500294 + }; +}; + +#endif // QANDROIDCAMERAFRAME_H diff --git a/src/plugins/multimedia/ffmpeg/qandroidimagecapture.cpp b/src/plugins/multimedia/ffmpeg/qandroidimagecapture.cpp new file mode 100644 index 000000000..ed0f2de9d --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidimagecapture.cpp @@ -0,0 +1,46 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qandroidimagecapture_p.h" +#include <qandroidcamera_p.h> + +QT_BEGIN_NAMESPACE + +QAndroidImageCapture::QAndroidImageCapture(QImageCapture *parent) + : QFFmpegImageCapture(parent) +{ + connect(this, &QPlatformImageCapture::imageSaved, this, &QAndroidImageCapture::updateExif); +} + +QAndroidImageCapture::~QAndroidImageCapture() +{ +} + +int QAndroidImageCapture::doCapture(const QString &fileName) +{ + auto ret = QFFmpegImageCapture::doCapture(fileName); + if (ret >= 0) { + auto androidCamera = qobject_cast<QAndroidCamera *>(videoSource()); + if (androidCamera) + androidCamera->capture(); + } + + return ret; +} + +void QAndroidImageCapture::setupVideoSourceConnections() +{ + auto androidCamera = qobject_cast<QAndroidCamera *>(videoSource()); + if (androidCamera) + connect(androidCamera, &QAndroidCamera::onCaptured, this, &QAndroidImageCapture::newVideoFrame); + else + QFFmpegImageCapture::setupVideoSourceConnections(); +} + +void QAndroidImageCapture::updateExif(int id, const QString &filename) +{ + Q_UNUSED(id); + auto androidCamera = qobject_cast<QAndroidCamera *>(videoSource()); + if (androidCamera) + androidCamera->updateExif(filename); +} diff --git a/src/plugins/multimedia/ffmpeg/qandroidimagecapture_p.h b/src/plugins/multimedia/ffmpeg/qandroidimagecapture_p.h new file mode 100644 index 000000000..8a997b595 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidimagecapture_p.h @@ -0,0 +1,38 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QANDROIDIMAGECAPTURE_H +#define QANDROIDIMAGECAPTURE_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpegimagecapture_p.h" + +QT_BEGIN_NAMESPACE + +class QAndroidImageCapture : public QFFmpegImageCapture +{ +public: + QAndroidImageCapture(QImageCapture *parent); + ~QAndroidImageCapture() override; + +protected: + void setupVideoSourceConnections() override; + int doCapture(const QString &fileName) override; + +private slots: + void updateExif(int id, const QString &filename); +}; + +QT_END_NAMESPACE + +#endif // QANDROIDIMAGECAPTURE_H diff --git a/src/plugins/multimedia/ffmpeg/qandroidvideodevices.cpp b/src/plugins/multimedia/ffmpeg/qandroidvideodevices.cpp new file mode 100644 index 000000000..0116171d0 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidvideodevices.cpp @@ -0,0 +1,146 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qandroidvideodevices_p.h" +#include "qandroidcameraframe_p.h" + +#include <private/qcameradevice_p.h> + +#include <QtCore/QLoggingCategory> +#include <QtCore/qcoreapplication.h> +#include <QtCore/private/qandroidextras_p.h> +#include <QtCore/qcoreapplication_platform.h> +#include <QtCore/qjnienvironment.h> +#include <jni.h> + +static Q_LOGGING_CATEGORY(qLCAndroidVideoDevices, "qt.multimedia.ffmpeg.android.videoDevices") + +QCameraFormat createCameraFormat(int width, int height, int fpsMin, int fpsMax) +{ + QCameraFormatPrivate *format = new QCameraFormatPrivate(); + + format->resolution = { width, height }; + + format->minFrameRate = fpsMin; + format->maxFrameRate = fpsMax; + + format->pixelFormat = QVideoFrameFormat::PixelFormat::Format_YUV420P; + + return format->create(); +} + +QList<QCameraDevice> QAndroidVideoDevices::findVideoDevices() +{ + QList<QCameraDevice> devices; + + QJniObject deviceManager(QtJniTypes::Traits<QtJniTypes::QtVideoDeviceManager>::className(), + QNativeInterface::QAndroidApplication::context()); + + if (!deviceManager.isValid()) { + qCWarning(qLCAndroidVideoDevices) << "Failed to connect to Qt Video Device Manager."; + return devices; + } + + QJniObject cameraIdList = deviceManager.callMethod<QtJniTypes::StringArray>("getCameraIdList"); + + QJniEnvironment jniEnv; + int numCameras = jniEnv->GetArrayLength(cameraIdList.object<jarray>()); + if (jniEnv.checkAndClearExceptions()) + return devices; + + for (int cameraIndex = 0; cameraIndex < numCameras; cameraIndex++) { + + QJniObject cameraIdObject = + jniEnv->GetObjectArrayElement(cameraIdList.object<jobjectArray>(), cameraIndex); + if (jniEnv.checkAndClearExceptions()) + continue; + + jstring cameraId = cameraIdObject.object<jstring>(); + + QCameraDevicePrivate *info = new QCameraDevicePrivate; + info->id = cameraIdObject.toString().toUtf8(); + + info->orientation = deviceManager.callMethod<jint>("getSensorOrientation", cameraId); + + int facing = deviceManager.callMethod<jint>("getLensFacing", cameraId); + + const int LENS_FACING_FRONT = 0; + const int LENS_FACING_BACK = 1; + const int LENS_FACING_EXTERNAL = 2; + + switch (facing) { + case LENS_FACING_EXTERNAL: + case LENS_FACING_BACK: + info->position = QCameraDevice::BackFace; + info->description = QString("Rear Camera: %1").arg(cameraIndex); + break; + case LENS_FACING_FRONT: + info->position = QCameraDevice::FrontFace; + info->description = QString("Front Camera: %1").arg(cameraIndex); + break; + } + + QJniObject fpsRangesObject = + deviceManager.callMethod<QtJniTypes::StringArray>("getFpsRange", cameraId); + jobjectArray fpsRanges = fpsRangesObject.object<jobjectArray>(); + + int numRanges = jniEnv->GetArrayLength(fpsRanges); + if (jniEnv.checkAndClearExceptions()) + continue; + + int maxFps = 0, minFps = 0; + + for (int rangeIndex = 0; rangeIndex < numRanges; rangeIndex++) { + + QJniObject rangeString = jniEnv->GetObjectArrayElement(fpsRanges, rangeIndex); + if (jniEnv.checkAndClearExceptions()) + continue; + + QString range = rangeString.toString(); + + range = range.remove("["); + range = range.remove("]"); + + auto split = range.split(","); + + int min = split[0].toInt(); + int max = split[1].toInt(); + + if (max > maxFps) { + maxFps = max; + minFps = min; + } + } + + const static int imageFormat = + QJniObject::getStaticField<QtJniTypes::AndroidImageFormat, jint>("YUV_420_888"); + + QJniObject sizesObject = deviceManager.callMethod<QtJniTypes::StringArray>( + "getStreamConfigurationsSizes", cameraId, imageFormat); + + jobjectArray streamSizes = sizesObject.object<jobjectArray>(); + int numSizes = jniEnv->GetArrayLength(streamSizes); + if (jniEnv.checkAndClearExceptions()) + continue; + + for (int sizesIndex = 0; sizesIndex < numSizes; sizesIndex++) { + + QJniObject sizeStringObject = jniEnv->GetObjectArrayElement(streamSizes, sizesIndex); + if (jniEnv.checkAndClearExceptions()) + continue; + + QString sizeString = sizeStringObject.toString(); + + auto split = sizeString.split("x"); + + int width = split[0].toInt(); + int height = split[1].toInt(); + + info->videoFormats.append(createCameraFormat(width, height, minFps, maxFps)); + } + + devices.push_back(info->create()); + } + + return devices; +} diff --git a/src/plugins/multimedia/ffmpeg/qandroidvideodevices_p.h b/src/plugins/multimedia/ffmpeg/qandroidvideodevices_p.h new file mode 100644 index 000000000..f89ea7f05 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qandroidvideodevices_p.h @@ -0,0 +1,35 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFANDROIDVIDEODEVICES_H +#define QFANDROIDVIDEODEVICES_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QObject> +#include <private/qplatformvideodevices_p.h> + +class QAndroidVideoDevices : public QPlatformVideoDevices +{ + Q_OBJECT +public: + QAndroidVideoDevices(QPlatformMediaIntegration *integration) + : QPlatformVideoDevices(integration), m_videoDevices(findVideoDevices()){}; + + QList<QCameraDevice> videoDevices() const override { return m_videoDevices; } + +private: + QList<QCameraDevice> findVideoDevices(); + QList<QCameraDevice> m_videoDevices; +}; + +#endif // QFANDROIDVIDEODEVICES_H diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera.mm b/src/plugins/multimedia/ffmpeg/qavfcamera.mm index 11eb4dd6a..891c4b376 100644 --- a/src/plugins/multimedia/ffmpeg/qavfcamera.mm +++ b/src/plugins/multimedia/ffmpeg/qavfcamera.mm @@ -1,49 +1,17 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <qavfcamera_p.h> #include <qpointer.h> #include <qmediacapturesession.h> #include <private/qplatformmediacapture_p.h> #include "avfcamerautility_p.h" #include "qavfhelpers_p.h" +#include "avfcameradebug_p.h" +#include "qavfsamplebufferdelegate_p.h" #include <qvideosink.h> -#include <private/qrhi_p.h> +#include <rhi/qrhi.h> +#include <QtCore/qcoreapplication.h> +#include <QtCore/qpermissions.h> #define AVMediaType XAVMediaType #include "qffmpegvideobuffer_p.h" #include "qffmpegvideosink_p.h" @@ -53,144 +21,16 @@ extern "C" { } #undef AVMediaType - - -#import <AVFoundation/AVFoundation.h> -#include <CoreVideo/CoreVideo.h> - -static void releaseHwFrame(void */*opaque*/, uint8_t *data) -{ - CVPixelBufferRelease(CVPixelBufferRef(data)); -} - -// Make sure this is compatible with the layout used in ffmpeg's hwcontext_videotoolbox -static AVFrame *allocHWFrame(AVBufferRef *hwContext, const CVPixelBufferRef &pixbuf) -{ - AVHWFramesContext *ctx = (AVHWFramesContext*)hwContext->data; - AVFrame *frame = av_frame_alloc(); - frame->hw_frames_ctx = av_buffer_ref(hwContext); - frame->extended_data = frame->data; - - frame->buf[0] = av_buffer_create((uint8_t *)pixbuf, 1, releaseHwFrame, NULL, 0); - frame->data[3] = (uint8_t *)pixbuf; - CVPixelBufferRetain(pixbuf); - frame->width = ctx->width; - frame->height = ctx->height; - frame->format = AV_PIX_FMT_VIDEOTOOLBOX; - if (frame->width != (int)CVPixelBufferGetWidth(pixbuf) || - frame->height != (int)CVPixelBufferGetHeight(pixbuf)) { - // This can happen while changing camera format - av_frame_free(&frame); - return nullptr; - } - return frame; -} - -static AVAuthorizationStatus m_cameraAuthorizationStatus = AVAuthorizationStatusNotDetermined; - -@interface QAVFSampleBufferDelegate : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate> - -- (QAVFSampleBufferDelegate *) initWithCamera:(QAVFCamera *)renderer; - -- (void) captureOutput:(AVCaptureOutput *)captureOutput - didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection; - -- (void) setHWAccel:(QFFmpeg::HWAccel *)accel; - -@end - -@implementation QAVFSampleBufferDelegate -{ -@private - QAVFCamera *m_camera; - AVBufferRef *hwFramesContext; - QFFmpeg::HWAccel m_accel; - qint64 startTime; - qint64 baseTime; -} - -- (QAVFSampleBufferDelegate *) initWithCamera:(QAVFCamera *)renderer -{ - if (!(self = [super init])) - return nil; - - m_camera = renderer; - hwFramesContext = nullptr; - startTime = 0; - baseTime = 0; - return self; -} - -- (void)captureOutput:(AVCaptureOutput *)captureOutput - didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer - fromConnection:(AVCaptureConnection *)connection -{ - Q_UNUSED(connection); - Q_UNUSED(captureOutput); - - // NB: on iOS captureOutput/connection can be nil (when recording a video - - // avfmediaassetwriter). - - CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); - - CMTime time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); - qint64 frameTime = time.timescale ? time.value*1000/time.timescale : 0; - if (baseTime == 0) { - // drop the first frame to get a valid frame start time - baseTime = frameTime; - startTime = 0; - return; - } - - AVFrame *avFrame = allocHWFrame(m_accel.hwFramesContextAsBuffer(), imageBuffer); - if (!avFrame) - return; - -#ifdef USE_SW_FRAMES - auto *swFrame = av_frame_alloc(); - /* retrieve data from GPU to CPU */ - int ret = av_hwframe_transfer_data(swFrame, avFrame, 0); - if (ret < 0) { - qWarning() << "Error transferring the data to system memory\n"; - av_frame_unref(swFrame); - } else { - av_frame_unref(avFrame); - avFrame = swFrame; - } -#endif - - QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(imageBuffer); - if (!format.isValid()) { - av_frame_unref(avFrame); - return; - } - - avFrame->pts = startTime; - - QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(avFrame); - QVideoFrame frame(buffer, format); - frame.setStartTime(startTime); - frame.setEndTime(frameTime); - startTime = frameTime; - - m_camera->syncHandleFrame(frame); -} - -- (void) setHWAccel:(QFFmpeg::HWAccel *)accel -{ - m_accel = *accel; -} - -@end - QT_BEGIN_NAMESPACE +using namespace QFFmpeg; + QAVFCamera::QAVFCamera(QCamera *parent) : QAVFCameraBase(parent) { m_captureSession = [[AVCaptureSession alloc] init]; - m_sampleBufferDelegate = [[QAVFSampleBufferDelegate alloc] initWithCamera:this]; + m_sampleBufferDelegate = [[QAVFSampleBufferDelegate alloc] + initWithFrameHandler:[this](const QVideoFrame &frame) { syncHandleFrame(frame); }]; } QAVFCamera::~QAVFCamera() @@ -201,53 +41,19 @@ QAVFCamera::~QAVFCamera() [m_captureSession release]; } -void QAVFCamera::requestCameraPermissionIfNeeded() +bool QAVFCamera::checkCameraPermission() { - if (m_cameraAuthorizationStatus == AVAuthorizationStatusAuthorized) - return; + const QCameraPermission permission; + const bool granted = qApp->checkPermission(permission) == Qt::PermissionStatus::Granted; + if (!granted) + qWarning() << "Access to camera not granted"; - switch ([AVCaptureDevice authorizationStatusForMediaType:AVMediaTypeVideo]) - { - case AVAuthorizationStatusAuthorized: - { - m_cameraAuthorizationStatus = AVAuthorizationStatusAuthorized; - break; - } - case AVAuthorizationStatusNotDetermined: - { - m_cameraAuthorizationStatus = AVAuthorizationStatusNotDetermined; - QPointer<QAVFCamera> guard(this); - [AVCaptureDevice requestAccessForMediaType:AVMediaTypeVideo completionHandler:^(BOOL granted) { - dispatch_async(dispatch_get_main_queue(), ^{ - if (guard) - cameraAuthorizationChanged(granted); - }); - }]; - break; - } - case AVAuthorizationStatusDenied: - case AVAuthorizationStatusRestricted: - { - m_cameraAuthorizationStatus = AVAuthorizationStatusDenied; - return; - } - } -} - -void QAVFCamera::cameraAuthorizationChanged(bool authorized) -{ - if (authorized) { - m_cameraAuthorizationStatus = AVAuthorizationStatusAuthorized; - } else { - m_cameraAuthorizationStatus = AVAuthorizationStatusDenied; - qWarning() << "User has denied access to camera"; - } + return granted; } void QAVFCamera::updateVideoInput() { - requestCameraPermissionIfNeeded(); - if (m_cameraAuthorizationStatus != AVAuthorizationStatusAuthorized) + if (!checkCameraPermission()) return; [m_captureSession beginConfiguration]; @@ -255,7 +61,7 @@ void QAVFCamera::updateVideoInput() attachVideoInputDevice(); if (!m_videoDataOutput) { - m_videoDataOutput = [[[AVCaptureVideoDataOutput alloc] init] autorelease]; + m_videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; // Configure video output m_delegateQueue = dispatch_queue_create("vf_queue", nullptr); @@ -342,8 +148,7 @@ void QAVFCamera::setActive(bool active) { if (m_active == active) return; - requestCameraPermissionIfNeeded(); - if (m_cameraAuthorizationStatus != AVAuthorizationStatusAuthorized) + if (!checkCameraPermission()) return; m_active = active; @@ -374,8 +179,7 @@ void QAVFCamera::setCamera(const QCameraDevice &camera) m_cameraDevice = camera; - requestCameraPermissionIfNeeded(); - if (m_cameraAuthorizationStatus == AVAuthorizationStatusAuthorized) + if (checkCameraPermission()) updateVideoInput(); setCameraFormat({}); } @@ -385,58 +189,142 @@ bool QAVFCamera::setCameraFormat(const QCameraFormat &format) if (m_cameraFormat == format && !format.isNull()) return true; - QAVFCameraBase::setCameraFormat(format); + if (!QAVFCameraBase::setCameraFormat(format)) + return false; + updateCameraFormat(); return true; } void QAVFCamera::updateCameraFormat() { + m_framePixelFormat = QVideoFrameFormat::Format_Invalid; + AVCaptureDevice *captureDevice = device(); if (!captureDevice) return; - uint avPixelFormat = 0; - AVCaptureDeviceFormat *newFormat = qt_convert_to_capture_device_format(captureDevice, m_cameraFormat); + AVCaptureDeviceFormat *newFormat = qt_convert_to_capture_device_format( + captureDevice, m_cameraFormat, &isCVFormatSupported); + + if (!newFormat) + newFormat = qt_convert_to_capture_device_format(captureDevice, m_cameraFormat); + + std::uint32_t cvPixelFormat = 0; if (newFormat) { qt_set_active_format(captureDevice, newFormat, false); - avPixelFormat = setPixelFormat(m_cameraFormat.pixelFormat()); + const auto captureDeviceCVFormat = + CMVideoFormatDescriptionGetCodecType(newFormat.formatDescription); + cvPixelFormat = setPixelFormat(m_cameraFormat.pixelFormat(), captureDeviceCVFormat); + if (captureDeviceCVFormat != cvPixelFormat) { + qCWarning(qLcCamera) << "Output CV format differs with capture device format!" + << cvPixelFormat << cvFormatToString(cvPixelFormat) << "vs" + << captureDeviceCVFormat + << cvFormatToString(captureDeviceCVFormat); + + m_framePixelFormat = QAVFHelpers::fromCVPixelFormat(cvPixelFormat); + } + } else { + qWarning() << "Cannot find AVCaptureDeviceFormat; Did you use format from another camera?"; + } + + const AVPixelFormat avPixelFormat = av_map_videotoolbox_format_to_pixfmt(cvPixelFormat); + + std::unique_ptr<HWAccel> hwAccel; + + if (avPixelFormat == AV_PIX_FMT_NONE) { + qCWarning(qLcCamera) << "Videotoolbox doesn't support cvPixelFormat:" << cvPixelFormat + << cvFormatToString(cvPixelFormat) + << "Camera pix format:" << m_cameraFormat.pixelFormat(); + } else { + hwAccel = HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX); + qCDebug(qLcCamera) << "Create VIDEOTOOLBOX hw context" << hwAccel.get() << "for camera"; + } + + if (hwAccel) { + hwAccel->createFramesContext(avPixelFormat, adjustedResolution()); + m_hwPixelFormat = hwAccel->hwFormat(); + } else { + m_hwPixelFormat = AV_PIX_FMT_NONE; } - hwAccel = QFFmpeg::HWAccel(AV_HWDEVICE_TYPE_VIDEOTOOLBOX); - hwAccel.createFramesContext(av_map_videotoolbox_format_to_pixfmt(avPixelFormat), m_cameraFormat.resolution()); - [m_sampleBufferDelegate setHWAccel:&hwAccel]; + [m_sampleBufferDelegate setHWAccel:std::move(hwAccel)]; + [m_sampleBufferDelegate setVideoFormatFrameRate:m_cameraFormat.maxFrameRate()]; } -uint QAVFCamera::setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat) +uint32_t QAVFCamera::setPixelFormat(QVideoFrameFormat::PixelFormat cameraPixelFormat, + uint32_t inputCvPixFormat) { - // Default to 32BGRA pixel formats on the viewfinder, in case the requested - // format can't be used (shouldn't happen unless the developers sets a wrong camera - // format on the camera). - unsigned avPixelFormat = kCVPixelFormatType_32BGRA; - if (!QAVFHelpers::toCVPixelFormat(pixelFormat, avPixelFormat)) - qWarning() << "QCamera::setCameraFormat: couldn't convert requested pixel format, using ARGB32"; - - bool isSupported = false; - NSArray *supportedPixelFormats = m_videoDataOutput.availableVideoCVPixelFormatTypes; - for (NSNumber *currentPixelFormat in supportedPixelFormats) - { - if ([currentPixelFormat unsignedIntValue] == avPixelFormat) { - isSupported = true; - break; + auto bestScore = MinAVScore; + NSNumber *bestFormat = nullptr; + for (NSNumber *cvPixFmtNumber in m_videoDataOutput.availableVideoCVPixelFormatTypes) { + auto cvPixFmt = [cvPixFmtNumber unsignedIntValue]; + const auto pixFmt = QAVFHelpers::fromCVPixelFormat(cvPixFmt); + if (pixFmt == QVideoFrameFormat::Format_Invalid) + continue; + + auto score = DefaultAVScore; + if (cvPixFmt == inputCvPixFormat) + score += 100; + if (pixFmt == cameraPixelFormat) + score += 10; + // if (cvPixFmt == kCVPixelFormatType_32BGRA) + // score += 1; + + // This flag determines priorities of using ffmpeg hw frames or + // the exact camera format match. + // Maybe configure more, e.g. by some env var? + constexpr bool ShouldSuppressNotSupportedByFFmpeg = false; + + if (!isCVFormatSupported(cvPixFmt)) + score -= ShouldSuppressNotSupportedByFFmpeg ? 100000 : 5; + + // qDebug() << "----FMT:" << pixFmt << cvPixFmt << score; + + if (score > bestScore) { + bestScore = score; + bestFormat = cvPixFmtNumber; } } - if (isSupported) { - NSDictionary* outputSettings = @{ - (NSString *)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithUnsignedInt:avPixelFormat], - (NSString *)kCVPixelBufferMetalCompatibilityKey: @true - }; - m_videoDataOutput.videoSettings = outputSettings; - } else { - qWarning() << "QCamera::setCameraFormat: requested pixel format not supported. Did you use a camera format from another camera?"; + if (!bestFormat) { + qWarning() << "QCamera::setCameraFormat: availableVideoCVPixelFormatTypes empty"; + return 0; } - return avPixelFormat; + + if (bestScore < DefaultAVScore) + qWarning() << "QCamera::setCameraFormat: Cannot find hw FFmpeg supported cv pix format"; + + NSDictionary *outputSettings = @{ + (NSString *)kCVPixelBufferPixelFormatTypeKey : bestFormat, + (NSString *)kCVPixelBufferMetalCompatibilityKey : @true + }; + m_videoDataOutput.videoSettings = outputSettings; + + return [bestFormat unsignedIntValue]; +} + +QSize QAVFCamera::adjustedResolution() const +{ +#ifdef Q_OS_MACOS + return m_cameraFormat.resolution(); +#else + // Check, that we have matching dimesnions. + QSize resolution = m_cameraFormat.resolution(); + AVCaptureConnection *connection = [m_videoDataOutput connectionWithMediaType:AVMediaTypeVideo]; + if (!connection.supportsVideoOrientation) + return resolution; + + // Either portrait but actually sizes of landscape, or + // landscape with dimensions of portrait - not what + // sample delegate will report (it depends on videoOrientation set). + const bool isPortraitOrientation = connection.videoOrientation == AVCaptureVideoOrientationPortrait; + const bool isPortraitResolution = resolution.height() > resolution.width(); + if (isPortraitOrientation != isPortraitResolution) + resolution.transpose(); + + return resolution; +#endif // Q_OS_MACOS } void QAVFCamera::syncHandleFrame(const QVideoFrame &frame) @@ -444,6 +332,18 @@ void QAVFCamera::syncHandleFrame(const QVideoFrame &frame) Q_EMIT newVideoFrame(frame); } +std::optional<int> QAVFCamera::ffmpegHWPixelFormat() const +{ + return m_hwPixelFormat == AV_PIX_FMT_NONE ? std::optional<int>{} : m_hwPixelFormat; +} + +int QAVFCamera::cameraPixelFormatScore(QVideoFrameFormat::PixelFormat pixelFormat, + QVideoFrameFormat::ColorRange colorRange) const +{ + auto cvFormat = QAVFHelpers::toCVPixelFormat(pixelFormat, colorRange); + return static_cast<int>(isCVFormatSupported(cvFormat)); +} + QT_END_NAMESPACE #include "moc_qavfcamera_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qavfcamera_p.h b/src/plugins/multimedia/ffmpeg/qavfcamera_p.h index 1c0341496..0c88c520c 100644 --- a/src/plugins/multimedia/ffmpeg/qavfcamera_p.h +++ b/src/plugins/multimedia/ffmpeg/qavfcamera_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QAVFCAMERA_H #define QAVFCAMERA_H @@ -94,15 +58,18 @@ public: void deviceOrientationChanged(int angle = -1); - const void *ffmpegHWAccel() const override { return &hwAccel; } + std::optional<int> ffmpegHWPixelFormat() const override; + + int cameraPixelFormatScore(QVideoFrameFormat::PixelFormat pixelFmt, + QVideoFrameFormat::ColorRange colorRange) const override; private: - void requestCameraPermissionIfNeeded(); - void cameraAuthorizationChanged(bool authorized); + bool checkCameraPermission(); void updateCameraFormat(); void updateVideoInput(); void attachVideoInputDevice(); - uint setPixelFormat(const QVideoFrameFormat::PixelFormat pixelFormat); + uint32_t setPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat, uint32_t inputCvPixFormat); + QSize adjustedResolution() const; AVCaptureDevice *device() const; @@ -113,7 +80,7 @@ private: QAVFSampleBufferDelegate *m_sampleBufferDelegate = nullptr; dispatch_queue_t m_delegateQueue; QVideoOutputOrientationHandler m_orientationHandler; - QFFmpeg::HWAccel hwAccel; + AVPixelFormat m_hwPixelFormat = AV_PIX_FMT_NONE; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm new file mode 100644 index 000000000..ecdce8266 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate.mm @@ -0,0 +1,224 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qavfsamplebufferdelegate_p.h" + +#define AVMediaType XAVMediaType + +#include "qffmpeghwaccel_p.h" +#include "qavfhelpers_p.h" +#include "qffmpegvideobuffer_p.h" +#include "private/qvideoframe_p.h" + +#undef AVMediaType + +#include <optional> + +QT_USE_NAMESPACE + +static void releaseHwFrame(void * /*opaque*/, uint8_t *data) +{ + CVPixelBufferRelease(CVPixelBufferRef(data)); +} + +namespace { + +class CVImageVideoBuffer : public QAbstractVideoBuffer +{ +public: + CVImageVideoBuffer(CVImageBufferRef imageBuffer) : m_buffer(imageBuffer) + { + CVPixelBufferRetain(imageBuffer); + } + + ~CVImageVideoBuffer() + { + Q_ASSERT(m_mode == QtVideo::MapMode::NotMapped); + CVPixelBufferRelease(m_buffer); + } + + CVImageVideoBuffer::MapData map(QtVideo::MapMode mode) override + { + MapData mapData; + + if (m_mode == QtVideo::MapMode::NotMapped) { + CVPixelBufferLockBaseAddress( + m_buffer, mode == QtVideo::MapMode::ReadOnly ? kCVPixelBufferLock_ReadOnly : 0); + m_mode = mode; + } + + mapData.planeCount = CVPixelBufferGetPlaneCount(m_buffer); + Q_ASSERT(mapData.planeCount <= 3); + + if (!mapData.planeCount) { + // single plane + mapData.bytesPerLine[0] = CVPixelBufferGetBytesPerRow(m_buffer); + mapData.data[0] = static_cast<uchar *>(CVPixelBufferGetBaseAddress(m_buffer)); + mapData.dataSize[0] = CVPixelBufferGetDataSize(m_buffer); + mapData.planeCount = mapData.data[0] ? 1 : 0; + return mapData; + } + + // For a bi-planar or tri-planar format we have to set the parameters correctly: + for (int i = 0; i < mapData.planeCount; ++i) { + mapData.bytesPerLine[i] = CVPixelBufferGetBytesPerRowOfPlane(m_buffer, i); + mapData.dataSize[i] = mapData.bytesPerLine[i] * CVPixelBufferGetHeightOfPlane(m_buffer, i); + mapData.data[i] = static_cast<uchar *>(CVPixelBufferGetBaseAddressOfPlane(m_buffer, i)); + } + + return mapData; + } + + void unmap() override + { + if (m_mode != QtVideo::MapMode::NotMapped) { + CVPixelBufferUnlockBaseAddress( + m_buffer, m_mode == QtVideo::MapMode::ReadOnly ? kCVPixelBufferLock_ReadOnly : 0); + m_mode = QtVideo::MapMode::NotMapped; + } + } + + QVideoFrameFormat format() const override { return {}; } + +private: + CVImageBufferRef m_buffer; + QtVideo::MapMode m_mode = QtVideo::MapMode::NotMapped; +}; + +} + +// Make sure this is compatible with the layout used in ffmpeg's hwcontext_videotoolbox +static QFFmpeg::AVFrameUPtr allocHWFrame(AVBufferRef *hwContext, const CVPixelBufferRef &pixbuf) +{ + AVHWFramesContext *ctx = (AVHWFramesContext *)hwContext->data; + auto frame = QFFmpeg::makeAVFrame(); + frame->hw_frames_ctx = av_buffer_ref(hwContext); + frame->extended_data = frame->data; + + frame->buf[0] = av_buffer_create((uint8_t *)pixbuf, 1, releaseHwFrame, NULL, 0); + frame->data[3] = (uint8_t *)pixbuf; + CVPixelBufferRetain(pixbuf); + frame->width = ctx->width; + frame->height = ctx->height; + frame->format = AV_PIX_FMT_VIDEOTOOLBOX; + if (frame->width != (int)CVPixelBufferGetWidth(pixbuf) + || frame->height != (int)CVPixelBufferGetHeight(pixbuf)) { + + // This can happen while changing camera format + return nullptr; + } + return frame; +} + +@implementation QAVFSampleBufferDelegate { +@private + std::function<void(const QVideoFrame &)> frameHandler; + AVBufferRef *hwFramesContext; + std::unique_ptr<QFFmpeg::HWAccel> m_accel; + qint64 startTime; + std::optional<qint64> baseTime; + qreal frameRate; +} + +static QVideoFrame createHwVideoFrame(QAVFSampleBufferDelegate &delegate, + CVImageBufferRef imageBuffer, QVideoFrameFormat format) +{ + Q_ASSERT(delegate.baseTime); + + if (!delegate.m_accel) + return {}; + + auto avFrame = allocHWFrame(delegate.m_accel->hwFramesContextAsBuffer(), imageBuffer); + if (!avFrame) + return {}; + +#ifdef USE_SW_FRAMES + { + auto swFrame = QFFmpeg::makeAVFrame(); + /* retrieve data from GPU to CPU */ + const int ret = av_hwframe_transfer_data(swFrame.get(), avFrame.get(), 0); + if (ret < 0) { + qWarning() << "Error transferring the data to system memory:" << ret; + } else { + avFrame = std::move(swFrame); + } + } +#endif + + avFrame->pts = delegate.startTime - *delegate.baseTime; + + return QVideoFramePrivate::createFrame(std::make_unique<QFFmpegVideoBuffer>(std::move(avFrame)), + format); +} + +- (instancetype)initWithFrameHandler:(std::function<void(const QVideoFrame &)>)handler +{ + if (!(self = [super init])) + return nil; + + Q_ASSERT(handler); + + frameHandler = std::move(handler); + hwFramesContext = nullptr; + startTime = 0; + frameRate = 0.; + return self; +} + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection +{ + Q_UNUSED(connection); + Q_UNUSED(captureOutput); + + // NB: on iOS captureOutput/connection can be nil (when recording a video - + // avfmediaassetwriter). + + CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer); + + if (!imageBuffer) { + qWarning() << "Cannot get image buffer from sample buffer"; + return; + } + + const CMTime time = CMSampleBufferGetPresentationTimeStamp(sampleBuffer); + const qint64 frameTime = time.timescale ? time.value * 1000000 / time.timescale : 0; + if (!baseTime) { + baseTime = frameTime; + startTime = frameTime; + } + + QVideoFrameFormat format = QAVFHelpers::videoFormatForImageBuffer(imageBuffer); + if (!format.isValid()) { + qWarning() << "Cannot get get video format for image buffer" + << CVPixelBufferGetWidth(imageBuffer) << 'x' + << CVPixelBufferGetHeight(imageBuffer); + return; + } + + format.setStreamFrameRate(frameRate); + + auto frame = createHwVideoFrame(*self, imageBuffer, format); + if (!frame.isValid()) + frame = QVideoFramePrivate::createFrame(std::make_unique<CVImageVideoBuffer>(imageBuffer), + std::move(format)); + + frame.setStartTime(startTime - *baseTime); + frame.setEndTime(frameTime - *baseTime); + startTime = frameTime; + + frameHandler(frame); +} + +- (void)setHWAccel:(std::unique_ptr<QFFmpeg::HWAccel> &&)accel +{ + m_accel = std::move(accel); +} + +- (void)setVideoFormatFrameRate:(qreal)rate +{ + frameRate = rate; +} + +@end diff --git a/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h new file mode 100644 index 000000000..5a7e16c71 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qavfsamplebufferdelegate_p.h @@ -0,0 +1,51 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QAVFSAMPLEBUFFERDELEGATE_P_H +#define QAVFSAMPLEBUFFERDELEGATE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#import <AVFoundation/AVFoundation.h> +#import <CoreVideo/CoreVideo.h> + +#include <qtconfigmacros.h> +#include <qtypes.h> + +#include <memory> +#include <functional> + +QT_BEGIN_NAMESPACE + +class QAVSampleBufferDelegateFrameHandler; +class QVideoFrame; +namespace QFFmpeg { +class HWAccel; +} + +QT_END_NAMESPACE + +@interface QAVFSampleBufferDelegate : NSObject <AVCaptureVideoDataOutputSampleBufferDelegate> + +- (instancetype)initWithFrameHandler:(std::function<void(const QVideoFrame &)>)handler; + +- (void)captureOutput:(AVCaptureOutput *)captureOutput + didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer + fromConnection:(AVCaptureConnection *)connection; + +- (void)setHWAccel:(std::unique_ptr<QT_PREPEND_NAMESPACE(QFFmpeg::HWAccel)> &&)accel; + +- (void)setVideoFormatFrameRate:(qreal)frameRate; + +@end + +#endif diff --git a/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm b/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm new file mode 100644 index 000000000..715dea09c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qavfscreencapture.mm @@ -0,0 +1,201 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qavfscreencapture_p.h" +#include "qavfsamplebufferdelegate_p.h" +#include "qffmpegsurfacecapturegrabber_p.h" + +#include <qscreen.h> + +#define AVMediaType XAVMediaType +#include "qffmpeghwaccel_p.h" + +extern "C" { +#include <libavutil/hwcontext_videotoolbox.h> +#include <libavutil/hwcontext.h> +} +#undef AVMediaType + +#include <AppKit/NSScreen.h> + +#include <dispatch/dispatch.h> + +namespace { + +const auto DefaultCVPixelFormat = kCVPixelFormatType_32BGRA; + +CGDirectDisplayID findDisplayByName(const QString &name) +{ + for (NSScreen *screen in NSScreen.screens) { + if (name == QString::fromNSString(screen.localizedName)) + return [screen.deviceDescription[@"NSScreenNumber"] unsignedIntValue]; + } + return kCGNullDirectDisplay; +} +} + +QT_BEGIN_NAMESPACE + +QAVFScreenCapture::QAVFScreenCapture() : QPlatformSurfaceCapture(ScreenSource{}) +{ + CGRequestScreenCaptureAccess(); +} + +QAVFScreenCapture::~QAVFScreenCapture() +{ + resetCapture(); +} + +bool QAVFScreenCapture::setActiveInternal(bool active) +{ + if (active) { + if (!CGPreflightScreenCaptureAccess()) { + updateError(CaptureFailed, QLatin1String("Permissions denied")); + return false; + } + + auto screen = source<ScreenSource>(); + + if (!checkScreenWithError(screen)) + return false; + + return initScreenCapture(screen); + } else { + resetCapture(); + } + + return true; +} + +void QAVFScreenCapture::onNewFrame(const QVideoFrame &frame) +{ + // Since writing of the format is supposed to be only from one thread, + // the read-only comparison without a mutex is thread-safe + if (!m_format || m_format != frame.surfaceFormat()) { + QMutexLocker<QMutex> locker(&m_formatMutex); + + m_format = frame.surfaceFormat(); + + locker.unlock(); + + m_waitForFormat.notify_one(); + } + + emit newVideoFrame(frame); +} + +QVideoFrameFormat QAVFScreenCapture::frameFormat() const +{ + if (!m_grabber) + return {}; + + QMutexLocker<QMutex> locker(&m_formatMutex); + while (!m_format) + m_waitForFormat.wait(&m_formatMutex); + return *m_format; +} + +std::optional<int> QAVFScreenCapture::ffmpegHWPixelFormat() const +{ + return AV_PIX_FMT_VIDEOTOOLBOX; +} + +class QAVFScreenCapture::Grabber +{ +public: + Grabber(QAVFScreenCapture &capture, QScreen *screen, CGDirectDisplayID screenID, + std::unique_ptr<QFFmpeg::HWAccel> hwAccel) + { + m_captureSession = [[AVCaptureSession alloc] init]; + + m_sampleBufferDelegate = [[QAVFSampleBufferDelegate alloc] + initWithFrameHandler:[&capture](const QVideoFrame &frame) { + capture.onNewFrame(frame); + }]; + + m_videoDataOutput = [[AVCaptureVideoDataOutput alloc] init]; + + NSDictionary *videoSettings = [NSDictionary + dictionaryWithObjectsAndKeys:[NSNumber numberWithUnsignedInt:DefaultCVPixelFormat], + kCVPixelBufferPixelFormatTypeKey, nil]; + + [m_videoDataOutput setVideoSettings:videoSettings]; + [m_videoDataOutput setAlwaysDiscardsLateVideoFrames:true]; + + // Configure video output + m_dispatchQueue = dispatch_queue_create("vf_queue", nullptr); + [m_videoDataOutput setSampleBufferDelegate:m_sampleBufferDelegate queue:m_dispatchQueue]; + + [m_captureSession addOutput:m_videoDataOutput]; + + [m_sampleBufferDelegate setHWAccel:std::move(hwAccel)]; + + const auto frameRate = std::min(screen->refreshRate(), MaxScreenCaptureFrameRate); + [m_sampleBufferDelegate setVideoFormatFrameRate:frameRate]; + + m_screenInput = [[AVCaptureScreenInput alloc] initWithDisplayID:screenID]; + [m_screenInput setMinFrameDuration:CMTimeMake(1, static_cast<int32_t>(frameRate))]; + [m_captureSession addInput:m_screenInput]; + + [m_captureSession startRunning]; + } + + ~Grabber() + { + if (m_captureSession) + [m_captureSession stopRunning]; + + if (m_dispatchQueue) + dispatch_release(m_dispatchQueue); + + [m_sampleBufferDelegate release]; + [m_screenInput release]; + [m_videoDataOutput release]; + [m_captureSession release]; + } + +private: + AVCaptureSession *m_captureSession = nullptr; + AVCaptureScreenInput *m_screenInput = nullptr; + AVCaptureVideoDataOutput *m_videoDataOutput = nullptr; + QAVFSampleBufferDelegate *m_sampleBufferDelegate = nullptr; + dispatch_queue_t m_dispatchQueue = nullptr; +}; + +bool QAVFScreenCapture::initScreenCapture(QScreen *screen) +{ + const auto screenID = findDisplayByName(screen->name()); + + if (screenID == kCGNullDirectDisplay) { + updateError(InternalError, QLatin1String("Screen exists but couldn't been found by name")); + return false; + } + + auto hwAccel = QFFmpeg::HWAccel::create(AV_HWDEVICE_TYPE_VIDEOTOOLBOX); + + if (!hwAccel) { + updateError(CaptureFailed, QLatin1String("Couldn't create videotoolbox hw acceleration")); + return false; + } + + hwAccel->createFramesContext(av_map_videotoolbox_format_to_pixfmt(DefaultCVPixelFormat), + screen->size() * screen->devicePixelRatio()); + + if (!hwAccel->hwFramesContextAsBuffer()) { + updateError(CaptureFailed, QLatin1String("Couldn't create hw frames context")); + return false; + } + + m_grabber = std::make_unique<Grabber>(*this, screen, screenID, std::move(hwAccel)); + return true; +} + +void QAVFScreenCapture::resetCapture() +{ + m_grabber.reset(); + m_format = {}; +} + +QT_END_NAMESPACE + +#include "moc_qavfscreencapture_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h b/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h new file mode 100644 index 000000000..b95aabcf3 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qavfscreencapture_p.h @@ -0,0 +1,60 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QAVFSCREENCAPTURE_H +#define QAVFSCREENCAPTURE_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformsurfacecapture_p.h" +#include <qmutex.h> +#include <qwaitcondition.h> + +QT_BEGIN_NAMESPACE + +class QFFmpegVideoSink; + +class QAVFScreenCapture : public QPlatformSurfaceCapture +{ + Q_OBJECT + + class Grabber; + +public: + explicit QAVFScreenCapture(); + ~QAVFScreenCapture() override; + + QVideoFrameFormat frameFormat() const override; + + std::optional<int> ffmpegHWPixelFormat() const override; + +protected: + bool setActiveInternal(bool active) override; + +private: + void onNewFrame(const QVideoFrame &frame); + + bool initScreenCapture(QScreen *screen); + + void resetCapture(); + +private: + std::optional<QVideoFrameFormat> m_format; + mutable QMutex m_formatMutex; + mutable QWaitCondition m_waitForFormat; + + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // QAVFSCREENCAPTURE_H diff --git a/src/plugins/multimedia/ffmpeg/qcgcapturablewindows.mm b/src/plugins/multimedia/ffmpeg/qcgcapturablewindows.mm new file mode 100644 index 000000000..eb2035208 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qcgcapturablewindows.mm @@ -0,0 +1,48 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qcgcapturablewindows_p.h" +#include "private/qcapturablewindow_p.h" +#include "QtCore/private/qcore_mac_p.h" + +#include <AppKit/NSWindow.h> + +QT_BEGIN_NAMESPACE + +QList<QCapturableWindow> QCGCapturableWindows::windows() const +{ + QList<QCapturableWindow> result; + QCFType<CFArrayRef> windowList( + CGWindowListCopyWindowInfo(kCGWindowListOptionOnScreenOnly, kCGNullWindowID)); + + // Iterate through the window dictionaries + CFIndex count = CFArrayGetCount(windowList); + for (CFIndex i = 0; i < count; ++i) { + auto windowInfo = (CFDictionaryRef)CFArrayGetValueAtIndex(windowList, i); + auto windowNumber = (CFNumberRef)CFDictionaryGetValue(windowInfo, kCGWindowNumber); + auto windowName = (CFStringRef)CFDictionaryGetValue(windowInfo, kCGWindowName); + + CGWindowID windowId = 0; + static_assert(sizeof(windowId) == 4, + "CGWindowID size is not compatible with kCFNumberSInt32Type"); + CFNumberGetValue(windowNumber, kCFNumberSInt32Type, &windowId); + + auto windowData = std::make_unique<QCapturableWindowPrivate>(); + windowData->id = static_cast<QCapturableWindowPrivate::Id>(windowId); + if (windowName) + windowData->description = QString::fromCFString(windowName); + + result.push_back(windowData.release()->create()); + } + + return result; +} + +bool QCGCapturableWindows::isWindowValid(const QCapturableWindowPrivate &window) const +{ + QCFType<CFArrayRef> windowList( + CGWindowListCreate(kCGWindowListOptionIncludingWindow, window.id)); + return CFArrayGetCount(windowList) > 0; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qcgcapturablewindows_p.h b/src/plugins/multimedia/ffmpeg/qcgcapturablewindows_p.h new file mode 100644 index 000000000..6f779ae0d --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qcgcapturablewindows_p.h @@ -0,0 +1,32 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QCGCAPTURABLEWINDOWS_P_H +#define QCGCAPTURABLEWINDOWS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformcapturablewindows_p.h" + +QT_BEGIN_NAMESPACE + +class QCGCapturableWindows : public QPlatformCapturableWindows +{ +public: + QList<QCapturableWindow> windows() const override; + + bool isWindowValid(const QCapturableWindowPrivate &window) const override; +}; + +QT_END_NAMESPACE + +#endif // QCGCAPTURABLEWINDOWS_P_H diff --git a/src/plugins/multimedia/ffmpeg/qcgwindowcapture.mm b/src/plugins/multimedia/ffmpeg/qcgwindowcapture.mm new file mode 100644 index 000000000..a671fcdd6 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qcgwindowcapture.mm @@ -0,0 +1,197 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qabstractvideobuffer.h" + +#include "qcgwindowcapture_p.h" +#include "private/qcapturablewindow_p.h" +#include "qffmpegsurfacecapturegrabber_p.h" +#include "private/qvideoframe_p.h" + +#include "qscreen.h" +#include "qguiapplication.h" +#include <qmutex.h> +#include <qwaitcondition.h> + +#include <ApplicationServices/ApplicationServices.h> +#include <IOKit/graphics/IOGraphicsLib.h> +#include <AppKit/NSScreen.h> +#include <AppKit/NSApplication.h> +#include <AppKit/NSWindow.h> + +namespace { + +std::optional<qreal> frameRateForWindow(CGWindowID /*wid*/) +{ + // TODO: detect the frame rate + // if (window && window.screen) { + // CGDirectDisplayID displayID = [window.screen.deviceDescription[@"NSScreenNumber"] + // unsignedIntValue]; const auto displayRefreshRate = + // CGDisplayModeGetRefreshRate(CGDisplayCopyDisplayMode(displayID)); if (displayRefreshRate + // > 0 && displayRefreshRate < frameRate) frameRate = displayRefreshRate; + // } + + return {}; +} + +} + +QT_BEGIN_NAMESPACE + +class QCGImageVideoBuffer : public QAbstractVideoBuffer +{ +public: + QCGImageVideoBuffer(CGImageRef image) + { + auto provider = CGImageGetDataProvider(image); + m_data = CGDataProviderCopyData(provider); + m_bytesPerLine = CGImageGetBytesPerRow(image); + } + + ~QCGImageVideoBuffer() override { CFRelease(m_data); } + + MapData map(QtVideo::MapMode /*mode*/) override + { + MapData mapData; + + mapData.planeCount = 1; + mapData.bytesPerLine[0] = static_cast<int>(m_bytesPerLine); + mapData.data[0] = (uchar *)CFDataGetBytePtr(m_data); + mapData.dataSize[0] = static_cast<int>(CFDataGetLength(m_data)); + + return mapData; + } + + QVideoFrameFormat format() const override { return {}; } + +private: + CFDataRef m_data; + size_t m_bytesPerLine = 0; +}; + +class QCGWindowCapture::Grabber : public QFFmpegSurfaceCaptureGrabber +{ +public: + Grabber(QCGWindowCapture &capture, CGWindowID wid) : m_capture(capture), m_wid(wid) + { + addFrameCallback(*this, &Grabber::onNewFrame); + connect(this, &Grabber::errorUpdated, &capture, &QCGWindowCapture::updateError); + + if (auto screen = QGuiApplication::primaryScreen()) + setFrameRate(screen->refreshRate()); + + start(); + } + + ~Grabber() override { stop(); } + + QVideoFrameFormat frameFormat() const + { + QMutexLocker<QMutex> locker(&m_formatMutex); + while (!m_format) + m_waitForFormat.wait(&m_formatMutex); + return *m_format; + } + +protected: + QVideoFrame grabFrame() override + { + if (auto rate = frameRateForWindow(m_wid)) + setFrameRate(*rate); + + auto imageRef = CGWindowListCreateImage(CGRectNull, kCGWindowListOptionIncludingWindow, + m_wid, kCGWindowImageBoundsIgnoreFraming); + if (!imageRef) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot create image by window")); + return {}; + } + + auto imageDeleter = qScopeGuard([imageRef]() { CGImageRelease(imageRef); }); + + if (CGImageGetBitsPerPixel(imageRef) != 32 + || CGImageGetPixelFormatInfo(imageRef) != kCGImagePixelFormatPacked + || CGImageGetByteOrderInfo(imageRef) != kCGImageByteOrder32Little) { + qWarning() << "Unexpected image format. PixelFormatInfo:" + << CGImageGetPixelFormatInfo(imageRef) + << "BitsPerPixel:" << CGImageGetBitsPerPixel(imageRef) << "AlphaInfo" + << CGImageGetAlphaInfo(imageRef) + << "ByteOrderInfo:" << CGImageGetByteOrderInfo(imageRef); + + updateError(QPlatformSurfaceCapture::CapturingNotSupported, + QLatin1String("Not supported pixel format")); + return {}; + } + + QVideoFrameFormat format(QSize(CGImageGetWidth(imageRef), CGImageGetHeight(imageRef)), + QVideoFrameFormat::Format_BGRA8888); + format.setStreamFrameRate(frameRate()); + + return QVideoFramePrivate::createFrame(std::make_unique<QCGImageVideoBuffer>(imageRef), + std::move(format)); + } + + void onNewFrame(QVideoFrame frame) + { + // Since writing of the format is supposed to be only from one thread, + // the read-only comparison without a mutex is thread-safe + if (!m_format || m_format != frame.surfaceFormat()) { + QMutexLocker<QMutex> locker(&m_formatMutex); + + m_format = frame.surfaceFormat(); + + locker.unlock(); + + m_waitForFormat.notify_one(); + } + + emit m_capture.newVideoFrame(frame); + } + +private: + QCGWindowCapture &m_capture; + std::optional<QVideoFrameFormat> m_format; + mutable QMutex m_formatMutex; + mutable QWaitCondition m_waitForFormat; + CGWindowID m_wid; +}; + +QCGWindowCapture::QCGWindowCapture() : QPlatformSurfaceCapture(WindowSource{}) +{ + CGRequestScreenCaptureAccess(); +} + +QCGWindowCapture::~QCGWindowCapture() = default; + +bool QCGWindowCapture::setActiveInternal(bool active) +{ + if (active) { + if (!CGPreflightScreenCaptureAccess()) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Permissions denied")); + return false; + } + + auto window = source<WindowSource>(); + + auto handle = QCapturableWindowPrivate::handle(window); + if (!handle || !handle->id) + updateError(QPlatformSurfaceCapture::NotFound, QLatin1String("Invalid window")); + else + m_grabber = std::make_unique<Grabber>(*this, handle->id); + + } else { + m_grabber.reset(); + } + + return active == static_cast<bool>(m_grabber); +} + +QVideoFrameFormat QCGWindowCapture::frameFormat() const +{ + return m_grabber ? m_grabber->frameFormat() : QVideoFrameFormat(); +} + +QT_END_NAMESPACE + +#include "moc_qcgwindowcapture_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qcgwindowcapture_p.h b/src/plugins/multimedia/ffmpeg/qcgwindowcapture_p.h new file mode 100644 index 000000000..796c01ab3 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qcgwindowcapture_p.h @@ -0,0 +1,43 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QCGWINDOWCAPTURE_H +#define QCGWINDOWCAPTURE_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformsurfacecapture_p.h" + +QT_BEGIN_NAMESPACE + +class QCGWindowCapture : public QPlatformSurfaceCapture +{ + Q_OBJECT + + class Grabber; + +public: + explicit QCGWindowCapture(); + ~QCGWindowCapture() override; + + QVideoFrameFormat frameFormat() const override; + +protected: + bool setActiveInternal(bool active) override; + +private: + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // QCGWINDOWCAPTURE_H diff --git a/src/plugins/multimedia/ffmpeg/qeglfsscreencapture.cpp b/src/plugins/multimedia/ffmpeg/qeglfsscreencapture.cpp new file mode 100644 index 000000000..871cafd4f --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qeglfsscreencapture.cpp @@ -0,0 +1,180 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qeglfsscreencapture_p.h" + +#include "qffmpegsurfacecapturegrabber_p.h" +#include "qguiapplication.h" +#include "qopenglvideobuffer_p.h" +#include "private/qimagevideobuffer_p.h" +#include "private/qvideoframe_p.h" + +#include <QtOpenGL/private/qopenglcompositor_p.h> +#include <QtOpenGL/private/qopenglframebufferobject_p.h> + +#include <QtQuick/qquickwindow.h> + +QT_BEGIN_NAMESPACE + +class QEglfsScreenCapture::Grabber : public QFFmpegSurfaceCaptureGrabber +{ +public: + Grabber(QEglfsScreenCapture &screenCapture, QScreen *screen) + : QFFmpegSurfaceCaptureGrabber(QFFmpegSurfaceCaptureGrabber::UseCurrentThread) + { + addFrameCallback(screenCapture, &QEglfsScreenCapture::newVideoFrame); + connect(this, &Grabber::errorUpdated, &screenCapture, &QEglfsScreenCapture::updateError); + // Limit frame rate to 30 fps for performance reasons, + // to be reviewed at the next optimization round + setFrameRate(std::min(screen->refreshRate(), 30.0)); + } + + ~Grabber() override { stop(); } + + QVideoFrameFormat format() { return m_format; } + +protected: + QVideoFrame grabFrame() override + { + auto nativeSize = QOpenGLCompositor::instance()->nativeTargetGeometry().size(); + auto fbo = std::make_unique<QOpenGLFramebufferObject>(nativeSize); + + if (!QOpenGLCompositor::instance()->grabToFrameBufferObject( + fbo.get(), QOpenGLCompositor::NotFlipped)) { + updateError(Error::InternalError, QLatin1String("Couldn't grab to framebuffer object")); + return {}; + } + + if (!fbo->isValid()) { + updateError(Error::InternalError, QLatin1String("Framebuffer object invalid")); + return {}; + } + + auto videoBuffer = std::make_unique<QOpenGLVideoBuffer>(std::move(fbo)); + + if (!m_format.isValid()) { + auto image = videoBuffer->ensureImageBuffer().underlyingImage(); + m_format = { image.size(), QVideoFrameFormat::pixelFormatFromImageFormat(image.format()) }; + m_format.setStreamFrameRate(frameRate()); + } + + return QVideoFramePrivate::createFrame(std::move(videoBuffer), m_format); + } + + QVideoFrameFormat m_format; +}; + +class QEglfsScreenCapture::QuickGrabber : public Grabber +{ +public: + QuickGrabber(QEglfsScreenCapture &screenCapture, QScreen *screen, QQuickWindow *quickWindow) + : Grabber(screenCapture, screen), m_quickWindow(quickWindow) + { + Q_ASSERT(m_quickWindow); + } + +protected: + QVideoFrame grabFrame() override + { + if (!m_quickWindow) { + updateError(Error::InternalError, QLatin1String("Window deleted")); + return {}; + } + + QImage image = m_quickWindow->grabWindow(); + + if (image.isNull()) { + updateError(Error::InternalError, QLatin1String("Image invalid")); + return {}; + } + + if (!m_format.isValid()) { + m_format = { image.size(), + QVideoFrameFormat::pixelFormatFromImageFormat(image.format()) }; + m_format.setStreamFrameRate(frameRate()); + } + + return QVideoFramePrivate::createFrame( + std::make_unique<QImageVideoBuffer>(std::move(image)), m_format); + } + +private: + QPointer<QQuickWindow> m_quickWindow; +}; + +QEglfsScreenCapture::QEglfsScreenCapture() : QPlatformSurfaceCapture(ScreenSource{}) { } + +QEglfsScreenCapture::~QEglfsScreenCapture() = default; + +QVideoFrameFormat QEglfsScreenCapture::frameFormat() const +{ + return m_grabber ? m_grabber->format() : QVideoFrameFormat(); +} + +bool QEglfsScreenCapture::setActiveInternal(bool active) +{ + if (static_cast<bool>(m_grabber) == active) + return true; + + if (m_grabber) + m_grabber.reset(); + + if (!active) + return true; + + m_grabber = createGrabber(); + + if (!m_grabber) { + // TODO: This could mean that the UI is not started yet, so we should wait and try again, + // and then give error if still not started. Might not be possible here. + return false; + } + + m_grabber->start(); + return true; +} + +bool QEglfsScreenCapture::isSupported() +{ + return QGuiApplication::platformName() == QLatin1String("eglfs"); +} + +std::unique_ptr<QEglfsScreenCapture::Grabber> QEglfsScreenCapture::createGrabber() +{ + auto screen = source<ScreenSource>(); + if (!checkScreenWithError(screen)) + return nullptr; + + QOpenGLCompositor *compositor = QOpenGLCompositor::instance(); + + if (compositor->context()) { + // Create OpenGL grabber + if (!compositor->targetWindow()) { + updateError(Error::CaptureFailed, + QLatin1String("Target window is not set for OpenGL compositor")); + return nullptr; + } + + return std::make_unique<Grabber>(*this, screen); + } + + // Check for QQuickWindow + auto windows = QGuiApplication::topLevelWindows(); + auto it = std::find_if(windows.begin(), windows.end(), [screen](QWindow *window) { + auto quickWindow = qobject_cast<QQuickWindow *>(window); + if (!quickWindow) + return false; + + return quickWindow->screen() == screen; + }); + + if (it != windows.end()) { + // Create grabber that calls QQuickWindow::grabWindow + return std::make_unique<QuickGrabber>(*this, screen, qobject_cast<QQuickWindow *>(*it)); + } + + updateError(Error::CaptureFailed, QLatin1String("No existing OpenGL context or QQuickWindow")); + return nullptr; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qeglfsscreencapture_p.h b/src/plugins/multimedia/ffmpeg/qeglfsscreencapture_p.h new file mode 100644 index 000000000..840cdbeb0 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qeglfsscreencapture_p.h @@ -0,0 +1,48 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QEGLFSSCREENCAPTURE_H +#define QEGLFSSCREENCAPTURE_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformsurfacecapture_p.h> +#include <memory> + +QT_BEGIN_NAMESPACE + +class QEglfsScreenCapture : public QPlatformSurfaceCapture +{ +public: + QEglfsScreenCapture(); + + ~QEglfsScreenCapture() override; + + QVideoFrameFormat frameFormat() const override; + + static bool isSupported(); + +private: + bool setActiveInternal(bool active) override; + +private: + class Grabber; + class QuickGrabber; + + std::unique_ptr<Grabber> createGrabber(); + + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // QEGLFSSCREENCAPTURE_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpeg.cpp b/src/plugins/multimedia/ffmpeg/qffmpeg.cpp new file mode 100644 index 000000000..8969c6132 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpeg.cpp @@ -0,0 +1,748 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpeg_p.h" + +#include <qdebug.h> +#include <qloggingcategory.h> +#include <qffmpeghwaccel_p.h> // TODO: probably decompose HWAccel and get rid of the header in the base utils + +#include <algorithm> +#include <vector> +#include <array> +#include <optional> +#include <unordered_set> + +extern "C" { +#include <libavutil/pixdesc.h> +#include <libavutil/samplefmt.h> + +#ifdef Q_OS_DARWIN +#include <libavutil/hwcontext_videotoolbox.h> +#endif +} + +#ifdef Q_OS_ANDROID +#include <QtCore/qjniobject.h> +#include <QtCore/qjniarray.h> +#include <QtCore/qjnitypes.h> +#endif + +QT_BEGIN_NAMESPACE + +#ifdef Q_OS_ANDROID +Q_DECLARE_JNI_CLASS(QtVideoDeviceManager, + "org/qtproject/qt/android/multimedia/QtVideoDeviceManager"); + +# ifndef QT_DECLARE_JNI_CLASS_STANDARD_TYPES +Q_DECLARE_JNI_CLASS(String, "java/lang/String"); +# endif +#endif // Q_OS_ANDROID + +Q_STATIC_LOGGING_CATEGORY(qLcFFmpegUtils, "qt.multimedia.ffmpeg.utils"); + +namespace QFFmpeg { + +namespace { + +enum CodecStorageType { + ENCODERS, + DECODERS, + + // TODO: maybe split sw/hw codecs + + CODEC_STORAGE_TYPE_COUNT +}; + +using CodecsStorage = std::vector<const AVCodec *>; + +struct CodecsComparator +{ + bool operator()(const AVCodec *a, const AVCodec *b) const + { + return a->id < b->id + || (a->id == b->id && isAVCodecExperimental(a) < isAVCodecExperimental(b)); + } + + bool operator()(const AVCodec *a, AVCodecID id) const { return a->id < id; } +}; + +template<typename FlagNames> +QString flagsToString(int flags, const FlagNames &flagNames) +{ + QString result; + int leftover = flags; + for (const auto &flagAndName : flagNames) + if ((flags & flagAndName.first) != 0) { + leftover &= ~flagAndName.first; + if (!result.isEmpty()) + result += ", "; + result += flagAndName.second; + } + + if (leftover) { + if (!result.isEmpty()) + result += ", "; + result += QString::number(leftover, 16); + } + return result; +} + +void dumpCodecInfo(const AVCodec *codec) +{ + using FlagNames = std::initializer_list<std::pair<int, const char *>>; + const auto mediaType = codec->type == AVMEDIA_TYPE_VIDEO ? "video" + : codec->type == AVMEDIA_TYPE_AUDIO ? "audio" + : codec->type == AVMEDIA_TYPE_SUBTITLE ? "subtitle" + : "other_type"; + + const auto type = av_codec_is_encoder(codec) + ? av_codec_is_decoder(codec) ? "encoder/decoder:" : "encoder:" + : "decoder:"; + + static const FlagNames capabilitiesNames = { + { AV_CODEC_CAP_DRAW_HORIZ_BAND, "DRAW_HORIZ_BAND" }, + { AV_CODEC_CAP_DR1, "DRAW_HORIZ_DR1" }, + { AV_CODEC_CAP_DELAY, "DELAY" }, + { AV_CODEC_CAP_SMALL_LAST_FRAME, "SMALL_LAST_FRAME" }, + { AV_CODEC_CAP_SUBFRAMES, "SUBFRAMES" }, + { AV_CODEC_CAP_EXPERIMENTAL, "EXPERIMENTAL" }, + { AV_CODEC_CAP_CHANNEL_CONF, "CHANNEL_CONF" }, + { AV_CODEC_CAP_FRAME_THREADS, "FRAME_THREADS" }, + { AV_CODEC_CAP_SLICE_THREADS, "SLICE_THREADS" }, + { AV_CODEC_CAP_PARAM_CHANGE, "PARAM_CHANGE" }, +#ifdef AV_CODEC_CAP_OTHER_THREADS + { AV_CODEC_CAP_OTHER_THREADS, "OTHER_THREADS" }, +#endif + { AV_CODEC_CAP_VARIABLE_FRAME_SIZE, "VARIABLE_FRAME_SIZE" }, + { AV_CODEC_CAP_AVOID_PROBING, "AVOID_PROBING" }, + { AV_CODEC_CAP_HARDWARE, "HARDWARE" }, + { AV_CODEC_CAP_HYBRID, "HYBRID" }, + { AV_CODEC_CAP_ENCODER_REORDERED_OPAQUE, "ENCODER_REORDERED_OPAQUE" }, +#ifdef AV_CODEC_CAP_ENCODER_FLUSH + { AV_CODEC_CAP_ENCODER_FLUSH, "ENCODER_FLUSH" }, +#endif + }; + + qCDebug(qLcFFmpegUtils) << mediaType << type << codec->name << "id:" << codec->id + << "capabilities:" + << flagsToString(codec->capabilities, capabilitiesNames); + + if (codec->pix_fmts) { + static const FlagNames flagNames = { + { AV_PIX_FMT_FLAG_BE, "BE" }, + { AV_PIX_FMT_FLAG_PAL, "PAL" }, + { AV_PIX_FMT_FLAG_BITSTREAM, "BITSTREAM" }, + { AV_PIX_FMT_FLAG_HWACCEL, "HWACCEL" }, + { AV_PIX_FMT_FLAG_PLANAR, "PLANAR" }, + { AV_PIX_FMT_FLAG_RGB, "RGB" }, + { AV_PIX_FMT_FLAG_ALPHA, "ALPHA" }, + { AV_PIX_FMT_FLAG_BAYER, "BAYER" }, + { AV_PIX_FMT_FLAG_FLOAT, "FLOAT" }, + }; + + qCDebug(qLcFFmpegUtils) << " pix_fmts:"; + for (auto f = codec->pix_fmts; *f != AV_PIX_FMT_NONE; ++f) { + auto desc = av_pix_fmt_desc_get(*f); + qCDebug(qLcFFmpegUtils) + << " id:" << *f << desc->name << "depth:" << desc->comp[0].depth + << "flags:" << flagsToString(desc->flags, flagNames); + } + } else if (codec->type == AVMEDIA_TYPE_VIDEO) { + qCDebug(qLcFFmpegUtils) << " pix_fmts: null"; + } + + if (codec->sample_fmts) { + qCDebug(qLcFFmpegUtils) << " sample_fmts:"; + for (auto f = codec->sample_fmts; *f != AV_SAMPLE_FMT_NONE; ++f) { + const auto name = av_get_sample_fmt_name(*f); + qCDebug(qLcFFmpegUtils) << " id:" << *f << (name ? name : "unknown") + << "bytes_per_sample:" << av_get_bytes_per_sample(*f) + << "is_planar:" << av_sample_fmt_is_planar(*f); + } + } else if (codec->type == AVMEDIA_TYPE_AUDIO) { + qCDebug(qLcFFmpegUtils) << " sample_fmts: null"; + } + + if (avcodec_get_hw_config(codec, 0)) { + static const FlagNames hwConfigMethodNames = { + { AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX, "HW_DEVICE_CTX" }, + { AV_CODEC_HW_CONFIG_METHOD_HW_FRAMES_CTX, "HW_FRAMES_CTX" }, + { AV_CODEC_HW_CONFIG_METHOD_INTERNAL, "INTERNAL" }, + { AV_CODEC_HW_CONFIG_METHOD_AD_HOC, "AD_HOC" } + }; + + qCDebug(qLcFFmpegUtils) << " hw config:"; + for (int index = 0; auto config = avcodec_get_hw_config(codec, index); ++index) { + const auto pixFmtForDevice = pixelFormatForHwDevice(config->device_type); + auto pixFmtDesc = av_pix_fmt_desc_get(config->pix_fmt); + auto pixFmtForDeviceDesc = av_pix_fmt_desc_get(pixFmtForDevice); + qCDebug(qLcFFmpegUtils) + << " device_type:" << config->device_type << "pix_fmt:" << config->pix_fmt + << (pixFmtDesc ? pixFmtDesc->name : "unknown") + << "pixelFormatForHwDevice:" << pixelFormatForHwDevice(config->device_type) + << (pixFmtForDeviceDesc ? pixFmtForDeviceDesc->name : "unknown") + << "hw_config_methods:" << flagsToString(config->methods, hwConfigMethodNames); + } + } +} + +bool isCodecValid(const AVCodec *codec, const std::vector<AVHWDeviceType> &availableHwDeviceTypes, + const std::optional<std::unordered_set<AVCodecID>> &codecAvailableOnDevice) +{ + if (codec->type != AVMEDIA_TYPE_VIDEO) + return true; + + if (!codec->pix_fmts) { +#if defined(Q_OS_LINUX) || defined(Q_OS_ANDROID) + // Disable V4L2 M2M codecs for encoding for now, + // TODO: Investigate on how to get them working + if (std::strstr(codec->name, "_v4l2m2m") && av_codec_is_encoder(codec)) + return false; + + // MediaCodec in Android is used for hardware-accelerated media processing. That is why + // before marking it as valid, we need to make sure if it is available on current device. + if (std::strstr(codec->name, "_mediacodec") + && (codec->capabilities & AV_CODEC_CAP_HARDWARE) + && codecAvailableOnDevice && codecAvailableOnDevice->count(codec->id) == 0) + return false; +#endif + + return true; // To be investigated. This happens for RAW_VIDEO, that is supposed to be OK, + // and with v4l2m2m codecs, that is suspicious. + } + + if (findAVPixelFormat(codec, &isHwPixelFormat) == AV_PIX_FMT_NONE) + return true; + + if ((codec->capabilities & AV_CODEC_CAP_HARDWARE) == 0) + return true; + + auto checkDeviceType = [codec](AVHWDeviceType type) { + return isAVFormatSupported(codec, pixelFormatForHwDevice(type)); + }; + + if (codecAvailableOnDevice && codecAvailableOnDevice->count(codec->id) == 0) + return false; + + return std::any_of(availableHwDeviceTypes.begin(), availableHwDeviceTypes.end(), + checkDeviceType); +} + +std::optional<std::unordered_set<AVCodecID>> availableHWCodecs(const CodecStorageType type) +{ +#ifdef Q_OS_ANDROID + using namespace Qt::StringLiterals; + using namespace QtJniTypes; + std::unordered_set<AVCodecID> availabeCodecs; + + auto getCodecId = [] (const QString& codecName) { + if (codecName == "3gpp"_L1) return AV_CODEC_ID_H263; + if (codecName == "avc"_L1) return AV_CODEC_ID_H264; + if (codecName == "hevc"_L1) return AV_CODEC_ID_HEVC; + if (codecName == "mp4v-es"_L1) return AV_CODEC_ID_MPEG4; + if (codecName == "x-vnd.on2.vp8"_L1) return AV_CODEC_ID_VP8; + if (codecName == "x-vnd.on2.vp9"_L1) return AV_CODEC_ID_VP9; + return AV_CODEC_ID_NONE; + }; + + const QJniArray jniCodecs = QtVideoDeviceManager::callStaticMethod<String[]>( + type == ENCODERS ? "getHWVideoEncoders" : "getHWVideoDecoders"); + + for (const auto &codec : jniCodecs) + availabeCodecs.insert(getCodecId(codec.toString())); + return availabeCodecs; +#else + Q_UNUSED(type); + return {}; +#endif +} + +const CodecsStorage &codecsStorage(CodecStorageType codecsType) +{ + static const auto &storages = []() { + std::array<CodecsStorage, CODEC_STORAGE_TYPE_COUNT> result; + void *opaque = nullptr; + const auto platformHwEncoders = availableHWCodecs(ENCODERS); + const auto platformHwDecoders = availableHWCodecs(DECODERS); + + while (auto codec = av_codec_iterate(&opaque)) { + // TODO: to be investigated + // FFmpeg functions avcodec_find_decoder/avcodec_find_encoder + // find experimental codecs in the last order, + // now we don't consider them at all since they are supposed to + // be not stable, maybe we shouldn't. + // Currently, it's possible to turn them on for testing purposes. + + static const auto experimentalCodecsEnabled = + qEnvironmentVariableIntValue("QT_ENABLE_EXPERIMENTAL_CODECS"); + + if (!experimentalCodecsEnabled && isAVCodecExperimental(codec)) { + qCDebug(qLcFFmpegUtils) << "Skip experimental codec" << codec->name; + continue; + } + + if (av_codec_is_decoder(codec)) { + if (isCodecValid(codec, HWAccel::decodingDeviceTypes(), platformHwDecoders)) + result[DECODERS].emplace_back(codec); + else + qCDebug(qLcFFmpegUtils) + << "Skip decoder" << codec->name + << "due to disabled matching hw acceleration, or dysfunctional codec"; + } + + if (av_codec_is_encoder(codec)) { + if (isCodecValid(codec, HWAccel::encodingDeviceTypes(), platformHwEncoders)) + result[ENCODERS].emplace_back(codec); + else + qCDebug(qLcFFmpegUtils) + << "Skip encoder" << codec->name + << "due to disabled matching hw acceleration, or dysfunctional codec"; + } + } + + for (auto &storage : result) { + storage.shrink_to_fit(); + + // we should ensure the original order + std::stable_sort(storage.begin(), storage.end(), CodecsComparator{}); + } + + // It print pretty much logs, so let's print it only for special case + const bool shouldDumpCodecsInfo = qLcFFmpegUtils().isEnabled(QtDebugMsg) + && qEnvironmentVariableIsSet("QT_FFMPEG_DEBUG"); + + if (shouldDumpCodecsInfo) { + qCDebug(qLcFFmpegUtils) << "Advanced FFmpeg codecs info:"; + for (auto &storage : result) { + std::for_each(storage.begin(), storage.end(), &dumpCodecInfo); + qCDebug(qLcFFmpegUtils) << "---------------------------"; + } + } + + return result; + }(); + + return storages[codecsType]; +} + +const char *preferredHwCodecNameSuffix(bool isEncoder, AVHWDeviceType deviceType) +{ + switch (deviceType) { + case AV_HWDEVICE_TYPE_VAAPI: + return "_vaapi"; + case AV_HWDEVICE_TYPE_MEDIACODEC: + return "_mediacodec"; + case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: + return "_videotoolbox"; + case AV_HWDEVICE_TYPE_D3D11VA: + case AV_HWDEVICE_TYPE_DXVA2: +#if QT_FFMPEG_HAS_D3D12VA + case AV_HWDEVICE_TYPE_D3D12VA: +#endif + return "_mf"; + case AV_HWDEVICE_TYPE_CUDA: + case AV_HWDEVICE_TYPE_VDPAU: + return isEncoder ? "_nvenc" : "_cuvid"; + default: + return nullptr; + } +} + +template<typename CodecScoreGetter> +const AVCodec *findAVCodec(CodecStorageType codecsType, AVCodecID codecId, + const CodecScoreGetter &scoreGetter) +{ + const auto &storage = codecsStorage(codecsType); + auto it = std::lower_bound(storage.begin(), storage.end(), codecId, CodecsComparator{}); + + const AVCodec *result = nullptr; + AVScore resultScore = NotSuitableAVScore; + + for (; it != storage.end() && (*it)->id == codecId && resultScore != BestAVScore; ++it) { + const auto score = scoreGetter(*it); + + if (score > resultScore) { + resultScore = score; + result = *it; + } + } + + return result; +} + +AVScore hwCodecNameScores(const AVCodec *codec, AVHWDeviceType deviceType) +{ + if (auto suffix = preferredHwCodecNameSuffix(av_codec_is_encoder(codec), deviceType)) { + const auto substr = strstr(codec->name, suffix); + if (substr && !substr[strlen(suffix)]) + return BestAVScore; + + return DefaultAVScore; + } + + return BestAVScore; +} + +const AVCodec *findAVCodec(CodecStorageType codecsType, AVCodecID codecId, + const std::optional<AVHWDeviceType> &deviceType, + const std::optional<PixelOrSampleFormat> &format) +{ + // TODO: remove deviceType and use only isAVFormatSupported to check the format + + return findAVCodec(codecsType, codecId, [&](const AVCodec *codec) { + if (format && !isAVFormatSupported(codec, *format)) + return NotSuitableAVScore; + + if (!deviceType) + return BestAVScore; // find any codec with the id + + if (*deviceType == AV_HWDEVICE_TYPE_NONE + && findAVFormat(codec->pix_fmts, &isSwPixelFormat) != AV_PIX_FMT_NONE) + return BestAVScore; + + if (*deviceType != AV_HWDEVICE_TYPE_NONE) { + for (int index = 0; auto config = avcodec_get_hw_config(codec, index); ++index) { + if (config->device_type != deviceType) + continue; + + if (format && config->pix_fmt != AV_PIX_FMT_NONE && config->pix_fmt != *format) + continue; + + return hwCodecNameScores(codec, *deviceType); + } + + // The situation happens mostly with encoders + // Probably, it's ffmpeg bug: avcodec_get_hw_config returns null even though + // hw acceleration is supported + // To be removed: only isAVFormatSupported should be used. + if (hasAVFormat(codec->pix_fmts, pixelFormatForHwDevice(*deviceType))) + return hwCodecNameScores(codec, *deviceType); + } + + return NotSuitableAVScore; + }); +} + +} // namespace + +const AVCodec *findAVDecoder(AVCodecID codecId, const std::optional<AVHWDeviceType> &deviceType, + const std::optional<PixelOrSampleFormat> &format) +{ + return findAVCodec(DECODERS, codecId, deviceType, format); +} + +const AVCodec *findAVEncoder(AVCodecID codecId, const std::optional<AVHWDeviceType> &deviceType, + const std::optional<PixelOrSampleFormat> &format) +{ + return findAVCodec(ENCODERS, codecId, deviceType, format); +} + +const AVCodec *findAVEncoder(AVCodecID codecId, + const std::function<AVScore(const AVCodec *)> &scoresGetter) +{ + return findAVCodec(ENCODERS, codecId, scoresGetter); +} + +bool isAVFormatSupported(const AVCodec *codec, PixelOrSampleFormat format) +{ + if (codec->type == AVMEDIA_TYPE_VIDEO) { + auto checkFormat = [format](AVPixelFormat f) { return f == format; }; + return findAVPixelFormat(codec, checkFormat) != AV_PIX_FMT_NONE; + } + + if (codec->type == AVMEDIA_TYPE_AUDIO) + return hasAVFormat(codec->sample_fmts, AVSampleFormat(format)); + + return false; +} + +bool isHwPixelFormat(AVPixelFormat format) +{ + const auto desc = av_pix_fmt_desc_get(format); + return desc && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) != 0; +} + +bool isAVCodecExperimental(const AVCodec *codec) +{ + return (codec->capabilities & AV_CODEC_CAP_EXPERIMENTAL) != 0; +} + +void applyExperimentalCodecOptions(const AVCodec *codec, AVDictionary** opts) +{ + if (isAVCodecExperimental(codec)) { + qCWarning(qLcFFmpegUtils) << "Applying the option 'strict -2' for the experimental codec" + << codec->name << ". it's unlikely to work properly"; + av_dict_set(opts, "strict", "-2", 0); + } +} + +AVPixelFormat pixelFormatForHwDevice(AVHWDeviceType deviceType) +{ + switch (deviceType) { + case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: + return AV_PIX_FMT_VIDEOTOOLBOX; + case AV_HWDEVICE_TYPE_VAAPI: + return AV_PIX_FMT_VAAPI; + case AV_HWDEVICE_TYPE_MEDIACODEC: + return AV_PIX_FMT_MEDIACODEC; + case AV_HWDEVICE_TYPE_CUDA: + return AV_PIX_FMT_CUDA; + case AV_HWDEVICE_TYPE_VDPAU: + return AV_PIX_FMT_VDPAU; + case AV_HWDEVICE_TYPE_OPENCL: + return AV_PIX_FMT_OPENCL; + case AV_HWDEVICE_TYPE_QSV: + return AV_PIX_FMT_QSV; + case AV_HWDEVICE_TYPE_D3D11VA: + return AV_PIX_FMT_D3D11; +#if QT_FFMPEG_HAS_D3D12VA + case AV_HWDEVICE_TYPE_D3D12VA: + return AV_PIX_FMT_D3D12; +#endif + case AV_HWDEVICE_TYPE_DXVA2: + return AV_PIX_FMT_DXVA2_VLD; + case AV_HWDEVICE_TYPE_DRM: + return AV_PIX_FMT_DRM_PRIME; +#if QT_FFMPEG_HAS_VULKAN + case AV_HWDEVICE_TYPE_VULKAN: + return AV_PIX_FMT_VULKAN; +#endif + default: + return AV_PIX_FMT_NONE; + } +} + +AVPacketSideData *addStreamSideData(AVStream *stream, AVPacketSideData sideData) +{ + QScopeGuard freeData([&sideData]() { av_free(sideData.data); }); +#if QT_FFMPEG_STREAM_SIDE_DATA_DEPRECATED + AVPacketSideData *result = av_packet_side_data_add( + &stream->codecpar->coded_side_data, + &stream->codecpar->nb_coded_side_data, + sideData.type, + sideData.data, + sideData.size, + 0); + if (result) { + // If the result is not null, the ownership is taken by AVStream, + // otherwise the data must be deleted. + freeData.dismiss(); + return result; + } +#else + Q_UNUSED(stream); + // TODO: implement for older FFmpeg versions + qWarning() << "Adding stream side data is not supported for FFmpeg < 6.1"; +#endif + + return nullptr; +} + +const AVPacketSideData *streamSideData(const AVStream *stream, AVPacketSideDataType type) +{ + Q_ASSERT(stream); + +#if QT_FFMPEG_STREAM_SIDE_DATA_DEPRECATED + return av_packet_side_data_get(stream->codecpar->coded_side_data, + stream->codecpar->nb_coded_side_data, type); +#else + auto checkType = [type](const auto &item) { return item.type == type; }; + const auto end = stream->side_data + stream->nb_side_data; + const auto found = std::find_if(stream->side_data, end, checkType); + return found == end ? nullptr : found; +#endif +} + +SwrContextUPtr createResampleContext(const AVAudioFormat &inputFormat, + const AVAudioFormat &outputFormat) +{ + SwrContext *resampler = nullptr; +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + resampler = swr_alloc_set_opts(nullptr, + outputFormat.channelLayoutMask, + outputFormat.sampleFormat, + outputFormat.sampleRate, + inputFormat.channelLayoutMask, + inputFormat.sampleFormat, + inputFormat.sampleRate, + 0, + nullptr); +#else + +#if QT_FFMPEG_SWR_CONST_CH_LAYOUT + using AVChannelLayoutPrm = const AVChannelLayout*; +#else + using AVChannelLayoutPrm = AVChannelLayout*; +#endif + + swr_alloc_set_opts2(&resampler, + const_cast<AVChannelLayoutPrm>(&outputFormat.channelLayout), + outputFormat.sampleFormat, + outputFormat.sampleRate, + const_cast<AVChannelLayoutPrm>(&inputFormat.channelLayout), + inputFormat.sampleFormat, + inputFormat.sampleRate, + 0, + nullptr); +#endif + + swr_init(resampler); + return SwrContextUPtr(resampler); +} + +QVideoFrameFormat::ColorTransfer fromAvColorTransfer(AVColorTransferCharacteristic colorTrc) { + switch (colorTrc) { + case AVCOL_TRC_BT709: + // The following three cases have transfer characteristics identical to BT709 + case AVCOL_TRC_BT1361_ECG: + case AVCOL_TRC_BT2020_10: + case AVCOL_TRC_BT2020_12: + case AVCOL_TRC_SMPTE240M: // almost identical to bt709 + return QVideoFrameFormat::ColorTransfer_BT709; + case AVCOL_TRC_GAMMA22: + case AVCOL_TRC_SMPTE428: // No idea, let's hope for the best... + case AVCOL_TRC_IEC61966_2_1: // sRGB, close enough to 2.2... + case AVCOL_TRC_IEC61966_2_4: // not quite, but probably close enough + return QVideoFrameFormat::ColorTransfer_Gamma22; + case AVCOL_TRC_GAMMA28: + return QVideoFrameFormat::ColorTransfer_Gamma28; + case AVCOL_TRC_SMPTE170M: + return QVideoFrameFormat::ColorTransfer_BT601; + case AVCOL_TRC_LINEAR: + return QVideoFrameFormat::ColorTransfer_Linear; + case AVCOL_TRC_SMPTE2084: + return QVideoFrameFormat::ColorTransfer_ST2084; + case AVCOL_TRC_ARIB_STD_B67: + return QVideoFrameFormat::ColorTransfer_STD_B67; + default: + break; + } + return QVideoFrameFormat::ColorTransfer_Unknown; +} + +AVColorTransferCharacteristic toAvColorTransfer(QVideoFrameFormat::ColorTransfer colorTrc) +{ + switch (colorTrc) { + case QVideoFrameFormat::ColorTransfer_BT709: + return AVCOL_TRC_BT709; + case QVideoFrameFormat::ColorTransfer_BT601: + return AVCOL_TRC_BT709; // which one is the best? + case QVideoFrameFormat::ColorTransfer_Linear: + return AVCOL_TRC_SMPTE2084; + case QVideoFrameFormat::ColorTransfer_Gamma22: + return AVCOL_TRC_GAMMA22; + case QVideoFrameFormat::ColorTransfer_Gamma28: + return AVCOL_TRC_GAMMA28; + case QVideoFrameFormat::ColorTransfer_ST2084: + return AVCOL_TRC_SMPTE2084; + case QVideoFrameFormat::ColorTransfer_STD_B67: + return AVCOL_TRC_ARIB_STD_B67; + default: + return AVCOL_TRC_UNSPECIFIED; + } +} + +QVideoFrameFormat::ColorSpace fromAvColorSpace(AVColorSpace colorSpace) +{ + switch (colorSpace) { + default: + case AVCOL_SPC_UNSPECIFIED: + case AVCOL_SPC_RESERVED: + case AVCOL_SPC_FCC: + case AVCOL_SPC_SMPTE240M: + case AVCOL_SPC_YCGCO: + case AVCOL_SPC_SMPTE2085: + case AVCOL_SPC_CHROMA_DERIVED_NCL: + case AVCOL_SPC_CHROMA_DERIVED_CL: + case AVCOL_SPC_ICTCP: // BT.2100 ICtCp + return QVideoFrameFormat::ColorSpace_Undefined; + case AVCOL_SPC_RGB: + return QVideoFrameFormat::ColorSpace_AdobeRgb; + case AVCOL_SPC_BT709: + return QVideoFrameFormat::ColorSpace_BT709; + case AVCOL_SPC_BT470BG: // BT601 + case AVCOL_SPC_SMPTE170M: // Also BT601 + return QVideoFrameFormat::ColorSpace_BT601; + case AVCOL_SPC_BT2020_NCL: // Non constant luminence + case AVCOL_SPC_BT2020_CL: // Constant luminence + return QVideoFrameFormat::ColorSpace_BT2020; + } +} + +AVColorSpace toAvColorSpace(QVideoFrameFormat::ColorSpace colorSpace) +{ + switch (colorSpace) { + case QVideoFrameFormat::ColorSpace_BT601: + return AVCOL_SPC_BT470BG; + case QVideoFrameFormat::ColorSpace_BT709: + return AVCOL_SPC_BT709; + case QVideoFrameFormat::ColorSpace_AdobeRgb: + return AVCOL_SPC_RGB; + case QVideoFrameFormat::ColorSpace_BT2020: + return AVCOL_SPC_BT2020_CL; + default: + return AVCOL_SPC_UNSPECIFIED; + } +} + +QVideoFrameFormat::ColorRange fromAvColorRange(AVColorRange colorRange) +{ + switch (colorRange) { + case AVCOL_RANGE_MPEG: + return QVideoFrameFormat::ColorRange_Video; + case AVCOL_RANGE_JPEG: + return QVideoFrameFormat::ColorRange_Full; + default: + return QVideoFrameFormat::ColorRange_Unknown; + } +} + +AVColorRange toAvColorRange(QVideoFrameFormat::ColorRange colorRange) +{ + switch (colorRange) { + case QVideoFrameFormat::ColorRange_Video: + return AVCOL_RANGE_MPEG; + case QVideoFrameFormat::ColorRange_Full: + return AVCOL_RANGE_JPEG; + default: + return AVCOL_RANGE_UNSPECIFIED; + } +} + +AVHWDeviceContext* avFrameDeviceContext(const AVFrame* frame) { + if (!frame) + return {}; + if (!frame->hw_frames_ctx) + return {}; + + const auto *frameCtx = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data); + if (!frameCtx) + return {}; + + return frameCtx->device_ctx; +} + +#ifdef Q_OS_DARWIN +bool isCVFormatSupported(uint32_t cvFormat) +{ + return av_map_videotoolbox_format_to_pixfmt(cvFormat) != AV_PIX_FMT_NONE; +} + +std::string cvFormatToString(uint32_t cvFormat) +{ + auto formatDescIt = std::make_reverse_iterator(reinterpret_cast<const char *>(&cvFormat)); + return std::string(formatDescIt - 4, formatDescIt); +} + +#endif + +} // namespace QFFmpeg + +QDebug operator<<(QDebug dbg, const AVRational &value) +{ + dbg << value.num << "/" << value.den; + return dbg; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpeg_p.h b/src/plugins/multimedia/ffmpeg/qffmpeg_p.h index ac9076bbf..a0b87ff5d 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeg_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpeg_p.h @@ -1,53 +1,34 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEG_P_H #define QFFMPEG_P_H -#include <private/qtmultimediaglobal_p.h> +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpegdefs_p.h" +#include "qffmpegavaudioformat_p.h" +#include <QtMultimedia/qvideoframeformat.h> + #include <qstring.h> +#include <optional> -extern "C" { -#include <libavformat/avformat.h> -#include <libavcodec/avcodec.h> -#include <libswresample/swresample.h> -#include <libavutil/avutil.h> -#include <libswscale/swscale.h> +inline bool operator==(const AVRational &lhs, const AVRational &rhs) +{ + return lhs.num == rhs.num && lhs.den == rhs.den; +} + +inline bool operator!=(const AVRational &lhs, const AVRational &rhs) +{ + return !(lhs == rhs); } QT_BEGIN_NAMESPACE @@ -55,17 +36,30 @@ QT_BEGIN_NAMESPACE namespace QFFmpeg { -inline qint64 timeStamp(qint64 ts, AVRational base) +inline std::optional<qint64> mul(qint64 a, AVRational b) { - return (1000*ts*base.num + 500)/base.den; + return b.den != 0 ? (a * b.num + b.den / 2) / b.den : std::optional<qint64>{}; } -inline qint64 timeStampUs(qint64 ts, AVRational base) +inline std::optional<qreal> mul(qreal a, AVRational b) { - return (1000000*ts*base.num + 500000)/base.den; + return b.den != 0 ? a * qreal(b.num) / qreal(b.den) : std::optional<qreal>{}; } -inline float toFloat(AVRational r) { return float(r.num)/float(r.den); } +inline std::optional<qint64> timeStampMs(qint64 ts, AVRational base) +{ + return mul(1'000 * ts, base); +} + +inline std::optional<qint64> timeStampUs(qint64 ts, AVRational base) +{ + return mul(1'000'000 * ts, base); +} + +inline std::optional<float> toFloat(AVRational r) +{ + return r.den != 0 ? float(r.num) / float(r.den) : std::optional<float>{}; +} inline QString err2str(int errnum) { @@ -74,8 +68,227 @@ inline QString err2str(int errnum) return QString::fromLocal8Bit(buffer); } -QT_END_NAMESPACE +inline void setAVFrameTime(AVFrame &frame, int64_t pts, const AVRational &timeBase) +{ + frame.pts = pts; +#if QT_FFMPEG_HAS_FRAME_TIME_BASE + frame.time_base = timeBase; +#else + Q_UNUSED(timeBase); +#endif +} + +inline void getAVFrameTime(const AVFrame &frame, int64_t &pts, AVRational &timeBase) +{ + pts = frame.pts; +#if QT_FFMPEG_HAS_FRAME_TIME_BASE + timeBase = frame.time_base; +#else + timeBase = { 0, 1 }; +#endif +} +inline int64_t getAVFrameDuration(const AVFrame &frame) +{ +#if QT_FFMPEG_HAS_FRAME_DURATION + return frame.duration; +#else + Q_UNUSED(frame); + return 0; +#endif } +struct AVDictionaryHolder +{ + AVDictionary *opts = nullptr; + + operator AVDictionary **() { return &opts; } + + AVDictionaryHolder() = default; + + Q_DISABLE_COPY(AVDictionaryHolder) + + AVDictionaryHolder(AVDictionaryHolder &&other) noexcept + : opts(std::exchange(other.opts, nullptr)) + { + } + + ~AVDictionaryHolder() + { + if (opts) + av_dict_free(&opts); + } +}; + +template<typename FunctionType, FunctionType F> +struct AVDeleter +{ + template<typename T> + void operator()(T *object) const + { + if (object) + F(&object); + } +}; + +using AVFrameUPtr = std::unique_ptr<AVFrame, AVDeleter<decltype(&av_frame_free), &av_frame_free>>; + +inline AVFrameUPtr makeAVFrame() +{ + return AVFrameUPtr(av_frame_alloc()); +} + +using AVPacketUPtr = + std::unique_ptr<AVPacket, AVDeleter<decltype(&av_packet_free), &av_packet_free>>; + +using AVCodecContextUPtr = + std::unique_ptr<AVCodecContext, + AVDeleter<decltype(&avcodec_free_context), &avcodec_free_context>>; + +using AVBufferUPtr = + std::unique_ptr<AVBufferRef, AVDeleter<decltype(&av_buffer_unref), &av_buffer_unref>>; + +using AVHWFramesConstraintsUPtr = std::unique_ptr< + AVHWFramesConstraints, + AVDeleter<decltype(&av_hwframe_constraints_free), &av_hwframe_constraints_free>>; + +using SwrContextUPtr = std::unique_ptr<SwrContext, AVDeleter<decltype(&swr_free), &swr_free>>; + +using PixelOrSampleFormat = int; +using AVScore = int; +constexpr AVScore BestAVScore = std::numeric_limits<AVScore>::max(); +constexpr AVScore DefaultAVScore = 0; +constexpr AVScore NotSuitableAVScore = std::numeric_limits<AVScore>::min(); +constexpr AVScore MinAVScore = NotSuitableAVScore + 1; + +template <typename T> +inline constexpr auto InvalidAvValue = T{}; + +template<> +inline constexpr auto InvalidAvValue<AVSampleFormat> = AV_SAMPLE_FMT_NONE; + +template<> +inline constexpr auto InvalidAvValue<AVPixelFormat> = AV_PIX_FMT_NONE; + +const AVCodec *findAVDecoder(AVCodecID codecId, + const std::optional<AVHWDeviceType> &deviceType = {}, + const std::optional<PixelOrSampleFormat> &format = {}); + +const AVCodec *findAVEncoder(AVCodecID codecId, + const std::optional<AVHWDeviceType> &deviceType = {}, + const std::optional<PixelOrSampleFormat> &format = {}); + +const AVCodec *findAVEncoder(AVCodecID codecId, + const std::function<AVScore(const AVCodec *)> &scoresGetter); + +bool isAVFormatSupported(const AVCodec *codec, PixelOrSampleFormat format); + +template<typename Format> +bool hasAVFormat(const Format *fmts, Format format) +{ + return findAVFormat(fmts, [format](Format f) { return f == format; }) != InvalidAvValue<Format>; +} + +template<typename Format, typename Predicate> +Format findAVFormat(const Format *fmts, const Predicate &predicate) +{ + auto scoresGetter = [&predicate](Format fmt) { + return predicate(fmt) ? BestAVScore : NotSuitableAVScore; + }; + return findBestAVValue(fmts, scoresGetter).first; +} + +template <typename Predicate> +const AVCodecHWConfig *findHwConfig(const AVCodec *codec, const Predicate &predicate) +{ + for (int i = 0; const auto hwConfig = avcodec_get_hw_config(codec, i); ++i) { + if (predicate(hwConfig)) + return hwConfig; + } + + return nullptr; +} + +template <typename Predicate> +AVPixelFormat findAVPixelFormat(const AVCodec *codec, const Predicate &predicate) +{ + const AVPixelFormat format = findAVFormat(codec->pix_fmts, predicate); + if (format != AV_PIX_FMT_NONE) + return format; + + auto checkHwConfig = [&predicate](const AVCodecHWConfig *config) { + return config->pix_fmt != AV_PIX_FMT_NONE && predicate(config->pix_fmt); + }; + + if (auto hwConfig = findHwConfig(codec, checkHwConfig)) + return hwConfig->pix_fmt; + + return AV_PIX_FMT_NONE; +} + +template <typename Value, typename CalculateScore> +auto findBestAVValue(const Value *values, const CalculateScore &calculateScore) +{ + using Limits = std::numeric_limits<decltype(calculateScore(*values))>; + + const Value invalidValue = InvalidAvValue<Value>; + std::pair result(invalidValue, Limits::min()); + if (values) { + + for (; *values != invalidValue && result.second != Limits::max(); ++values) { + const auto score = calculateScore(*values); + if (score > result.second) + result = { *values, score }; + } + } + + return result; +} + +bool isHwPixelFormat(AVPixelFormat format); + +inline bool isSwPixelFormat(AVPixelFormat format) +{ + return !isHwPixelFormat(format); +} + +bool isAVCodecExperimental(const AVCodec *codec); + +void applyExperimentalCodecOptions(const AVCodec *codec, AVDictionary** opts); + +AVPixelFormat pixelFormatForHwDevice(AVHWDeviceType deviceType); + +AVPacketSideData *addStreamSideData(AVStream *stream, AVPacketSideData sideData); + +const AVPacketSideData *streamSideData(const AVStream *stream, AVPacketSideDataType type); + +SwrContextUPtr createResampleContext(const AVAudioFormat &inputFormat, + const AVAudioFormat &outputFormat); + +QVideoFrameFormat::ColorTransfer fromAvColorTransfer(AVColorTransferCharacteristic colorTrc); + +AVColorTransferCharacteristic toAvColorTransfer(QVideoFrameFormat::ColorTransfer colorTrc); + +QVideoFrameFormat::ColorSpace fromAvColorSpace(AVColorSpace colorSpace); + +AVColorSpace toAvColorSpace(QVideoFrameFormat::ColorSpace colorSpace); + +QVideoFrameFormat::ColorRange fromAvColorRange(AVColorRange colorRange); + +AVColorRange toAvColorRange(QVideoFrameFormat::ColorRange colorRange); + +AVHWDeviceContext *avFrameDeviceContext(const AVFrame *frame); + +#ifdef Q_OS_DARWIN +bool isCVFormatSupported(uint32_t format); + +std::string cvFormatToString(uint32_t format); + +#endif +} + +QDebug operator<<(QDebug, const AVRational &); + +QT_END_NAMESPACE + #endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp index 86508f24d..20c27982c 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder.cpp @@ -1,54 +1,15 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -//#define DEBUG_DECODER - +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegaudiodecoder_p.h" -#include "qffmpegdecoder_p.h" -#include "qffmpegmediaformatinfo_p.h" #include "qffmpegresampler_p.h" #include "qaudiobuffer.h" -#include <qloggingcategory.h> +#include "qffmpegplaybackengine_p.h" +#include "playbackengine/qffmpegrenderer_p.h" -Q_LOGGING_CATEGORY(qLcAudioDecoder, "qt.multimedia.ffmpeg.audioDecoder") +#include <qloggingcategory.h> -#define MAX_BUFFERS_IN_QUEUE 4 +Q_STATIC_LOGGING_CATEGORY(qLcAudioDecoder, "qt.multimedia.ffmpeg.audioDecoder") QT_BEGIN_NAMESPACE @@ -57,96 +18,67 @@ namespace QFFmpeg class SteppingAudioRenderer : public Renderer { + Q_OBJECT public: - SteppingAudioRenderer(AudioDecoder *decoder, const QAudioFormat &format); - ~SteppingAudioRenderer() + SteppingAudioRenderer(const QAudioFormat &format) : Renderer({}), m_format(format) { } + + RenderingResult renderInternal(Frame frame) override { + if (!frame.isValid()) + return {}; + + if (!m_resampler) + m_resampler = std::make_unique<QFFmpegResampler>(frame.codec(), m_format); + + emit newAudioBuffer(m_resampler->resample(frame.avFrame())); + + return {}; } - void loop() override; - AudioDecoder *m_decoder; +signals: + void newAudioBuffer(QAudioBuffer); + +private: QAudioFormat m_format; - std::unique_ptr<Resampler> resampler; - bool atEndEmitted = false; + std::unique_ptr<QFFmpegResampler> m_resampler; }; -class AudioDecoder : public Decoder +class AudioDecoder : public PlaybackEngine { Q_OBJECT public: - explicit AudioDecoder(QFFmpegAudioDecoder *audioDecoder) - : Decoder(audioDecoder) - {} + explicit AudioDecoder(const QAudioFormat &format) : m_format(format) { } - void setup(const QAudioFormat &format) + RendererPtr createRenderer(QPlatformMediaPlayer::TrackType trackType) override { - connect(this, &AudioDecoder::newAudioBuffer, audioDecoder, &QFFmpegAudioDecoder::newAudioBuffer); - connect(this, &AudioDecoder::isAtEnd, audioDecoder, &QFFmpegAudioDecoder::done); - m_format = format; - audioRenderer = new SteppingAudioRenderer(this, format); - audioRenderer->start(); - auto *stream = demuxer->addStream(m_currentAVStreamIndex[QPlatformMediaPlayer::AudioStream]); - audioRenderer->setStream(stream); + if (trackType != QPlatformMediaPlayer::AudioStream) + return RendererPtr{ {}, {} }; + + auto result = createPlaybackEngineObject<SteppingAudioRenderer>(m_format); + m_audioRenderer = result.get(); + + connect(result.get(), &SteppingAudioRenderer::newAudioBuffer, this, + &AudioDecoder::newAudioBuffer); + + return result; } void nextBuffer() { - audioRenderer->setPaused(false); + Q_ASSERT(m_audioRenderer); + Q_ASSERT(!m_audioRenderer->isStepForced()); + + m_audioRenderer->doForceStep(); + // updateObjectsPausedState(); } -Q_SIGNALS: - void newAudioBuffer(const QAudioBuffer &b); - void isAtEnd(); +signals: + void newAudioBuffer(QAudioBuffer); private: + QPointer<Renderer> m_audioRenderer; QAudioFormat m_format; }; - -SteppingAudioRenderer::SteppingAudioRenderer(AudioDecoder *decoder, const QAudioFormat &format) - : Renderer(QPlatformMediaPlayer::AudioStream) - , m_decoder(decoder) - , m_format(format) -{ -} - - -void SteppingAudioRenderer::loop() -{ - if (!streamDecoder) { - qCDebug(qLcAudioDecoder) << "no stream"; - timeOut = -1; // Avoid CPU load before play() - return; - } - - Frame frame = streamDecoder->takeFrame(); - if (!frame.isValid()) { - if (streamDecoder->isAtEnd()) { - if (!atEndEmitted) - emit m_decoder->isAtEnd(); - atEndEmitted = true; - paused = true; - doneStep(); - timeOut = -1; - return; - } - timeOut = 10; - streamDecoder->wake(); - return; - } - qCDebug(qLcAudioDecoder) << " got frame"; - - doneStep(); - - if (!resampler) - resampler.reset(new Resampler(frame.codec(), m_format)); - - auto buffer = resampler->resample(frame.avFrame()); - paused = true; - timeOut = -1; - - emit m_decoder->newAudioBuffer(buffer); -} - } @@ -155,10 +87,7 @@ QFFmpegAudioDecoder::QFFmpegAudioDecoder(QAudioDecoder *parent) { } -QFFmpegAudioDecoder::~QFFmpegAudioDecoder() -{ - delete decoder; -} +QFFmpegAudioDecoder::~QFFmpegAudioDecoder() = default; QUrl QFFmpegAudioDecoder::source() const { @@ -170,11 +99,8 @@ void QFFmpegAudioDecoder::setSource(const QUrl &fileName) stop(); m_sourceDevice = nullptr; - if (m_url == fileName) - return; - m_url = fileName; - - emit sourceChanged(); + if (std::exchange(m_url, fileName) != fileName) + sourceChanged(); } QIODevice *QFFmpegAudioDecoder::sourceDevice() const @@ -186,47 +112,65 @@ void QFFmpegAudioDecoder::setSourceDevice(QIODevice *device) { stop(); m_url.clear(); - bool isSignalRequired = (m_sourceDevice != device); - m_sourceDevice = device; - if (isSignalRequired) + if (std::exchange(m_sourceDevice, device) != device) sourceChanged(); } void QFFmpegAudioDecoder::start() { qCDebug(qLcAudioDecoder) << "start"; - delete decoder; - decoder = new QFFmpeg::AudioDecoder(this); - decoder->setMedia(m_url, m_sourceDevice); - if (error() != QAudioDecoder::NoError) - goto error; - - decoder->setup(m_audioFormat); - if (error() != QAudioDecoder::NoError) - goto error; - decoder->play(); - if (error() != QAudioDecoder::NoError) - goto error; - decoder->nextBuffer(); - if (error() != QAudioDecoder::NoError) - goto error; + auto checkNoError = [this]() { + if (error() == QAudioDecoder::NoError) + return true; + + durationChanged(-1); + positionChanged(-1); + + m_decoder.reset(); + + return false; + }; + + m_decoder = std::make_unique<AudioDecoder>(m_audioFormat); + connect(m_decoder.get(), &AudioDecoder::errorOccured, this, &QFFmpegAudioDecoder::errorSignal); + connect(m_decoder.get(), &AudioDecoder::endOfStream, this, &QFFmpegAudioDecoder::done); + connect(m_decoder.get(), &AudioDecoder::newAudioBuffer, this, + &QFFmpegAudioDecoder::newAudioBuffer); + + QFFmpeg::MediaDataHolder::Maybe media = QFFmpeg::MediaDataHolder::create(m_url, m_sourceDevice, nullptr); + + if (media) { + Q_ASSERT(media.value()); + if (media.value()->streamInfo(QPlatformMediaPlayer::AudioStream).isEmpty()) + error(QAudioDecoder::FormatError, + QLatin1String("The media doesn't contain an audio stream")); + else + m_decoder->setMedia(std::move(*media.value())); + } else { + auto [code, description] = media.error(); + errorSignal(code, description); + } - setIsDecoding(true); - return; + if (!checkNoError()) + return; + + m_decoder->setState(QMediaPlayer::PausedState); + if (!checkNoError()) + return; - error: - durationChanged(-1); - positionChanged(-1); - delete decoder; - decoder = nullptr; + m_decoder->nextBuffer(); + if (!checkNoError()) + return; + durationChanged(m_decoder->duration() / 1000); + setIsDecoding(true); } void QFFmpegAudioDecoder::stop() { qCDebug(qLcAudioDecoder) << ">>>>> stop"; - if (decoder) { - decoder->stop(); + if (m_decoder) { + m_decoder.reset(); done(); } } @@ -238,26 +182,28 @@ QAudioFormat QFFmpegAudioDecoder::audioFormat() const void QFFmpegAudioDecoder::setAudioFormat(const QAudioFormat &format) { - if (m_audioFormat == format) - return; - - m_audioFormat = format; - formatChanged(m_audioFormat); + if (std::exchange(m_audioFormat, format) != format) + formatChanged(m_audioFormat); } QAudioBuffer QFFmpegAudioDecoder::read() { - auto b = m_audioBuffer; - qCDebug(qLcAudioDecoder) << "reading buffer" << b.startTime(); - m_audioBuffer = {}; + auto buffer = std::exchange(m_audioBuffer, QAudioBuffer{}); + if (!buffer.isValid()) + return buffer; + qCDebug(qLcAudioDecoder) << "reading buffer" << buffer.startTime(); bufferAvailableChanged(false); - if (decoder) - decoder->nextBuffer(); - return b; + if (m_decoder) + m_decoder->nextBuffer(); + return buffer; } void QFFmpegAudioDecoder::newAudioBuffer(const QAudioBuffer &b) { + Q_ASSERT(b.isValid()); + Q_ASSERT(!m_audioBuffer.isValid()); + Q_ASSERT(!bufferAvailable()); + qCDebug(qLcAudioDecoder) << "new audio buffer" << b.startTime(); m_audioBuffer = b; const qint64 pos = b.startTime(); @@ -272,6 +218,30 @@ void QFFmpegAudioDecoder::done() finished(); } +void QFFmpegAudioDecoder::errorSignal(int err, const QString &errorString) +{ + // unfortunately the error enums for QAudioDecoder and QMediaPlayer aren't identical. + // Map them. + switch (QMediaPlayer::Error(err)) { + case QMediaPlayer::NoError: + error(QAudioDecoder::NoError, errorString); + break; + case QMediaPlayer::ResourceError: + error(QAudioDecoder::ResourceError, errorString); + break; + case QMediaPlayer::FormatError: + error(QAudioDecoder::FormatError, errorString); + break; + case QMediaPlayer::NetworkError: + // fall through, Network error doesn't exist in QAudioDecoder + case QMediaPlayer::AccessDeniedError: + error(QAudioDecoder::AccessDeniedError, errorString); + break; + } +} + QT_END_NAMESPACE +#include "moc_qffmpegaudiodecoder_p.cpp" + #include "qffmpegaudiodecoder.moc" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder_p.h b/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder_p.h index 790a29b04..11816cd6f 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegaudiodecoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGAUDIODECODER_H #define QFFMPEGAUDIODECODER_H @@ -53,10 +17,7 @@ #include "private/qplatformaudiodecoder_p.h" #include <qffmpeg_p.h> - -#include <qmutex.h> #include <qurl.h> -#include <qqueue.h> QT_BEGIN_NAMESPACE @@ -89,11 +50,14 @@ public: public Q_SLOTS: void newAudioBuffer(const QAudioBuffer &b); void done(); + void errorSignal(int err, const QString &errorString); private: + using AudioDecoder = QFFmpeg::AudioDecoder; + QUrl m_url; QIODevice *m_sourceDevice = nullptr; - QFFmpeg::AudioDecoder *decoder = nullptr; + std::unique_ptr<AudioDecoder> m_decoder; QAudioFormat m_audioFormat; QAudioBuffer m_audioBuffer; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegaudioinput.cpp b/src/plugins/multimedia/ffmpeg/qffmpegaudioinput.cpp index 71979e04f..014c53ca4 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegaudioinput.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegaudioinput.cpp @@ -1,45 +1,10 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegaudioinput_p.h" #include <qiodevice.h> #include <qaudiosource.h> #include <qaudiobuffer.h> +#include <qatomic.h> #include <qdebug.h> QT_BEGIN_NAMESPACE @@ -49,37 +14,33 @@ namespace QFFmpeg { class AudioSourceIO : public QIODevice { Q_OBJECT - public: - AudioSourceIO(QFFmpegAudioInput *audioInput) - : QIODevice() - , input(audioInput) +public: + AudioSourceIO(QFFmpegAudioInput *audioInput) : QIODevice(), m_input(audioInput) { - m_muted = input->muted; - m_volume = input->volume; + m_muted = m_input->muted; + m_volume = m_input->volume; updateVolume(); open(QIODevice::WriteOnly); } - ~AudioSourceIO() - { - delete m_src; - } + + ~AudioSourceIO() override = default; void setDevice(const QAudioDevice &device) { - QMutexLocker locker(&mutex); + QMutexLocker locker(&m_mutex); if (m_device == device) return; m_device = device; QMetaObject::invokeMethod(this, "updateSource"); } - void setFrameSize(int s) + void setFrameSize(int frameSize) { - QMutexLocker locker(&mutex); - frameSize = s; - bufferSize = m_format.bytesForFrames(frameSize); + m_bufferSize.storeRelease((frameSize > 0 && m_format.isValid()) + ? m_format.bytesForFrames(frameSize) + : DefaultAudioInputBufferSize); } void setRunning(bool r) { - QMutexLocker locker(&mutex); + QMutexLocker locker(&m_mutex); if (m_running == r) return; m_running = r; @@ -87,16 +48,17 @@ class AudioSourceIO : public QIODevice } void setVolume(float vol) { - QMutexLocker locker(&mutex); + QMutexLocker locker(&m_mutex); m_volume = vol; QMetaObject::invokeMethod(this, "updateVolume"); } void setMuted(bool muted) { - QMutexLocker locker(&mutex); + QMutexLocker locker(&m_mutex); m_muted = muted; QMetaObject::invokeMethod(this, "updateVolume"); } + int bufferSize() const { return m_bufferSize.loadAcquire(); } protected: qint64 readData(char *, qint64) override @@ -107,11 +69,12 @@ protected: { int l = len; while (len > 0) { - int toAppend = qMin(len, bufferSize - pcm.size()); - pcm.append(data, toAppend); + const auto bufferSize = m_bufferSize.loadAcquire(); + int toAppend = qMin(len, bufferSize - m_pcm.size()); + m_pcm.append(data, toAppend); data += toAppend; len -= toAppend; - if (pcm.size() == bufferSize) + if (m_pcm.size() == bufferSize) sendBuffer(); } @@ -120,13 +83,12 @@ protected: private Q_SLOTS: void updateSource() { - QMutexLocker locker(&mutex); + QMutexLocker locker(&m_mutex); m_format = m_device.preferredFormat(); - if (m_src) { - delete m_src; - pcm.clear(); - } - m_src = new QAudioSource(m_device, m_format); + if (std::exchange(m_src, nullptr)) + m_pcm.clear(); + + m_src = std::make_unique<QAudioSource>(m_device, m_format); updateVolume(); if (m_running) m_src->start(this); @@ -138,7 +100,7 @@ private Q_SLOTS: } void updateRunning() { - QMutexLocker locker(&mutex); + QMutexLocker locker(&m_mutex); if (m_running) { if (!m_src) updateSource(); @@ -153,26 +115,25 @@ private: void sendBuffer() { QAudioFormat fmt = m_src->format(); - qint64 time = fmt.durationForBytes(processed); - QAudioBuffer buffer(pcm, fmt, time); - emit input->newAudioBuffer(buffer); - processed += bufferSize; - pcm.clear(); + qint64 time = fmt.durationForBytes(m_processed); + QAudioBuffer buffer(m_pcm, fmt, time); + emit m_input->newAudioBuffer(buffer); + m_processed += m_pcm.size(); + m_pcm.clear(); } - QMutex mutex; + QMutex m_mutex; QAudioDevice m_device; float m_volume = 1.; bool m_muted = false; bool m_running = false; - QFFmpegAudioInput *input = nullptr; - QAudioSource *m_src = nullptr; + QFFmpegAudioInput *m_input = nullptr; + std::unique_ptr<QAudioSource> m_src; QAudioFormat m_format; - int frameSize = 0; - int bufferSize = 0; - qint64 processed = 0; - QByteArray pcm; + QAtomicInt m_bufferSize = DefaultAudioInputBufferSize; + qint64 m_processed = 0; + QByteArray m_pcm; }; } @@ -182,17 +143,19 @@ QFFmpegAudioInput::QFFmpegAudioInput(QAudioInput *qq) { qRegisterMetaType<QAudioBuffer>(); - inputThread = new QThread; + inputThread = std::make_unique<QThread>(); audioIO = new QFFmpeg::AudioSourceIO(this); - audioIO->moveToThread(inputThread); + audioIO->moveToThread(inputThread.get()); inputThread->start(); } QFFmpegAudioInput::~QFFmpegAudioInput() { + // Ensure that COM is uninitialized by nested QWindowsResampler + // on the same thread that initialized it. + audioIO->deleteLater(); inputThread->exit(); inputThread->wait(); - delete inputThread; } void QFFmpegAudioInput::setAudioDevice(const QAudioDevice &device) @@ -220,6 +183,13 @@ void QFFmpegAudioInput::setRunning(bool b) audioIO->setRunning(b); } +int QFFmpegAudioInput::bufferSize() const +{ + return audioIO->bufferSize(); +} + QT_END_NAMESPACE +#include "moc_qffmpegaudioinput_p.cpp" + #include "qffmpegaudioinput.moc" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegaudioinput_p.h b/src/plugins/multimedia/ffmpeg/qffmpegaudioinput_p.h index c4f87df9d..288b3f432 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegaudioinput_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegaudioinput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGAUDIOINPUT_H #define QFFMPEGAUDIOINPUT_H @@ -51,6 +15,7 @@ // #include <private/qplatformaudioinput_p.h> +#include <private/qplatformaudiobufferinput_p.h> #include "qffmpegthread_p.h" #include <qaudioinput.h> @@ -62,8 +27,11 @@ namespace QFFmpeg { class AudioSourceIO; } -class QFFmpegAudioInput : public QObject, public QPlatformAudioInput +constexpr int DefaultAudioInputBufferSize = 4096; + +class QFFmpegAudioInput : public QPlatformAudioBufferInputBase, public QPlatformAudioInput { + // for qobject_cast Q_OBJECT public: QFFmpegAudioInput(QAudioInput *qq); @@ -73,15 +41,14 @@ public: void setMuted(bool /*muted*/) override; void setVolume(float /*volume*/) override; - void setFrameSize(int s); + void setFrameSize(int frameSize); void setRunning(bool b); -Q_SIGNALS: - void newAudioBuffer(const QAudioBuffer &buffer); + int bufferSize() const; private: - QThread *inputThread = nullptr; QFFmpeg::AudioSourceIO *audioIO = nullptr; + std::unique_ptr<QThread> inputThread; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegavaudioformat.cpp b/src/plugins/multimedia/ffmpeg/qffmpegavaudioformat.cpp new file mode 100644 index 000000000..417219a48 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegavaudioformat.cpp @@ -0,0 +1,78 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegavaudioformat_p.h" +#include "qaudioformat.h" +#include "qffmpegmediaformatinfo_p.h" + +extern "C" { +#include <libavutil/opt.h> +} + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +AVAudioFormat::AVAudioFormat(const AVCodecContext *context) + : sampleFormat(context->sample_fmt), sampleRate(context->sample_rate) +{ +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + if (context->channel_layout) { + channelLayoutMask = context->channel_layout; + } else { + const auto channelConfig = + QAudioFormat::defaultChannelConfigForChannelCount(context->channels); + channelLayoutMask = QFFmpegMediaFormatInfo::avChannelLayout(channelConfig); + } +#else + channelLayout = context->ch_layout; +#endif +} + +AVAudioFormat::AVAudioFormat(const AVCodecParameters *codecPar) + : sampleFormat(AVSampleFormat(codecPar->format)), sampleRate(codecPar->sample_rate) +{ +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + if (codecPar->channel_layout) { + channelLayoutMask = codecPar->channel_layout; + } else { + const auto channelConfig = + QAudioFormat::defaultChannelConfigForChannelCount(codecPar->channels); + channelLayoutMask = QFFmpegMediaFormatInfo::avChannelLayout(channelConfig); + } +#else + channelLayout = codecPar->ch_layout; +#endif +} + +AVAudioFormat::AVAudioFormat(const QAudioFormat &audioFormat) + : sampleFormat(QFFmpegMediaFormatInfo::avSampleFormat(audioFormat.sampleFormat())), + sampleRate(audioFormat.sampleRate()) +{ + const auto channelConfig = audioFormat.channelConfig() == QAudioFormat::ChannelConfigUnknown + ? QAudioFormat::defaultChannelConfigForChannelCount(audioFormat.channelCount()) + : audioFormat.channelConfig(); + + const auto mask = QFFmpegMediaFormatInfo::avChannelLayout(channelConfig); + +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + channelLayoutMask = mask; +#else + av_channel_layout_from_mask(&channelLayout, mask); +#endif +} + +bool operator==(const AVAudioFormat &lhs, const AVAudioFormat &rhs) +{ + return lhs.sampleFormat == rhs.sampleFormat && lhs.sampleRate == rhs.sampleRate && +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + lhs.channelLayoutMask == rhs.channelLayoutMask +#else + lhs.channelLayout == rhs.channelLayout +#endif + ; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegavaudioformat_p.h b/src/plugins/multimedia/ffmpeg/qffmpegavaudioformat_p.h new file mode 100644 index 000000000..9fd4dacdf --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegavaudioformat_p.h @@ -0,0 +1,68 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGAVAUDIOFORMAT_P_H +#define QFFMPEGAVAUDIOFORMAT_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpegdefs_p.h" +#include <private/qtmultimediaglobal_p.h> + +#if !QT_FFMPEG_OLD_CHANNEL_LAYOUT +inline bool operator==(const AVChannelLayout &lhs, const AVChannelLayout &rhs) +{ + return lhs.order == rhs.order && lhs.nb_channels == rhs.nb_channels && lhs.u.mask == rhs.u.mask; +} + +inline bool operator!=(const AVChannelLayout &lhs, const AVChannelLayout &rhs) +{ + return !(lhs == rhs); +} + +#endif + +QT_BEGIN_NAMESPACE + +class QAudioFormat; + +namespace QFFmpeg { + +struct AVAudioFormat +{ + AVAudioFormat(const AVCodecContext *context); + + AVAudioFormat(const AVCodecParameters *codecPar); + + AVAudioFormat(const QAudioFormat &audioFormat); + +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + uint64_t channelLayoutMask; +#else + AVChannelLayout channelLayout; +#endif + AVSampleFormat sampleFormat; + int sampleRate; +}; + +bool operator==(const AVAudioFormat &lhs, const AVAudioFormat &rhs); + +inline bool operator!=(const AVAudioFormat &lhs, const AVAudioFormat &rhs) +{ + return !(lhs == rhs); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGAVAUDIOFORMAT_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegclock.cpp b/src/plugins/multimedia/ffmpeg/qffmpegclock.cpp deleted file mode 100644 index 8d463f081..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegclock.cpp +++ /dev/null @@ -1,214 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include <qffmpegclock_p.h> -#include <qloggingcategory.h> - -Q_LOGGING_CATEGORY(qLcClock, "qt.multimedia.ffmpeg.clock") - -QT_BEGIN_NAMESPACE - -QFFmpeg::Clock::Clock(ClockController *controller) - : controller(controller) -{ - Q_ASSERT(controller); - controller->addClock(this); -} - -QFFmpeg::Clock::~Clock() -{ - if (controller) - controller->removeClock(this); -} - -qint64 QFFmpeg::Clock::currentTime() const -{ - return controller ? controller->currentTime() : 0.; -} - -void QFFmpeg::Clock::syncTo(qint64 time) -{ - qCDebug(qLcClock) << "syncTo" << time << isMaster(); -} - -void QFFmpeg::Clock::setPlaybackRate(float rate, qint64 currentTime) -{ - qCDebug(qLcClock) << "Clock::setPlaybackRate" << rate; - Q_UNUSED(rate) - Q_UNUSED(currentTime) -} - -void QFFmpeg::Clock::setPaused(bool paused) -{ - qCDebug(qLcClock) << "Clock::setPaused" << paused; - Q_UNUSED(paused) -} - -qint64 QFFmpeg::Clock::timeUpdated(qint64 currentTime) -{ - if (controller) - return controller->timeUpdated(this, currentTime); - return currentTime; -} - -qint64 QFFmpeg::Clock::usecsTo(qint64 currentTime, qint64 displayTime) -{ - if (!controller || controller->m_isPaused) - return -1; - int t = qRound64((displayTime - currentTime)/playbackRate()); - return t < 0 ? 0 : t; -} - -QFFmpeg::Clock::Type QFFmpeg::Clock::type() const -{ - return SystemClock; -} - -QFFmpeg::ClockController::~ClockController() -{ - QMutexLocker l(&m_mutex); - for (auto *p : qAsConst(m_clocks)) - p->setController(nullptr); -} - -qint64 QFFmpeg::ClockController::timeUpdated(Clock *clock, qint64 time) -{ - QMutexLocker l(&m_mutex); - if (clock != m_master) { - // If the clock isn't the master clock, simply return the current time - // so we can make adjustments as needed - return currentTimeNoLock(); - } - - // if the clock is the master, adjust our base timing - m_baseTime = time; - m_timer.restart(); - - // Avoid posting too many updates to the notifyObject, or we can overload - // the event queue with too many notifications - if (qAbs(time - m_lastMasterTime) < 5000) - return time; - m_lastMasterTime = time; -// qCDebug(qLcClock) << "ClockController::timeUpdated(master)" << time << "skew" << skew(); - if (notifyObject) - notify.invoke(notifyObject, Qt::QueuedConnection, Q_ARG(qint64, time)); - return time; -} - -void QFFmpeg::ClockController::addClock(Clock *clock) -{ - QMutexLocker l(&m_mutex); - qCDebug(qLcClock) << "addClock" << clock; - Q_ASSERT(clock != nullptr); - - if (m_clocks.contains(clock)) - return; - - if (!m_master) - m_master = clock; - - m_clocks.append(clock); - clock->syncTo(currentTimeNoLock()); - clock->setPaused(m_isPaused); - - // update master clock - if (m_master != clock && clock->type() > m_master->type()) - m_master = clock; -} - -void QFFmpeg::ClockController::removeClock(Clock *clock) -{ - QMutexLocker l(&m_mutex); - qCDebug(qLcClock) << "removeClock" << clock; - m_clocks.removeAll(clock); - if (m_master == clock) { - // find a new master clock - m_master = nullptr; - for (auto *c : qAsConst(m_clocks)) { - if (!m_master || m_master->type() < c->type()) - m_master = c; - } - } -} - -qint64 QFFmpeg::ClockController::currentTime() const -{ - QMutexLocker l(&m_mutex); - return currentTimeNoLock(); -} - -void QFFmpeg::ClockController::syncTo(qint64 usecs) -{ - QMutexLocker l(&m_mutex); - qCDebug(qLcClock) << "syncTo" << usecs; - m_baseTime = usecs; - m_seekTime = usecs; - m_timer.restart(); - for (auto *p : qAsConst(m_clocks)) - p->syncTo(usecs); -} - -void QFFmpeg::ClockController::setPlaybackRate(float s) -{ - QMutexLocker l(&m_mutex); - qCDebug(qLcClock) << "setPlaybackRate" << s; - m_baseTime = currentTimeNoLock(); - m_timer.restart(); - m_playbackRate = s; - for (auto *p : qAsConst(m_clocks)) - p->setPlaybackRate(s, m_baseTime); -} - -void QFFmpeg::ClockController::setPaused(bool paused) -{ - QMutexLocker l(&m_mutex); - if (m_isPaused == paused) - return; - qCDebug(qLcClock) << "setPaused" << paused; - m_isPaused = paused; - if (m_isPaused) { - m_baseTime = currentTimeNoLock(); - m_seekTime = m_baseTime; - } else { - m_timer.restart(); - } - for (auto *p : qAsConst(m_clocks)) - p->setPaused(paused); -} - -QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegclock_p.h b/src/plugins/multimedia/ffmpeg/qffmpegclock_p.h deleted file mode 100644 index 7ca75c20e..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegclock_p.h +++ /dev/null @@ -1,157 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef QFFMPEGCLOCK_P_H -#define QFFMPEGCLOCK_P_H - -#include "qffmpeg_p.h" - -#include <qelapsedtimer.h> -#include <qlist.h> -#include <qmutex.h> -#include <qmetaobject.h> - -QT_BEGIN_NAMESPACE - -namespace QFFmpeg { - -class ClockController; - -// Clock runs in displayTime, ie. if playbackRate is not 1, it runs faster or slower -// than a regular clock. All methods take displayTime -// Exception: usecsTo() will return the real time that should pass until we will -// hit the requested display time -class Clock -{ - ClockController *controller = nullptr; -public: - enum Type { - SystemClock, - AudioClock - }; - Clock(ClockController *controller); - virtual ~Clock(); - virtual Type type() const; - - float playbackRate() const; - bool isMaster() const; - - // all times in usecs - qint64 currentTime() const; - qint64 seekTime() const; - qint64 usecsTo(qint64 currentTime, qint64 displayTime); - -protected: - virtual void syncTo(qint64 usecs); - virtual void setPlaybackRate(float rate, qint64 currentTime); - virtual void setPaused(bool paused); - - qint64 timeUpdated(qint64 currentTime); - -private: - friend class ClockController; - void setController(ClockController *c) - { - controller = c; - } -}; - -class ClockController -{ - mutable QMutex m_mutex; - QList<Clock *> m_clocks; - Clock *m_master = nullptr; - - QElapsedTimer m_timer; - qint64 m_baseTime = 0; - qint64 m_seekTime = 0; - float m_playbackRate = 1.; - bool m_isPaused = true; - - qint64 m_lastMasterTime = 0; - QObject *notifyObject = nullptr; - QMetaMethod notify; - qint64 currentTimeNoLock() const { return m_isPaused ? m_baseTime : m_baseTime + m_timer.elapsed()/m_playbackRate; } - - friend class Clock; - qint64 timeUpdated(Clock *clock, qint64 time); - void addClock(Clock *provider); - void removeClock(Clock *provider); -public: - // max 25 msecs tolerance for the clock - enum { ClockTolerance = 25000 }; - ClockController() = default; - ~ClockController(); - - - qint64 currentTime() const; - - void syncTo(qint64 usecs); - - void setPlaybackRate(float s); - float playbackRate() const { return m_playbackRate; } - void setPaused(bool paused); - - void setNotify(QObject *object, QMetaMethod method) - { - notifyObject = object; - notify = method; - } -}; - -inline float Clock::playbackRate() const -{ - return controller ? controller->m_playbackRate : 1.; -} - -inline bool Clock::isMaster() const -{ - return controller ? controller->m_master == this : false; -} - -inline qint64 Clock::seekTime() const -{ - return controller ? controller->m_seekTime : 0; -} - - -} - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegconverter.cpp b/src/plugins/multimedia/ffmpeg/qffmpegconverter.cpp new file mode 100644 index 000000000..ba87ce3ed --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegconverter.cpp @@ -0,0 +1,272 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegconverter_p.h" +#include <QtMultimedia/qvideoframeformat.h> +#include <QtMultimedia/qvideoframe.h> +#include <QtCore/qloggingcategory.h> +#include <private/qvideotexturehelper_p.h> + +extern "C" { +#include <libswscale/swscale.h> +} + +QT_BEGIN_NAMESPACE + +namespace { + +Q_LOGGING_CATEGORY(lc, "qt.multimedia.ffmpeg.converter"); + + +// Converts to FFmpeg pixel format. This function differs from +// QFFmpegVideoBuffer::toAVPixelFormat which only covers the subset +// of pixel formats required for encoding. Here we need to cover more +// pixel formats to be able to generate test images for decoding/display +AVPixelFormat toAVPixelFormat(QVideoFrameFormat::PixelFormat pixelFormat) +{ + switch (pixelFormat) { + default: + case QVideoFrameFormat::Format_Invalid: + return AV_PIX_FMT_NONE; + case QVideoFrameFormat::Format_AYUV: + case QVideoFrameFormat::Format_AYUV_Premultiplied: + return AV_PIX_FMT_NONE; // TODO: Fixme (No corresponding FFmpeg format available) + case QVideoFrameFormat::Format_YV12: + case QVideoFrameFormat::Format_IMC1: + case QVideoFrameFormat::Format_IMC3: + case QVideoFrameFormat::Format_IMC2: + case QVideoFrameFormat::Format_IMC4: + return AV_PIX_FMT_YUV420P; + case QVideoFrameFormat::Format_Jpeg: + return AV_PIX_FMT_BGRA; + case QVideoFrameFormat::Format_ARGB8888: + return AV_PIX_FMT_ARGB; + case QVideoFrameFormat::Format_ARGB8888_Premultiplied: + case QVideoFrameFormat::Format_XRGB8888: + return AV_PIX_FMT_0RGB; + case QVideoFrameFormat::Format_BGRA8888: + return AV_PIX_FMT_BGRA; + case QVideoFrameFormat::Format_BGRA8888_Premultiplied: + case QVideoFrameFormat::Format_BGRX8888: + return AV_PIX_FMT_BGR0; + case QVideoFrameFormat::Format_ABGR8888: + return AV_PIX_FMT_ABGR; + case QVideoFrameFormat::Format_XBGR8888: + return AV_PIX_FMT_0BGR; + case QVideoFrameFormat::Format_RGBA8888: + return AV_PIX_FMT_RGBA; + case QVideoFrameFormat::Format_RGBX8888: + return AV_PIX_FMT_RGB0; + case QVideoFrameFormat::Format_YUV422P: + return AV_PIX_FMT_YUV422P; + case QVideoFrameFormat::Format_YUV420P: + return AV_PIX_FMT_YUV420P; + case QVideoFrameFormat::Format_YUV420P10: + return AV_PIX_FMT_YUV420P10; + case QVideoFrameFormat::Format_UYVY: + return AV_PIX_FMT_UYVY422; + case QVideoFrameFormat::Format_YUYV: + return AV_PIX_FMT_YUYV422; + case QVideoFrameFormat::Format_NV12: + return AV_PIX_FMT_NV12; + case QVideoFrameFormat::Format_NV21: + return AV_PIX_FMT_NV21; + case QVideoFrameFormat::Format_Y8: + return AV_PIX_FMT_GRAY8; + case QVideoFrameFormat::Format_Y16: + return AV_PIX_FMT_GRAY16; + case QVideoFrameFormat::Format_P010: + return AV_PIX_FMT_P010; + case QVideoFrameFormat::Format_P016: + return AV_PIX_FMT_P016; + case QVideoFrameFormat::Format_SamplerExternalOES: + return AV_PIX_FMT_MEDIACODEC; + } +} + +struct SwsFrameData +{ + static constexpr int arraySize = 4; // Array size required by sws_scale + std::array<uchar *, arraySize> bits; + std::array<int, arraySize> stride; +}; + +SwsFrameData getSwsData(QVideoFrame &dst) +{ + switch (dst.pixelFormat()) { + case QVideoFrameFormat::Format_YV12: + case QVideoFrameFormat::Format_IMC1: + return { { dst.bits(0), dst.bits(2), dst.bits(1), nullptr }, + { dst.bytesPerLine(0), dst.bytesPerLine(2), dst.bytesPerLine(1), 0 } }; + + case QVideoFrameFormat::Format_IMC2: + return { { dst.bits(0), dst.bits(1) + dst.bytesPerLine(1) / 2, dst.bits(1), nullptr }, + { dst.bytesPerLine(0), dst.bytesPerLine(1), dst.bytesPerLine(1), 0 } }; + + case QVideoFrameFormat::Format_IMC4: + return { { dst.bits(0), dst.bits(1), dst.bits(1) + dst.bytesPerLine(1) / 2, nullptr }, + { dst.bytesPerLine(0), dst.bytesPerLine(1), dst.bytesPerLine(1), 0 } }; + default: + return { { dst.bits(0), dst.bits(1), dst.bits(2), nullptr }, + { dst.bytesPerLine(0), dst.bytesPerLine(1), dst.bytesPerLine(2), 0 } }; + } +} + +struct SwsColorSpace +{ + int colorSpace; + int colorRange; // 0 - mpeg/video, 1 - jpeg/full +}; + +// Qt heuristics for determining color space requires checking +// both frame color space and range. This function mimics logic +// used elsewhere in Qt Multimedia. +SwsColorSpace toSwsColorSpace(QVideoFrameFormat::ColorRange colorRange, + QVideoFrameFormat::ColorSpace colorSpace) +{ + const int avRange = colorRange == QVideoFrameFormat::ColorRange_Video ? 0 : 1; + + switch (colorSpace) { + case QVideoFrameFormat::ColorSpace_BT601: + if (colorRange == QVideoFrameFormat::ColorRange_Full) + return { SWS_CS_ITU709, 1 }; // TODO: FIXME - Not exact match + return { SWS_CS_ITU601, 0 }; + case QVideoFrameFormat::ColorSpace_BT709: + return { SWS_CS_ITU709, avRange }; + case QVideoFrameFormat::ColorSpace_AdobeRgb: + return { SWS_CS_ITU601, 1 }; // TODO: Why do ITU601 and Adobe RGB match well? + case QVideoFrameFormat::ColorSpace_BT2020: + return { SWS_CS_BT2020, avRange }; + case QVideoFrameFormat::ColorSpace_Undefined: + default: + return { SWS_CS_DEFAULT, avRange }; + } +} + +using SwsContextUPtr = std::unique_ptr<SwsContext, decltype(&sws_freeContext)>; +using PixelFormat = QVideoFrameFormat::PixelFormat; + +// clang-format off + +SwsContextUPtr createConverter(const QSize &srcSize, PixelFormat srcPixFmt, + const QSize &dstSize, PixelFormat dstPixFmt) +{ + SwsContext* context = sws_getContext( + srcSize.width(), srcSize.height(), toAVPixelFormat(srcPixFmt), + dstSize.width(), dstSize.height(), toAVPixelFormat(dstPixFmt), + SWS_BILINEAR, nullptr, nullptr, nullptr); + + return { context, &sws_freeContext }; +} + +bool setColorSpaceDetails(SwsContext *context, + const QVideoFrameFormat &srcFormat, + const QVideoFrameFormat &dstFormat) +{ + const SwsColorSpace src = toSwsColorSpace(srcFormat.colorRange(), srcFormat.colorSpace()); + const SwsColorSpace dst = toSwsColorSpace(dstFormat.colorRange(), dstFormat.colorSpace()); + + constexpr int brightness = 0; + constexpr int contrast = 0; + constexpr int saturation = 0; + const int status = sws_setColorspaceDetails(context, + sws_getCoefficients(src.colorSpace), src.colorRange, + sws_getCoefficients(dst.colorSpace), dst.colorRange, + brightness, contrast, saturation); + + return status == 0; +} + +bool convert(SwsContext *context, QVideoFrame &src, int srcHeight, QVideoFrame &dst) +{ + if (!src.map(QtVideo::MapMode::ReadOnly)) + return false; + + QScopeGuard unmapSrc{[&] { + src.unmap(); + }}; + + if (!dst.map(QtVideo::MapMode::WriteOnly)) + return false; + + QScopeGuard unmapDst{[&] { + dst.unmap(); + }}; + + const SwsFrameData srcData = getSwsData(src); + const SwsFrameData dstData = getSwsData(dst); + + constexpr int firstSrcSliceRow = 0; + const int scaledHeight = sws_scale(context, + srcData.bits.data(), srcData.stride.data(), + firstSrcSliceRow, srcHeight, + dstData.bits.data(), dstData.stride.data()); + + if (scaledHeight != srcHeight) + return false; + + return true; +} + +// Ensure even size if using planar format with chroma subsampling +QSize adjustSize(const QSize& size, PixelFormat srcFmt, PixelFormat dstFmt) +{ + const auto* srcDesc = QVideoTextureHelper::textureDescription(srcFmt); + const auto* dstDesc = QVideoTextureHelper::textureDescription(dstFmt); + + QSize output = size; + for (const auto desc : { srcDesc, dstDesc }) { + for (int i = 0; i < desc->nplanes; ++i) { + // TODO: Assumes that max subsampling is 2 + if (desc->sizeScale[i].x != 1) + output.setWidth(output.width() & ~1); // Make even + + if (desc->sizeScale[i].y != 1) + output.setHeight(output.height() & ~1); // Make even + } + } + + return output; +} + +} // namespace + +// Converts a video frame to the dstFormat video frame format. +QVideoFrame convertFrame(QVideoFrame &src, const QVideoFrameFormat &dstFormat) +{ + if (src.size() != dstFormat.frameSize()) { + qCCritical(lc) << "Resizing is not supported"; + return {}; + } + + // Adjust size to even width/height if we have chroma subsampling + const QSize size = adjustSize(src.size(), src.pixelFormat(), dstFormat.pixelFormat()); + if (size != src.size()) + qCWarning(lc) << "Input truncated to even width/height"; + + const SwsContextUPtr conv = createConverter( + size, src.pixelFormat(), size, dstFormat.pixelFormat()); + + if (!conv) { + qCCritical(lc) << "Failed to create SW converter"; + return {}; + } + + if (!setColorSpaceDetails(conv.get(), src.surfaceFormat(), dstFormat)) { + qCCritical(lc) << "Failed to set color space details"; + return {}; + } + + QVideoFrame dst{ dstFormat }; + + if (!convert(conv.get(), src, size.height(), dst)) { + qCCritical(lc) << "Frame conversion failed"; + return {}; + } + + return dst; +} + +// clang-format on + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegconverter_p.h b/src/plugins/multimedia/ffmpeg/qffmpegconverter_p.h new file mode 100644 index 000000000..57ee3135f --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegconverter_p.h @@ -0,0 +1,30 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGCONVERTER_P_H +#define QFFMPEGCONVERTER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/qtconfigmacros.h> +#include <private/qtmultimediaglobal_p.h> + +QT_BEGIN_NAMESPACE + +class QVideoFrameFormat; +class QVideoFrame; + +QVideoFrame convertFrame(QVideoFrame &src, const QVideoFrameFormat &dstFormat); + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp deleted file mode 100644 index 31b334fbc..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegdecoder.cpp +++ /dev/null @@ -1,1362 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qffmpegdecoder_p.h" -#include "qffmpegmediaformatinfo_p.h" -#include "qffmpeg_p.h" -#include "qffmpegmediametadata_p.h" -#include "qffmpegvideobuffer_p.h" -#include "private/qplatformaudiooutput_p.h" -#include "qffmpeghwaccel_p.h" -#include "qffmpegvideosink_p.h" -#include "qvideosink.h" -#include "qaudiosink.h" -#include "qaudiooutput.h" -#include "qffmpegaudiodecoder_p.h" -#include "qffmpegresampler_p.h" - -#include <qlocale.h> -#include <qtimer.h> - -#include <qloggingcategory.h> - -extern "C" { -#include <libavutil/hwcontext.h> -} - -QT_BEGIN_NAMESPACE - -using namespace QFFmpeg; - -Q_LOGGING_CATEGORY(qLcDemuxer, "qt.multimedia.ffmpeg.demuxer") -Q_LOGGING_CATEGORY(qLcDecoder, "qt.multimedia.ffmpeg.decoder") -Q_LOGGING_CATEGORY(qLcVideoRenderer, "qt.multimedia.ffmpeg.videoRenderer") -Q_LOGGING_CATEGORY(qLcAudioRenderer, "qt.multimedia.ffmpeg.audioRenderer") - -Codec::Data::Data(AVCodecContext *context, AVStream *stream, const HWAccel &hwAccel) - : context(context) - , stream(stream) - , hwAccel(hwAccel) -{ -} - -Codec::Data::~Data() -{ - if (!context) - return; - avcodec_close(context); - avcodec_free_context(&context); -} - -Codec::Codec(AVFormatContext *format, int streamIndex) -{ - qCDebug(qLcDecoder) << "Codec::Codec" << streamIndex; - Q_ASSERT(streamIndex >= 0 && streamIndex < (int)format->nb_streams); - - AVStream *stream = format->streams[streamIndex]; - const AVCodec *decoder = avcodec_find_decoder(stream->codecpar->codec_id); - if (!decoder) { - qCDebug(qLcDecoder) << "Failed to find a valid FFmpeg decoder"; - return; - } - - QFFmpeg::HWAccel hwAccel; - if (decoder->type == AVMEDIA_TYPE_VIDEO) { - hwAccel = QFFmpeg::HWAccel(decoder); - } - - auto *context = avcodec_alloc_context3(decoder); - if (!context) { - qCDebug(qLcDecoder) << "Failed to allocate a FFmpeg codec context"; - return; - } - - int ret = avcodec_parameters_to_context(context, stream->codecpar); - if (ret < 0) { - qCDebug(qLcDecoder) << "Failed to set FFmpeg codec parameters"; - return; - } - - auto *buf = hwAccel.hwDeviceContextAsBuffer(); - if (buf) - context->hw_device_ctx = av_buffer_ref(buf); - // ### This still gives errors about wrong HW formats (as we accept all of them) - // But it would be good to get so we can filter out pixel format we don't support natively - context->get_format = QFFmpeg::getFormat; - - /* Init the decoder, with reference counting and threading */ - AVDictionary *opts = nullptr; - av_dict_set(&opts, "refcounted_frames", "1", 0); - av_dict_set(&opts, "threads", "auto", 0); - ret = avcodec_open2(context, decoder, &opts); - if (ret < 0) { - qCDebug(qLcDecoder) << "Failed to open FFmpeg codec context."; - avcodec_free_context(&context); - return; - } - - d = new Data(context, stream, hwAccel); -} - - -Demuxer::Demuxer(Decoder *decoder, AVFormatContext *context) - : Thread() - , decoder(decoder) - , context(context) -{ - QString objectName = QLatin1String("Demuxer"); - setObjectName(objectName); - - streamDecoders.resize(context->nb_streams); -} - -Demuxer::~Demuxer() -{ - if (context) { - if (context->pb) { - av_free(context->pb); - context->pb = nullptr; - } - avformat_free_context(context); - } -} - -StreamDecoder *Demuxer::addStream(int streamIndex) -{ - if (streamIndex < 0) - return nullptr; - QMutexLocker locker(&mutex); - Codec codec(context, streamIndex); - if (!codec.isValid()) { - decoder->error(QMediaPlayer::FormatError, "Invalid media file"); - return nullptr; - } - - Q_ASSERT(codec.context()->codec_type == AVMEDIA_TYPE_AUDIO || - codec.context()->codec_type == AVMEDIA_TYPE_VIDEO || - codec.context()->codec_type == AVMEDIA_TYPE_SUBTITLE); - auto *stream = new StreamDecoder(this, codec); - Q_ASSERT(!streamDecoders.at(streamIndex)); - streamDecoders[streamIndex] = stream; - stream->start(); - updateEnabledStreams(); - return stream; -} - -void Demuxer::removeStream(int streamIndex) -{ - if (streamIndex < 0) - return; - QMutexLocker locker(&mutex); - Q_ASSERT(streamIndex < (int)context->nb_streams); - Q_ASSERT(streamDecoders.at(streamIndex) != nullptr); - streamDecoders[streamIndex] = nullptr; - updateEnabledStreams(); -} - -void Demuxer::stopDecoding() -{ - qCDebug(qLcDemuxer) << "StopDecoding"; - QMutexLocker locker(&mutex); - sendFinalPacketToStreams(); -} -int Demuxer::seek(qint64 pos) -{ - QMutexLocker locker(&mutex); - for (StreamDecoder *d : qAsConst(streamDecoders)) { - if (d) - d->mutex.lock(); - } - for (StreamDecoder *d : qAsConst(streamDecoders)) { - if (d) - d->flush(); - } - for (StreamDecoder *d : qAsConst(streamDecoders)) { - if (d) - d->mutex.unlock(); - } - qint64 seekPos = pos*AV_TIME_BASE/1000000; // usecs to AV_TIME_BASE - av_seek_frame(context, -1, seekPos, AVSEEK_FLAG_BACKWARD); - last_pts = -1; - loop(); - qCDebug(qLcDemuxer) << "Demuxer::seek" << pos << last_pts; - return last_pts; -} - -void Demuxer::updateEnabledStreams() -{ - if (isStopped()) - return; - for (uint i = 0; i < context->nb_streams; ++i) { - AVDiscard discard = AVDISCARD_DEFAULT; - if (!streamDecoders.at(i)) - discard = AVDISCARD_ALL; - context->streams[i]->discard = discard; - } -} - -void Demuxer::sendFinalPacketToStreams() -{ - if (m_isStopped.loadAcquire()) - return; - for (auto *streamDecoder : qAsConst(streamDecoders)) { - qCDebug(qLcDemuxer) << "Demuxer: sending last packet to stream" << streamDecoder; - if (!streamDecoder) - continue; - streamDecoder->addPacket(nullptr); - } - m_isStopped.storeRelease(true); -} - -void Demuxer::init() -{ - qCDebug(qLcDemuxer) << "Demuxer started"; -} - -void Demuxer::cleanup() -{ - qCDebug(qLcDemuxer) << "Demuxer::cleanup"; -#ifndef QT_NO_DEBUG - for (auto *streamDecoder : qAsConst(streamDecoders)) { - Q_ASSERT(!streamDecoder); - } -#endif - avformat_close_input(&context); - Thread::cleanup(); -} - -bool Demuxer::shouldWait() const -{ - if (m_isStopped) - return true; -// qCDebug(qLcDemuxer) << "XXXX Demuxer::shouldWait" << this << data->seek_pos.loadRelaxed(); - // require a minimum of 200ms of data - qint64 queueSize = 0; - bool buffersFull = true; - for (auto *d : streamDecoders) { - if (!d) - continue; - if (d->queuedDuration() < 200) - buffersFull = false; - queueSize += d->queuedPacketSize(); - } -// qCDebug(qLcDemuxer) << " queue size" << queueSize << MaxQueueSize; - if (queueSize > MaxQueueSize) - return true; -// qCDebug(qLcDemuxer) << " waiting!"; - return buffersFull; - -} - -void Demuxer::loop() -{ - AVPacket *packet = av_packet_alloc(); - if (av_read_frame(context, packet) < 0) { - sendFinalPacketToStreams(); - av_packet_free(&packet); - return; - } - - if (last_pts < 0 && packet->pts != AV_NOPTS_VALUE) { - auto *stream = context->streams[packet->stream_index]; - last_pts = timeStamp(packet->pts, stream->time_base); - } - - auto *streamDecoder = streamDecoders.at(packet->stream_index); - if (!streamDecoder) { - av_packet_free(&packet); - return; - } - streamDecoder->addPacket(packet); -} - - -StreamDecoder::StreamDecoder(Demuxer *demuxer, const Codec &codec) - : Thread() - , demuxer(demuxer) - , codec(codec) -{ - Q_ASSERT(codec.context()->codec_type == AVMEDIA_TYPE_AUDIO || - codec.context()->codec_type == AVMEDIA_TYPE_VIDEO || - codec.context()->codec_type == AVMEDIA_TYPE_SUBTITLE); - - QString objectName; - switch (codec.context()->codec_type) { - case AVMEDIA_TYPE_AUDIO: - objectName = QLatin1String("AudioDecoderThread"); - // Queue size: 3 frames for video/subtitle, 9 for audio - frameQueue.maxSize = 9; - break; - case AVMEDIA_TYPE_VIDEO: - objectName = QLatin1String("VideoDecoderThread"); - break; - case AVMEDIA_TYPE_SUBTITLE: - objectName = QLatin1String("SubtitleDecoderThread"); - break; - default: - Q_UNREACHABLE(); - } - setObjectName(objectName); -} - -void StreamDecoder::addPacket(AVPacket *packet) -{ - { - QMutexLocker locker(&packetQueue.mutex); -// qCDebug(qLcDecoder) << "enqueuing packet of type" << type() -// << "size" << packet->size -// << "stream index" << packet->stream_index -// << "pts" << codec.toMs(packet->pts) -// << "duration" << codec.toMs(packet->duration); - packetQueue.queue.enqueue(Packet(packet)); - if (packet) { - packetQueue.size += packet->size; - packetQueue.duration += codec.toMs(packet->duration); - } - eos.storeRelease(false); - } - wake(); -} - -void StreamDecoder::flush() -{ - qCDebug(qLcDecoder) << ">>>> flushing stream decoder" << type(); - avcodec_flush_buffers(codec.context()); - { - QMutexLocker locker(&packetQueue.mutex); - packetQueue.queue.clear(); - packetQueue.size = 0; - packetQueue.duration = 0; - } - { - QMutexLocker locker(&frameQueue.mutex); - frameQueue.queue.clear(); - } - qCDebug(qLcDecoder) << ">>>> done flushing stream decoder" << type(); -} - -void StreamDecoder::setRenderer(Renderer *r) -{ - QMutexLocker locker(&mutex); - m_renderer = r; - if (m_renderer) - m_renderer->wake(); -} - -void StreamDecoder::killHelper() -{ - m_renderer = nullptr; - demuxer->removeStream(codec.streamIndex()); -} - -Packet StreamDecoder::peekPacket() -{ - QMutexLocker locker(&packetQueue.mutex); - if (packetQueue.queue.isEmpty()) { - if (demuxer) - demuxer->wake(); - return {}; - } - auto packet = packetQueue.queue.first(); - - if (demuxer) - demuxer->wake(); - return packet; -} - -Packet StreamDecoder::takePacket() -{ - QMutexLocker locker(&packetQueue.mutex); - if (packetQueue.queue.isEmpty()) { - if (demuxer) - demuxer->wake(); - return {}; - } - auto packet = packetQueue.queue.dequeue(); - if (packet.avPacket()) { - packetQueue.size -= packet.avPacket()->size; - packetQueue.duration -= codec.toMs(packet.avPacket()->duration); - } -// qCDebug(qLcDecoder) << "<<<< dequeuing packet of type" << type() -// << "size" << packet.avPacket()->size -// << "stream index" << packet.avPacket()->stream_index -// << "pts" << codec.toMs(packet.avPacket()->pts) -// << "duration" << codec.toMs(packet.avPacket()->duration) -// << "ts" << decoder->clockController.currentTime(); - if (demuxer) - demuxer->wake(); - return packet; -} - -void StreamDecoder::addFrame(const Frame &f) -{ - Q_ASSERT(f.isValid()); - QMutexLocker locker(&frameQueue.mutex); - frameQueue.queue.append(std::move(f)); - if (m_renderer) - m_renderer->wake(); -} - -Frame StreamDecoder::takeFrame() -{ - QMutexLocker locker(&frameQueue.mutex); - // wake up the decoder so it delivers more frames - if (frameQueue.queue.isEmpty()) { - wake(); - return {}; - } - auto f = frameQueue.queue.dequeue(); - wake(); - return f; -} - -void StreamDecoder::init() -{ - qCDebug(qLcDecoder) << "Starting decoder"; -} - -bool StreamDecoder::shouldWait() const -{ - if (eos.loadAcquire() || (hasNoPackets() && decoderHasNoFrames) || hasEnoughFrames()) - return true; - return false; -} - -void StreamDecoder::loop() -{ - if (codec.context()->codec->type == AVMEDIA_TYPE_SUBTITLE) - decodeSubtitle(); - else - decode(); -} - -void StreamDecoder::decode() -{ - Q_ASSERT(codec.context()); - - AVFrame *frame = av_frame_alloc(); -// if (type() == 0) -// qDebug() << "receiving frame"; - int res = avcodec_receive_frame(codec.context(), frame); - - if (res >= 0) { - qint64 pts; - if (frame->pts != AV_NOPTS_VALUE) - pts = codec.toUs(frame->pts); - else - pts = codec.toUs(frame->best_effort_timestamp); - addFrame(Frame{frame, codec, pts}); - } else if (res == AVERROR(EOF) || res == AVERROR_EOF) { - eos.storeRelease(true); - av_frame_free(&frame); - timeOut = -1; - return; - } else if (res != AVERROR(EAGAIN)) { - char buf[512]; - av_make_error_string(buf, 512, res); - qWarning() << "error in decoder" << res << buf; - av_frame_free(&frame); - return; - } else { - // EAGAIN - decoderHasNoFrames = true; - av_frame_free(&frame); - } - - Packet packet = peekPacket(); - if (!packet.isValid()) { - timeOut = -1; - return; - } - - res = avcodec_send_packet(codec.context(), packet.avPacket()); - if (res != AVERROR(EAGAIN)) { - takePacket(); - } - decoderHasNoFrames = false; -} - -void StreamDecoder::decodeSubtitle() -{ - // qCDebug(qLcDecoder) << " decoding subtitle" << "has delay:" << (codec->codec->capabilities & AV_CODEC_CAP_DELAY); - AVSubtitle subtitle; - memset(&subtitle, 0, sizeof(subtitle)); - int gotSubtitle = 0; - Packet packet = takePacket(); - if (!packet.isValid()) - return; - - int res = avcodec_decode_subtitle2(codec.context(), &subtitle, &gotSubtitle, packet.avPacket()); - // qCDebug(qLcDecoder) << " subtitle got:" << res << gotSubtitle << subtitle.format << Qt::hex << (quint64)subtitle.pts; - if (res >= 0 && gotSubtitle) { - // apparently the timestamps in the AVSubtitle structure are not always filled in - // if they are missing, use the packets pts and duration values instead - qint64 start, end; - if (subtitle.pts == AV_NOPTS_VALUE) { - start = codec.toUs(packet.avPacket()->pts); - end = start + codec.toUs(packet.avPacket()->duration); - } else { - qint64 pts = timeStampUs(subtitle.pts, AVRational{1, AV_TIME_BASE}); - start = pts + qint64(subtitle.start_display_time)*1000; - end = pts + qint64(subtitle.end_display_time)*1000; - } - // qCDebug(qLcDecoder) << " got subtitle (" << start << "--" << end << "):"; - QString text; - for (uint i = 0; i < subtitle.num_rects; ++i) { - const auto *r = subtitle.rects[i]; - // qCDebug(qLcDecoder) << " subtitletext:" << r->text << "/" << r->ass; - if (i) - text += QLatin1Char('\n'); - if (r->text) - text += QString::fromUtf8(r->text); - else { - const char *ass = r->ass; - int nCommas = 0; - while (*ass) { - if (nCommas == 9) - break; - if (*ass == ',') - ++nCommas; - ++ass; - } - text += QString::fromUtf8(ass); - } - } - text.replace(QLatin1String("\\N"), QLatin1String("\n")); - text.replace(QLatin1String("\\n"), QLatin1String("\n")); - text.replace(QLatin1String("\r\n"), QLatin1String("\n")); - if (text.endsWith(QLatin1Char('\n'))) - text.chop(1); - -// qCDebug(qLcDecoder) << " >>> subtitle adding" << text << start << end; - Frame sub{text, start, end - start}; - addFrame(sub); - } -} - -QPlatformMediaPlayer::TrackType StreamDecoder::type() const -{ - switch (codec.stream()->codecpar->codec_type) { - case AVMEDIA_TYPE_AUDIO: - return QPlatformMediaPlayer::AudioStream; - case AVMEDIA_TYPE_VIDEO: - return QPlatformMediaPlayer::VideoStream; - case AVMEDIA_TYPE_SUBTITLE: - return QPlatformMediaPlayer::SubtitleStream; - default: - return QPlatformMediaPlayer::NTrackTypes; - } -} - -Renderer::Renderer(QPlatformMediaPlayer::TrackType type) - : Thread() - , type(type) -{ - QString objectName; - if (type == QPlatformMediaPlayer::AudioStream) - objectName = QLatin1String("AudioRenderThread"); - else - objectName = QLatin1String("VideoRenderThread"); - setObjectName(objectName); -} - -void Renderer::setStream(StreamDecoder *stream) -{ - QMutexLocker locker(&mutex); - if (streamDecoder == stream) - return; - if (streamDecoder) - streamDecoder->kill(); - streamDecoder = stream; - if (streamDecoder) - streamDecoder->setRenderer(this); - streamChanged(); - wake(); -} - -void Renderer::killHelper() -{ - if (streamDecoder) - streamDecoder->kill(); - streamDecoder = nullptr; -} - -bool Renderer::shouldWait() const -{ - if (!streamDecoder) - return true; - if (!paused) - return false; - if (step) - return false; - return true; -} - - -void ClockedRenderer::setPaused(bool paused) -{ - Clock::setPaused(paused); - Renderer::setPaused(paused); -} - -VideoRenderer::VideoRenderer(Decoder *decoder, QVideoSink *sink) - : ClockedRenderer(decoder, QPlatformMediaPlayer::VideoStream) - , sink(sink) -{} - -void VideoRenderer::killHelper() -{ - if (subtitleStreamDecoder) - subtitleStreamDecoder->kill(); - subtitleStreamDecoder = nullptr; - if (streamDecoder) - streamDecoder->kill(); - streamDecoder = nullptr; -} - -void VideoRenderer::setSubtitleStream(StreamDecoder *stream) -{ - QMutexLocker locker(&mutex); - qCDebug(qLcVideoRenderer) << "setting subtitle stream to" << stream; - if (stream == subtitleStreamDecoder) - return; - if (subtitleStreamDecoder) - subtitleStreamDecoder->kill(); - subtitleStreamDecoder = stream; - if (subtitleStreamDecoder) - subtitleStreamDecoder->setRenderer(this); - sink->setSubtitleText({}); - wake(); -} - -void VideoRenderer::init() -{ - qCDebug(qLcVideoRenderer) << "starting video renderer"; - ClockedRenderer::init(); -} - -void VideoRenderer::loop() -{ - if (!streamDecoder) { - timeOut = -1; // Avoid 100% CPU load before play() - return; - } - - Frame frame = streamDecoder->takeFrame(); - if (!frame.isValid()) { - if (streamDecoder->isAtEnd()) { - timeOut = -1; - eos.storeRelease(true); - emit atEnd(); - return; - } - timeOut = 1; -// qDebug() << "no valid frame" << timer.elapsed(); - return; - } - eos.storeRelease(false); -// qCDebug(qLcVideoRenderer) << "received video frame" << frame.pts(); - if (frame.pts() < seekTime()) { - qCDebug(qLcVideoRenderer) << " discarding" << frame.pts() << seekTime(); - return; - } - - AVStream *stream = frame.codec()->stream(); - qint64 startTime = frame.pts(); - qint64 duration = (1000000*stream->avg_frame_rate.den + (stream->avg_frame_rate.num>>1)) - /stream->avg_frame_rate.num; - - if (sink) { - qint64 startTime = frame.pts(); -// qDebug() << "RHI:" << accel.isNull() << accel.rhi() << sink->rhi(); - QFFmpegVideoBuffer *buffer = new QFFmpegVideoBuffer(frame.takeAVFrame()); - QVideoFrameFormat format(buffer->size(), buffer->pixelFormat()); - format.setColorSpace(buffer->colorSpace()); - format.setColorTransfer(buffer->colorTransfer()); - format.setColorRange(buffer->colorRange()); - format.setMaxLuminance(buffer->maxNits()); - QVideoFrame videoFrame(buffer, format); - videoFrame.setStartTime(startTime); - videoFrame.setEndTime(startTime + duration); -// qDebug() << "Creating video frame" << startTime << (startTime + duration) << subtitleStreamDecoder; - - // add in subtitles - const Frame *currentSubtitle = nullptr; - if (subtitleStreamDecoder) - currentSubtitle = subtitleStreamDecoder->lockAndPeekFrame(); - - if (currentSubtitle && currentSubtitle->isValid()) { -// qDebug() << "frame: subtitle" << currentSubtitle->text() << currentSubtitle->pts() << currentSubtitle->duration(); - qCDebug(qLcVideoRenderer) << " " << currentSubtitle->pts() << currentSubtitle->duration() << currentSubtitle->text(); - if (currentSubtitle->pts() <= startTime && currentSubtitle->end() > startTime) { -// qCDebug(qLcVideoRenderer) << " setting text"; - sink->setSubtitleText(currentSubtitle->text()); - } - if (currentSubtitle->end() < startTime) { -// qCDebug(qLcVideoRenderer) << " removing subtitle item"; - sink->setSubtitleText({}); - subtitleStreamDecoder->removePeekedFrame(); - } - } else { - sink->setSubtitleText({}); - } - if (subtitleStreamDecoder) - subtitleStreamDecoder->unlockAndReleaseFrame(); - -// qCDebug(qLcVideoRenderer) << " sending a video frame" << startTime << duration << decoder->baseTimer.elapsed(); - sink->setVideoFrame(videoFrame); - doneStep(); - } - const Frame *nextFrame = streamDecoder->lockAndPeekFrame(); - qint64 nextFrameTime = 0; - if (nextFrame) - nextFrameTime = nextFrame->pts(); - else - nextFrameTime = startTime + duration; - streamDecoder->unlockAndReleaseFrame(); - qint64 mtime = timeUpdated(startTime); - timeOut = usecsTo(mtime, nextFrameTime)/1000; -// qDebug() << " next video frame in" << startTime << nextFrameTime << currentTime() << timeOut; -} - -AudioRenderer::AudioRenderer(Decoder *decoder, QAudioOutput *output) - : ClockedRenderer(decoder, QPlatformMediaPlayer::AudioStream) - , output(output) -{ - connect(output, &QAudioOutput::deviceChanged, this, &AudioRenderer::updateAudio); -} - -void AudioRenderer::syncTo(qint64 usecs) -{ - Clock::syncTo(usecs); - audioBaseTime = usecs; - processedBase = processedUSecs; -} - -void AudioRenderer::setPlaybackRate(float rate, qint64 currentTime) -{ - audioBaseTime = currentTime; - processedBase = processedUSecs; - Clock::setPlaybackRate(rate, currentTime); - deviceChanged = true; -} - -void AudioRenderer::updateOutput(const Codec *codec) -{ - qCDebug(qLcAudioRenderer) << ">>>>>> updateOutput" << currentTime() << seekTime() << processedUSecs << isMaster(); - freeOutput(); - qCDebug(qLcAudioRenderer) << " " << currentTime() << seekTime() << processedUSecs; - - AVStream *audioStream = codec->stream(); - - auto dev = output->device(); - format = QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(audioStream->codecpar); - format.setChannelConfig(dev.channelConfiguration()); - - if (playbackRate() < 0.5 || playbackRate() > 2) - audioMuted = true; - - audioSink = new QAudioSink(dev, format); - audioSink->setBufferSize(format.bytesForDuration(100000)); - audioDevice = audioSink->start(); - latencyUSecs = format.durationForBytes(audioSink->bufferSize()); // ### ideally get full latency - qCDebug(qLcAudioRenderer) << " -> have an audio sink" << audioDevice; - - // init resampler. It's ok to always do this, as the resampler will be a no-op if - // formats agree. - AVSampleFormat requiredFormat = QFFmpegMediaFormatInfo::avSampleFormat(format.sampleFormat()); - - qCDebug(qLcAudioRenderer) << "init resampler" << requiredFormat << audioStream->codecpar->channels; - resampler.reset(new Resampler(codec, format)); -} - -void AudioRenderer::freeOutput() -{ - if (audioSink) { - audioSink->reset(); - delete audioSink; - audioSink = nullptr; - audioDevice = nullptr; - } - audioMuted = false; - bufferedData = {}; - bufferWritten = 0; - - audioBaseTime = currentTime(); - processedBase = 0; - processedUSecs = writtenUSecs = 0; -} - -void AudioRenderer::init() -{ - qCDebug(qLcAudioRenderer) << "Starting audio renderer"; - ClockedRenderer::init(); -} - -void AudioRenderer::cleanup() -{ - freeOutput(); -} - -void AudioRenderer::loop() -{ - if (!streamDecoder) { - timeOut = -1; // Avoid 100% CPU load before play() - return; - } - - if (deviceChanged) - freeOutput(); - deviceChanged = false; - doneStep(); - - qint64 bytesWritten = 0; - if (bufferedData.isValid()) { - bytesWritten = audioDevice->write(bufferedData.constData<char>() + bufferWritten, bufferedData.byteCount() - bufferWritten); - bufferWritten += bytesWritten; - if (bufferWritten == bufferedData.byteCount()) { - bufferedData = {}; - bufferWritten = 0; - } - processedUSecs = audioSink->processedUSecs(); - } else { - Frame frame = streamDecoder->takeFrame(); - if (!frame.isValid()) { - if (streamDecoder->isAtEnd()) { - if (audioSink) - processedUSecs = audioSink->processedUSecs(); - timeOut = -1; - eos.storeRelease(true); - emit atEnd(); - return; - } - timeOut = 1; - return; - } - eos.storeRelease(false); - - if (!audioSink) - updateOutput(frame.codec()); - - qint64 startTime = frame.pts(); - if (startTime < seekTime()) - return; - - if (!paused) { - auto buffer = resampler->resample(frame.avFrame()); - - if (audioMuted) - // This is somewhat inefficient, but it'll work - memset(buffer.data<char>(), 0, buffer.byteCount()); - - bytesWritten = audioDevice->write(buffer.constData<char>(), buffer.byteCount()); - if (bytesWritten < buffer.byteCount()) { - bufferedData = buffer; - bufferWritten = bytesWritten; - } - - processedUSecs = audioSink->processedUSecs(); - } - } - - qint64 duration = format.durationForBytes(bytesWritten); - writtenUSecs += duration; - - timeOut = (writtenUSecs - processedUSecs - latencyUSecs)/1000; - if (timeOut < 0) - // Don't use a zero timeout if the sink didn't want any more data, rather wait for 10ms. - timeOut = bytesWritten > 0 ? 0 : 10; - -// if (!bufferedData.isEmpty()) -// qDebug() << ">>>>>>>>>>>>>>>>>>>>>>>> could not write all data" << (bufferedData.size() - bufferWritten); -// qDebug() << "Audio: processed" << processedUSecs << "written" << writtenUSecs -// << "delta" << (writtenUSecs - processedUSecs) << "timeOut" << timeOut; -// qDebug() << " updating time to" << currentTimeNoLock(); - timeUpdated(audioBaseTime + (processedUSecs - processedBase)*playbackRate()); -} - -void AudioRenderer::streamChanged() -{ - // mutex is already locked - deviceChanged = true; -} - -void AudioRenderer::updateAudio() -{ - QMutexLocker locker(&mutex); - deviceChanged = true; -} - -Decoder::Decoder() -{ -} - -Decoder::~Decoder() -{ - pause(); - if (videoRenderer) - videoRenderer->kill(); - if (audioRenderer) - audioRenderer->kill(); - if (demuxer) - demuxer->kill(); -} - -static int read(void *opaque, uint8_t *buf, int buf_size) -{ - auto *dev = static_cast<QIODevice *>(opaque); - if (dev->atEnd()) - return AVERROR_EOF; - return dev->read(reinterpret_cast<char *>(buf), buf_size); -} - -static int64_t seek(void *opaque, int64_t offset, int whence) -{ - QIODevice *dev = static_cast<QIODevice *>(opaque); - - if (dev->isSequential()) - return AVERROR(EINVAL); - - if (whence & AVSEEK_SIZE) - return dev->size(); - - whence &= ~AVSEEK_FORCE; - - if (whence == SEEK_CUR) - offset += dev->pos(); - else if (whence == SEEK_END) - offset += dev->size(); - - if (!dev->seek(offset)) - return AVERROR(EINVAL); - return offset; -} - -void Decoder::setMedia(const QUrl &media, QIODevice *stream) -{ - QByteArray url = media.toEncoded(QUrl::PreferLocalFile); - - AVFormatContext *context = nullptr; - - if (stream) { - if (!stream->isOpen()) { - if (!stream->open(QIODevice::ReadOnly)) { - emitError(QMediaPlayer::ResourceError, QLatin1String("Could not open source device.")); - return; - } - } - if (!stream->isSequential()) - stream->seek(0); - context = avformat_alloc_context(); - constexpr int bufferSize = 32768; - unsigned char *buffer = (unsigned char *)av_malloc(bufferSize); - context->pb = avio_alloc_context(buffer, bufferSize, false, stream, ::read, nullptr, ::seek); - } - - int ret = avformat_open_input(&context, url.constData(), nullptr, nullptr); - if (ret < 0) { - auto code = QMediaPlayer::ResourceError; - if (ret == AVERROR(EACCES)) - code = QMediaPlayer::AccessDeniedError; - else if (ret == AVERROR(EINVAL)) - code = QMediaPlayer::FormatError; - - emitError(code, QMediaPlayer::tr("Could not open file")); - return; - } - - ret = avformat_find_stream_info(context, nullptr); - if (ret < 0) { - emitError(QMediaPlayer::FormatError, QMediaPlayer::tr("Could not find stream information for media file")); - return; - } - -#ifndef QT_NO_DEBUG - av_dump_format(context, 0, url.constData(), 0); -#endif - - m_metaData = QFFmpegMetaData::fromAVMetaData(context->metadata); - m_metaData.insert(QMediaMetaData::FileFormat, - QVariant::fromValue(QFFmpegMediaFormatInfo::fileFormatForAVInputFormat(context->iformat))); - - checkStreams(context); - - m_isSeekable = !(context->ctx_flags & AVFMTCTX_UNSEEKABLE); - - demuxer = new Demuxer(this, context); - demuxer->start(); - - qCDebug(qLcDecoder) << ">>>>>> index:" << metaObject()->indexOfSlot("updateCurrentTime(qint64)"); - clockController.setNotify(this, metaObject()->method(metaObject()->indexOfSlot("updateCurrentTime(qint64)"))); -} - -static void insertVideoData(QMediaMetaData &metaData, AVStream *stream) -{ - Q_ASSERT(stream); - auto *codecPar = stream->codecpar; - metaData.insert(QMediaMetaData::VideoBitRate, (int)codecPar->bit_rate); - metaData.insert(QMediaMetaData::VideoCodec, QVariant::fromValue(QFFmpegMediaFormatInfo::videoCodecForAVCodecId(codecPar->codec_id))); - metaData.insert(QMediaMetaData::Resolution, QSize(codecPar->width, codecPar->height)); - metaData.insert(QMediaMetaData::VideoFrameRate, - qreal(stream->avg_frame_rate.num)/qreal(stream->avg_frame_rate.den)); -}; - -static void insertAudioData(QMediaMetaData &metaData, AVStream *stream) -{ - Q_ASSERT(stream); - auto *codecPar = stream->codecpar; - metaData.insert(QMediaMetaData::AudioBitRate, (int)codecPar->bit_rate); - metaData.insert(QMediaMetaData::AudioCodec, - QVariant::fromValue(QFFmpegMediaFormatInfo::audioCodecForAVCodecId(codecPar->codec_id))); -}; - -void Decoder::checkStreams(AVFormatContext *context) -{ - qint64 duration = 0; - AVStream *firstAudioStream = nullptr; - AVStream *defaultAudioStream = nullptr; - AVStream *firstVideoStream = nullptr; - AVStream *defaultVideoStream = nullptr; - - for (unsigned int i = 0; i < context->nb_streams; ++i) { - auto *stream = context->streams[i]; - - QMediaMetaData metaData = QFFmpegMetaData::fromAVMetaData(stream->metadata); - QPlatformMediaPlayer::TrackType type = QPlatformMediaPlayer::VideoStream; - auto *codecPar = stream->codecpar; - - bool isDefault = stream->disposition & AV_DISPOSITION_DEFAULT; - switch (codecPar->codec_type) { - case AVMEDIA_TYPE_UNKNOWN: - case AVMEDIA_TYPE_DATA: ///< Opaque data information usually continuous - case AVMEDIA_TYPE_ATTACHMENT: ///< Opaque data information usually sparse - case AVMEDIA_TYPE_NB: - continue; - case AVMEDIA_TYPE_VIDEO: - type = QPlatformMediaPlayer::VideoStream; - insertVideoData(metaData, stream); - if (!firstVideoStream) - firstVideoStream = stream; - if (isDefault && !defaultVideoStream) - defaultVideoStream = stream; - break; - case AVMEDIA_TYPE_AUDIO: - type = QPlatformMediaPlayer::AudioStream; - insertAudioData(metaData, stream); - if (!firstAudioStream) - firstAudioStream = stream; - if (isDefault && !defaultAudioStream) - defaultAudioStream = stream; - break; - case AVMEDIA_TYPE_SUBTITLE: - type = QPlatformMediaPlayer::SubtitleStream; - break; - } - if (isDefault && m_requestedStreams[type] < 0) - m_requestedStreams[type] = m_streamMap[type].size(); - - m_streamMap[type].append({ (int)i, isDefault, metaData }); - duration = qMax(duration, 1000000*stream->duration*stream->time_base.num/stream->time_base.den); - } - - if (m_requestedStreams[QPlatformMediaPlayer::VideoStream] < 0 && m_streamMap[QPlatformMediaPlayer::VideoStream].size()) { - m_requestedStreams[QPlatformMediaPlayer::VideoStream] = 0; - defaultVideoStream = firstVideoStream; - } - if (m_requestedStreams[QPlatformMediaPlayer::AudioStream] < 0 && m_streamMap[QPlatformMediaPlayer::AudioStream].size()) { - m_requestedStreams[QPlatformMediaPlayer::AudioStream] = 0; - defaultAudioStream = firstAudioStream; - } - if (defaultVideoStream) { - insertVideoData(m_metaData, defaultVideoStream); - m_currentAVStreamIndex[QPlatformMediaPlayer::VideoStream] = defaultVideoStream->index; - } - if (defaultAudioStream) { - insertAudioData(m_metaData, defaultAudioStream); - m_currentAVStreamIndex[QPlatformMediaPlayer::AudioStream] = defaultAudioStream->index; - } - m_requestedStreams[QPlatformMediaPlayer::SubtitleStream] = -1; - m_currentAVStreamIndex[QPlatformMediaPlayer::SubtitleStream] = -1; - - if (player) - player->tracksChanged(); - - if (m_duration != duration) { - m_duration = duration; - if (player) - player->durationChanged(duration/1000); - else if (audioDecoder) - audioDecoder->durationChanged(duration/1000); - } -} - -int Decoder::activeTrack(QPlatformMediaPlayer::TrackType type) -{ - return m_requestedStreams[type]; -} - -void Decoder::setActiveTrack(QPlatformMediaPlayer::TrackType type, int streamNumber) -{ - if (streamNumber < 0 || streamNumber >= m_streamMap[type].size()) - streamNumber = -1; - if (m_requestedStreams[type] == streamNumber) - return; - m_requestedStreams[type] = streamNumber; - int avStreamIndex = m_streamMap[type].value(streamNumber).avStreamIndex; - changeAVTrack(type, avStreamIndex); -} - -void Decoder::error(int errorCode, const QString &errorString) -{ - QMetaObject::invokeMethod(this, "emitError", Q_ARG(int, errorCode), Q_ARG(QString, errorString)); -} - -void Decoder::emitError(int error, const QString &errorString) -{ - if (player) - player->error(error, errorString); - else if (audioDecoder) { - // unfortunately the error enums for QAudioDecoder and QMediaPlayer aren't identical. - // Map them. - switch (QMediaPlayer::Error(error)) { - case QMediaPlayer::NoError: - error = QAudioDecoder::NoError; - break; - case QMediaPlayer::ResourceError: - error = QAudioDecoder::ResourceError; - break; - case QMediaPlayer::FormatError: - error = QAudioDecoder::FormatError; - break; - case QMediaPlayer::NetworkError: - // fall through, Network error doesn't exist in QAudioDecoder - case QMediaPlayer::AccessDeniedError: - error = QAudioDecoder::AccessDeniedError; - break; - } - - audioDecoder->error(error, errorString); - } -} - -void Decoder::setState(QMediaPlayer::PlaybackState state) -{ - if (m_state == state) - return; - - switch (state) { - case QMediaPlayer::StoppedState: - qCDebug(qLcDecoder) << "Decoder::stop"; - setPaused(true); - if (demuxer) - demuxer->stopDecoding(); - seek(0); - if (videoSink) - videoSink->setVideoFrame({}); - updateCurrentTime(0); - qCDebug(qLcDecoder) << "Decoder::stop: done"; - break; - case QMediaPlayer::PausedState: - qCDebug(qLcDecoder) << "Decoder::pause"; - setPaused(true); - if (demuxer) { - demuxer->startDecoding(); - demuxer->wake(); - if (m_state == QMediaPlayer::StoppedState) - triggerStep(); - } - break; - case QMediaPlayer::PlayingState: - qCDebug(qLcDecoder) << "Decoder::play"; - setPaused(false); - if (demuxer) - demuxer->startDecoding(); - break; - } - m_state = state; -} - -void Decoder::setPaused(bool b) -{ - clockController.setPaused(b); -} - -void Decoder::triggerStep() -{ - if (audioRenderer) - audioRenderer->singleStep(); - if (videoRenderer) - videoRenderer->singleStep(); -} - -void Decoder::setVideoSink(QVideoSink *sink) -{ - qCDebug(qLcDecoder) << "setVideoSink" << sink; - if (sink == videoSink) - return; - videoSink = sink; - if (!videoSink || m_currentAVStreamIndex[QPlatformMediaPlayer::VideoStream] < 0) { - if (videoRenderer) { - videoRenderer->kill(); - videoRenderer = nullptr; - } - } else if (!videoRenderer) { - videoRenderer = new VideoRenderer(this, sink); - connect(audioRenderer, &Renderer::atEnd, this, &Decoder::streamAtEnd); - videoRenderer->start(); - StreamDecoder *stream = demuxer->addStream(m_currentAVStreamIndex[QPlatformMediaPlayer::VideoStream]); - videoRenderer->setStream(stream); - stream = demuxer->addStream(m_currentAVStreamIndex[QPlatformMediaPlayer::SubtitleStream]); - videoRenderer->setSubtitleStream(stream); - } -} - -void Decoder::setAudioSink(QPlatformAudioOutput *output) -{ - if (audioOutput == output) - return; - - qCDebug(qLcDecoder) << "setAudioSink" << audioOutput; - audioOutput = output; - if (!output || m_currentAVStreamIndex[QPlatformMediaPlayer::AudioStream] < 0) { - if (audioRenderer) { - audioRenderer->kill(); - audioRenderer = nullptr; - } - } else if (!audioRenderer) { - audioRenderer = new AudioRenderer(this, output->q); - connect(audioRenderer, &Renderer::atEnd, this, &Decoder::streamAtEnd); - audioRenderer->start(); - auto *stream = demuxer->addStream(m_currentAVStreamIndex[QPlatformMediaPlayer::AudioStream]); - audioRenderer->setStream(stream); - } -} - -void Decoder::changeAVTrack(QPlatformMediaPlayer::TrackType type, int streamIndex) -{ - int oldIndex = m_currentAVStreamIndex[type]; - qCDebug(qLcDecoder) << ">>>>> change track" << type << "from" << oldIndex << "to" << streamIndex << clockController.currentTime(); - m_currentAVStreamIndex[type] = streamIndex; - if (!demuxer) - return; - qCDebug(qLcDecoder) << " applying to renderer."; - if (m_state == QMediaPlayer::PlayingState) - setPaused(true); - auto *streamDecoder = demuxer->addStream(streamIndex); - switch (type) { - case QPlatformMediaPlayer::AudioStream: - audioRenderer->setStream(streamDecoder); - break; - case QPlatformMediaPlayer::VideoStream: - videoRenderer->setStream(streamDecoder); - break; - case QPlatformMediaPlayer::SubtitleStream: - videoRenderer->setSubtitleStream(streamDecoder); - break; - default: - Q_UNREACHABLE(); - } - demuxer->seek(clockController.currentTime()); - if (m_state == QMediaPlayer::PlayingState) - setPaused(false); - else - triggerStep(); -} - -QPlatformMediaPlayer::TrackType trackType(AVMediaType mediaType) -{ - switch (mediaType) { - case AVMEDIA_TYPE_VIDEO: - return QPlatformMediaPlayer::VideoStream; - case AVMEDIA_TYPE_AUDIO: - return QPlatformMediaPlayer::AudioStream; - case AVMEDIA_TYPE_SUBTITLE: - return QPlatformMediaPlayer::SubtitleStream; - default: - break; - } - return QPlatformMediaPlayer::NTrackTypes; -} - -void Decoder::seek(qint64 pos) -{ - if (!demuxer) - return; - pos = qBound(0, pos, m_duration); - demuxer->seek(pos); - clockController.syncTo(pos); - if (player) - player->positionChanged(pos/1000); - demuxer->wake(); - if (m_state == QMediaPlayer::PausedState) - triggerStep(); -} - -void Decoder::setPlaybackRate(float rate) -{ - if (m_state == QMediaPlayer::PlayingState) - setPaused(true); - clockController.setPlaybackRate(rate); - if (m_state == QMediaPlayer::PlayingState) - setPaused(false); -} - -void Decoder::updateCurrentTime(qint64 time) -{ - if (player) - player->positionChanged(time/1000); -} - -void Decoder::streamAtEnd() -{ - if (audioRenderer && !audioRenderer->isAtEnd()) - return; - if (videoRenderer && !videoRenderer->isAtEnd()) - return; - pause(); - // take a local copy, as the signals below could lead to this object being deleted - auto *p = player; - if (p) { - p->positionChanged(m_duration/1000); - p->stateChanged(QMediaPlayer::StoppedState); - p->mediaStatusChanged(QMediaPlayer::EndOfMedia); - } -} - -QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegdecoder_p.h b/src/plugins/multimedia/ffmpeg/qffmpegdecoder_p.h deleted file mode 100644 index 414524838..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegdecoder_p.h +++ /dev/null @@ -1,548 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef QFFMPEGDECODER_P_H -#define QFFMPEGDECODER_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "qffmpegthread_p.h" -#include "qffmpeg_p.h" -#include "qffmpegmediaplayer_p.h" -#include "qffmpeghwaccel_p.h" -#include "qffmpegclock_p.h" -#include "qaudiobuffer.h" -#include "qffmpegresampler_p.h" - -#include <qshareddata.h> -#include <qtimer.h> -#include <qqueue.h> - -QT_BEGIN_NAMESPACE - -class QAudioSink; -class QFFmpegAudioDecoder; -class QFFmpegMediaPlayer; - -namespace QFFmpeg -{ - -class Resampler; - -// queue up max 16M of encoded data, that should always be enough -// (it's around 2 secs of 4K HDR video, longer for almost all other formats) -enum { MaxQueueSize = 16*1024*1024 }; - -struct Packet -{ - struct Data { - Data(AVPacket *p) - : packet(p) - {} - ~Data() { - if (packet) - av_packet_free(&packet); - } - QAtomicInt ref; - AVPacket *packet = nullptr; - }; - Packet() = default; - Packet(AVPacket *p) - : d(new Data(p)) - {} - - bool isValid() const { return !!d; } - AVPacket *avPacket() const { return d->packet; } -private: - QExplicitlySharedDataPointer<Data> d; -}; - -struct Codec -{ - struct Data { - Data(AVCodecContext *context, AVStream *stream, const QFFmpeg::HWAccel &hwAccel); - ~Data(); - QAtomicInt ref; - AVCodecContext *context = nullptr; - AVStream *stream = nullptr; - QFFmpeg::HWAccel hwAccel; - int streamIndex = -1; - }; - - Codec() = default; - Codec(AVFormatContext *format, int streamIndex); - bool isValid() const { return !!d; } - - AVCodecContext *context() const { return d->context; } - AVStream *stream() const { return d->stream; } - uint streamIndex() const { return d->stream->index; } - HWAccel hwAccel() const { return d->hwAccel; } - qint64 toMs(qint64 ts) const { return timeStamp(ts, d->stream->time_base); } - qint64 toUs(qint64 ts) const { return timeStampUs(ts, d->stream->time_base); } - -private: - QExplicitlySharedDataPointer<Data> d; -}; - - -struct Frame -{ - struct Data { - Data(AVFrame *f, const Codec &codec, qint64 pts) - : codec(codec) - , frame(f) - , pts(pts) - {} - Data(const QString &text, qint64 pts, qint64 duration) - : text(text), pts(pts), duration(duration) - {} - ~Data() { - if (frame) - av_frame_free(&frame); - } - QAtomicInt ref; - Codec codec; - AVFrame *frame = nullptr; - QString text; - qint64 pts = -1; - qint64 duration = -1; - }; - Frame() = default; - Frame(AVFrame *f, const Codec &codec, qint64 pts) - : d(new Data(f, codec, pts)) - {} - Frame(const QString &text, qint64 pts, qint64 duration) - : d(new Data(text, pts, duration)) - {} - bool isValid() const { return !!d; } - - AVFrame *avFrame() const { return d->frame; } - AVFrame *takeAVFrame() const { - AVFrame *f = d->frame; - d->frame = nullptr; - return f; - } - const Codec *codec() const { return &d->codec; } - qint64 pts() const { return d->pts; } - qint64 duration() const { return d->duration; } - qint64 end() const { return d->pts + d->duration; } - QString text() const { return d->text; } -private: - QExplicitlySharedDataPointer<Data> d; -}; - -class Demuxer; -class StreamDecoder; -class Renderer; -class AudioRenderer; -class VideoRenderer; - -class Decoder : public QObject -{ - Q_OBJECT -public: - Decoder(); - Decoder(QFFmpegMediaPlayer *player) - : player(player) - { - } - Decoder(QFFmpegAudioDecoder *decoder) - : audioDecoder(decoder) - { - } - ~Decoder(); - - void setMedia(const QUrl &media, QIODevice *stream); - - void init(); - void setState(QMediaPlayer::PlaybackState state); - void play() { - setState(QMediaPlayer::PlayingState); - } - void pause() { - setState(QMediaPlayer::PausedState); - } - void stop() { - setState(QMediaPlayer::StoppedState); - } - - void triggerStep(); - - void setVideoSink(QVideoSink *sink); - void setAudioSink(QPlatformAudioOutput *output); - - void changeAVTrack(QPlatformMediaPlayer::TrackType type, int index); - - void seek(qint64 pos); - void setPlaybackRate(float rate); - - void checkStreams(AVFormatContext *context); - - int activeTrack(QPlatformMediaPlayer::TrackType type); - void setActiveTrack(QPlatformMediaPlayer::TrackType type, int streamNumber); - - bool isSeekable() const - { - return m_isSeekable; - } - - // threadsafe - void error(int errorCode, const QString &errorString); - -public Q_SLOTS: - void emitError(int error, const QString &errorString); - void updateCurrentTime(qint64 time); - void streamAtEnd(); - -public: - - // Accessed from multiple threads, but API is threadsafe - ClockController clockController; - -private: - void setPaused(bool b); - -protected: - friend QFFmpegMediaPlayer; - - QFFmpegMediaPlayer *player = nullptr; - QFFmpegAudioDecoder *audioDecoder = nullptr; - - QMediaPlayer::PlaybackState m_state = QMediaPlayer::StoppedState; - bool m_isSeekable = false; - - Demuxer *demuxer = nullptr; - int m_currentAVStreamIndex[QPlatformMediaPlayer::NTrackTypes] = { -1, -1, -1 }; - - QVideoSink *videoSink = nullptr; - Renderer *videoRenderer = nullptr; - - QPlatformAudioOutput *audioOutput = nullptr; - Renderer *audioRenderer = nullptr; - - bool playing = false; - - struct StreamInfo { - int avStreamIndex = -1; - bool isDefault = false; - QMediaMetaData metaData; - }; - - QList<StreamInfo> m_streamMap[QPlatformMediaPlayer::NTrackTypes]; - int m_requestedStreams[3] = { -1, -1, -1 }; - qint64 m_duration = 0; - QMediaMetaData m_metaData; -}; - -class Demuxer : public Thread -{ - Q_OBJECT -public: - Demuxer(Decoder *decoder, AVFormatContext *context); - ~Demuxer(); - - StreamDecoder *addStream(int streamIndex); - void removeStream(int streamIndex); - - bool isStopped() const - { - return m_isStopped.loadRelaxed(); - } - void startDecoding() - { - m_isStopped.storeRelaxed(false); - updateEnabledStreams(); - wake(); - } - void stopDecoding(); - - int seek(qint64 pos); - -private: - void updateEnabledStreams(); - void sendFinalPacketToStreams(); - - void init() override; - void cleanup() override; - bool shouldWait() const override; - void loop() override; - - Decoder *decoder; - AVFormatContext *context = nullptr; - QList<StreamDecoder *> streamDecoders; - - QAtomicInteger<bool> m_isStopped = true; - qint64 last_pts = -1; -}; - - -class StreamDecoder : public Thread -{ - Q_OBJECT -protected: - Demuxer *demuxer = nullptr; - Renderer *m_renderer = nullptr; - - struct PacketQueue { - mutable QMutex mutex; - QQueue<Packet> queue; - qint64 size = 0; - qint64 duration = 0; - }; - PacketQueue packetQueue; - - struct FrameQueue { - mutable QMutex mutex; - QQueue<Frame> queue; - int maxSize = 3; - }; - FrameQueue frameQueue; - QAtomicInteger<bool> eos = false; - bool decoderHasNoFrames = false; - -public: - StreamDecoder(Demuxer *demuxer, const Codec &codec); - - void addPacket(AVPacket *packet); - - qint64 queuedPacketSize() const { - QMutexLocker locker(&packetQueue.mutex); - return packetQueue.size; - } - qint64 queuedDuration() const { - QMutexLocker locker(&packetQueue.mutex); - return packetQueue.duration; - } - - const Frame *lockAndPeekFrame() - { - frameQueue.mutex.lock(); - return frameQueue.queue.isEmpty() ? nullptr : &frameQueue.queue.first(); - } - void removePeekedFrame() - { - frameQueue.queue.takeFirst(); - wake(); - } - void unlockAndReleaseFrame() - { - frameQueue.mutex.unlock(); - } - Frame takeFrame(); - - void flush(); - - Codec codec; - - void setRenderer(Renderer *r); - Renderer *renderer() const { return m_renderer; } - - bool isAtEnd() const { return eos.loadAcquire(); } - - void killHelper() override; - -private: - Packet takePacket(); - Packet peekPacket(); - - void addFrame(const Frame &f); - - bool hasEnoughFrames() const - { - QMutexLocker locker(&frameQueue.mutex); - return frameQueue.queue.size() >= frameQueue.maxSize; - } - bool hasNoPackets() const - { - QMutexLocker locker(&packetQueue.mutex); - return packetQueue.queue.isEmpty(); - } - - void init() override; - bool shouldWait() const override; - void loop() override; - - void decode(); - void decodeSubtitle(); - - QPlatformMediaPlayer::TrackType type() const; -}; - -class Renderer : public Thread -{ - Q_OBJECT -protected: - QPlatformMediaPlayer::TrackType type; - - bool step = false; - bool paused = true; - StreamDecoder *streamDecoder = nullptr; - QAtomicInteger<bool> eos = false; - -public: - Renderer(QPlatformMediaPlayer::TrackType type); - - void setPaused(bool p) { - QMutexLocker locker(&mutex); - paused = p; - if (!p) - wake(); - } - void singleStep() { - QMutexLocker locker(&mutex); - if (!paused) - return; - step = true; - wake(); - } - void doneStep() { - step = false; - } - bool isAtEnd() { return !streamDecoder || eos.loadAcquire(); } - - void setStream(StreamDecoder *stream); - virtual void setSubtitleStream(StreamDecoder *) {} - - void killHelper() override; - - virtual void streamChanged() {} - -Q_SIGNALS: - void atEnd(); - -protected: - bool shouldWait() const override; - -public: -}; - -class ClockedRenderer : public Renderer, public Clock -{ -public: - ClockedRenderer(Decoder *decoder, QPlatformMediaPlayer::TrackType type) - : Renderer(type) - , Clock(&decoder->clockController) - { - } - ~ClockedRenderer() - { - } - void setPaused(bool paused) override; -}; - -class VideoRenderer : public ClockedRenderer -{ - Q_OBJECT - - StreamDecoder *subtitleStreamDecoder = nullptr; -public: - VideoRenderer(Decoder *decoder, QVideoSink *sink); - - void killHelper() override; - - void setSubtitleStream(StreamDecoder *stream) override; -private: - - void init() override; - void loop() override; - - QVideoSink *sink; -}; - -class AudioRenderer : public ClockedRenderer -{ - Q_OBJECT -public: - AudioRenderer(Decoder *decoder, QAudioOutput *output); - ~AudioRenderer() = default; - - // Clock interface - void syncTo(qint64 usecs) override; - void setPlaybackRate(float rate, qint64 currentTime) override; - -private slots: - void updateAudio(); - -private: - void updateOutput(const Codec *codec); - void freeOutput(); - - void init() override; - void cleanup() override; - void loop() override; - void streamChanged() override; - Type type() const override { return AudioClock; } - - int outputSamples(int inputSamples) { - return qRound(inputSamples/playbackRate()); - } - - // Used for timing update calculations based on processed data - qint64 audioBaseTime = 0; - qint64 processedBase = 0; - qint64 processedUSecs = 0; - - bool deviceChanged = false; - QAudioOutput *output = nullptr; - bool audioMuted = false; - qint64 writtenUSecs = 0; - qint64 latencyUSecs = 0; - - QAudioFormat format; - QAudioSink *audioSink = nullptr; - QIODevice *audioDevice = nullptr; - std::unique_ptr<Resampler> resampler; - QAudioBuffer bufferedData; - qsizetype bufferWritten = 0; -}; - -} - -QT_END_NAMESPACE - -#endif - diff --git a/src/plugins/multimedia/ffmpeg/qffmpegdefs_p.h b/src/plugins/multimedia/ffmpeg/qffmpegdefs_p.h new file mode 100644 index 000000000..239d8ff0c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegdefs_p.h @@ -0,0 +1,41 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGDEFS_P_H +#define QFFMPEGDEFS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +extern "C" { +#include <libavformat/avformat.h> +#include <libavcodec/avcodec.h> +#include <libswresample/swresample.h> +#include <libavutil/avutil.h> +#include <libswscale/swscale.h> +} + +#define QT_FFMPEG_OLD_CHANNEL_LAYOUT (LIBAVCODEC_VERSION_INT < AV_VERSION_INT(59, 24, 100)) +#define QT_FFMPEG_HAS_VULKAN \ + (LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(58, 91, 100)) // since ffmpeg n4.3 +#define QT_FFMPEG_HAS_FRAME_TIME_BASE \ + (LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(59, 18, 100)) // since ffmpeg n5.0 +#define QT_FFMPEG_HAS_FRAME_DURATION \ + (LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(60, 3, 100)) // since ffmpeg n6.0 +#define QT_FFMPEG_STREAM_SIDE_DATA_DEPRECATED \ + (LIBAVCODEC_VERSION_INT >= AV_VERSION_INT(60, 15, 100)) // since ffmpeg n6.1 +#define QT_FFMPEG_HAS_D3D12VA \ + (LIBAVUTIL_VERSION_INT >= AV_VERSION_INT(59, 8, 100)) // since ffmpeg n7.0 +#define QT_FFMPEG_SWR_CONST_CH_LAYOUT (LIBSWRESAMPLE_VERSION_INT >= AV_VERSION_INT(4, 9, 100)) +#define QT_FFMPEG_AVIO_WRITE_CONST \ + (LIBAVFORMAT_VERSION_MAJOR >= 61) + +#endif // QFFMPEGDEFS_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp deleted file mode 100644 index f8fa064bc..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegencoder.cpp +++ /dev/null @@ -1,567 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#include "qffmpegencoder_p.h" -#include "qffmpegmediaformatinfo_p.h" -#include "qffmpegvideoframeencoder_p.h" -#include "private/qmultimediautils_p.h" - -#include <qdebug.h> -#include <qiodevice.h> -#include <qaudiosource.h> -#include <qaudiobuffer.h> -#include "qffmpegaudioinput_p.h" -#include <private/qplatformcamera_p.h> -#include "qffmpegvideobuffer_p.h" -#include "qffmpegmediametadata_p.h" -#include "qffmpegencoderoptions_p.h" - -#include <qloggingcategory.h> - -extern "C" { -#include <libavutil/pixdesc.h> -#include <libavutil/common.h> -} - -QT_BEGIN_NAMESPACE - -Q_LOGGING_CATEGORY(qLcFFmpegEncoder, "qt.multimedia.ffmpeg.encoder") - -namespace QFFmpeg -{ - -Encoder::Encoder(const QMediaEncoderSettings &settings, const QUrl &url) - : settings(settings) -{ - const AVOutputFormat *avFormat = QFFmpegMediaFormatInfo::outputFormatForFileFormat(settings.fileFormat()); - - formatContext = avformat_alloc_context(); - formatContext->oformat = const_cast<AVOutputFormat *>(avFormat); // constness varies - - QByteArray encoded = url.toEncoded(); - formatContext->url = (char *)av_malloc(encoded.size() + 1); - memcpy(formatContext->url, encoded.constData(), encoded.size() + 1); - formatContext->pb = nullptr; - avio_open2(&formatContext->pb, formatContext->url, AVIO_FLAG_WRITE, nullptr, nullptr); - qCDebug(qLcFFmpegEncoder) << "opened" << formatContext->url; - - muxer = new Muxer(this); -} - -Encoder::~Encoder() -{ -} - -void Encoder::addAudioInput(QFFmpegAudioInput *input) -{ - audioEncode = new AudioEncoder(this, input, settings); - connect(input, &QFFmpegAudioInput::newAudioBuffer, this, &Encoder::newAudioBuffer); - input->setRunning(true); -} - -void Encoder::addVideoSource(QPlatformCamera *source) -{ - videoEncode = new VideoEncoder(this, source, settings); - connect(source, &QPlatformCamera::newVideoFrame, this, &Encoder::newVideoFrame); -} - -void Encoder::start() -{ - qCDebug(qLcFFmpegEncoder) << "Encoder::start!"; - - formatContext->metadata = QFFmpegMetaData::toAVMetaData(metaData); - - int res = avformat_write_header(formatContext, nullptr); - if (res < 0) - qWarning() << "could not write header" << res; - - muxer->start(); - if (audioEncode) - audioEncode->start(); - if (videoEncode) - videoEncode->start(); - isRecording = true; -} - -void EncodingFinalizer::run() -{ - if (encoder->audioEncode) - encoder->audioEncode->kill(); - if (encoder->videoEncode) - encoder->videoEncode->kill(); - encoder->muxer->kill(); - - int res = av_write_trailer(encoder->formatContext); - if (res < 0) - qWarning() << "could not write trailer" << res; - - avformat_free_context(encoder->formatContext); - qDebug() << " done finalizing."; - emit encoder->finalizationDone(); - delete encoder; - deleteLater(); -} - -void Encoder::finalize() -{ - qDebug() << ">>>>>>>>>>>>>>> finalize"; - - isRecording = false; - auto *finalizer = new EncodingFinalizer(this); - finalizer->start(); -} - -void Encoder::setPaused(bool p) -{ - if (audioEncode) - audioEncode->setPaused(p); - if (videoEncode) - videoEncode->setPaused(p); -} - -void Encoder::setMetaData(const QMediaMetaData &metaData) -{ - this->metaData = metaData; -} - -void Encoder::newAudioBuffer(const QAudioBuffer &buffer) -{ - if (audioEncode && isRecording) - audioEncode->addBuffer(buffer); -} - -void Encoder::newVideoFrame(const QVideoFrame &frame) -{ - if (videoEncode && isRecording) - videoEncode->addFrame(frame); -} - -void Encoder::newTimeStamp(qint64 time) -{ - QMutexLocker locker(&timeMutex); - if (time > timeRecorded) { - timeRecorded = time; - emit durationChanged(time); - } -} - -Muxer::Muxer(Encoder *encoder) - : encoder(encoder) -{ - setObjectName(QLatin1String("Muxer")); -} - -void Muxer::addPacket(AVPacket *packet) -{ -// qCDebug(qLcFFmpegEncoder) << "Muxer::addPacket" << packet->pts << packet->stream_index; - QMutexLocker locker(&queueMutex); - packetQueue.enqueue(packet); - wake(); -} - -AVPacket *Muxer::takePacket() -{ - QMutexLocker locker(&queueMutex); - if (packetQueue.isEmpty()) - return nullptr; -// qCDebug(qLcFFmpegEncoder) << "Muxer::takePacket" << packetQueue.first()->pts; - return packetQueue.dequeue(); -} - -void Muxer::init() -{ -} - -void Muxer::cleanup() -{ -} - -bool QFFmpeg::Muxer::shouldWait() const -{ - QMutexLocker locker(&queueMutex); - return packetQueue.isEmpty(); -} - -void Muxer::loop() -{ - auto *packet = takePacket(); -// qCDebug(qLcFFmpegEncoder) << "writing packet to file" << packet->pts << packet->duration << packet->stream_index; - av_interleaved_write_frame(encoder->formatContext, packet); -} - - -static AVSampleFormat bestMatchingSampleFormat(AVSampleFormat requested, const AVSampleFormat *available) -{ - if (!available) - return requested; - - const AVSampleFormat *f = available; - AVSampleFormat best = *f; -/* - enum { - First, - Planar, - Exact, - } score = First; -*/ - for (; *f != AV_SAMPLE_FMT_NONE; ++f) { - qCDebug(qLcFFmpegEncoder) << "format:" << *f; - if (*f == requested) { - best = *f; -// score = Exact; - break; - } - - if (av_get_planar_sample_fmt(requested) == *f) { -// score = Planar; - best = *f; - } - } - return best; -} - -AudioEncoder::AudioEncoder(Encoder *encoder, QFFmpegAudioInput *input, const QMediaEncoderSettings &settings) - : input(input) -{ - this->encoder = encoder; - - setObjectName(QLatin1String("AudioEncoder")); - qCDebug(qLcFFmpegEncoder) << "AudioEncoder" << settings.audioCodec(); - - format = input->device.preferredFormat(); - auto codecID = QFFmpegMediaFormatInfo::codecIdForAudioCodec(settings.audioCodec()); - Q_ASSERT(avformat_query_codec(encoder->formatContext->oformat, codecID, FF_COMPLIANCE_NORMAL)); - - auto *avCodec = avcodec_find_encoder(codecID); - - AVSampleFormat requested = QFFmpegMediaFormatInfo::avSampleFormat(format.sampleFormat()); - AVSampleFormat bestSampleFormat = bestMatchingSampleFormat(requested, avCodec->sample_fmts); - - stream = avformat_new_stream(encoder->formatContext, nullptr); - stream->id = encoder->formatContext->nb_streams - 1; - stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; - stream->codecpar->codec_id = codecID; - stream->codecpar->channel_layout = av_get_default_channel_layout(format.channelCount()); - stream->codecpar->channels = format.channelCount(); - stream->codecpar->sample_rate = format.sampleRate(); - stream->codecpar->frame_size = 1024; - stream->codecpar->format = bestSampleFormat; - stream->time_base = AVRational{ 1, format.sampleRate() }; - - Q_ASSERT(avCodec); - codec = avcodec_alloc_context3(avCodec); - avcodec_parameters_to_context(codec, stream->codecpar); - - AVDictionary *opts = nullptr; - applyAudioEncoderOptions(settings, avCodec->name, codec, &opts); - - int res = avcodec_open2(codec, avCodec, &opts); - qCDebug(qLcFFmpegEncoder) << "audio codec opened" << res; - qCDebug(qLcFFmpegEncoder) << "audio codec params: fmt=" << codec->sample_fmt << "rate=" << codec->sample_rate; - - if (codec->sample_fmt != requested) { - resampler = swr_alloc_set_opts(nullptr, // we're allocating a new context - codec->channel_layout, // out_ch_layout - codec->sample_fmt, // out_sample_fmt - codec->sample_rate, // out_sample_rate - av_get_default_channel_layout(format.channelCount()), // in_ch_layout - requested, // in_sample_fmt - format.sampleRate(), // in_sample_rate - 0, // log_offset - nullptr); - swr_init(resampler); - } -} - -void AudioEncoder::addBuffer(const QAudioBuffer &buffer) -{ - QMutexLocker locker(&queueMutex); - audioBufferQueue.enqueue(buffer); - wake(); -} - -QAudioBuffer AudioEncoder::takeBuffer() -{ - QMutexLocker locker(&queueMutex); - if (audioBufferQueue.isEmpty()) - return QAudioBuffer(); - return audioBufferQueue.dequeue(); -} - -void AudioEncoder::init() -{ - if (input) { - input->setFrameSize(codec->frame_size); - } - qCDebug(qLcFFmpegEncoder) << "AudioEncoder::init started audio device thread."; -} - -void AudioEncoder::cleanup() -{ - while (!audioBufferQueue.isEmpty()) - loop(); - while (avcodec_send_frame(codec, nullptr) == AVERROR(EAGAIN)) - retrievePackets(); - retrievePackets(); -} - -bool AudioEncoder::shouldWait() const -{ - QMutexLocker locker(&queueMutex); - return audioBufferQueue.isEmpty(); -} - -void AudioEncoder::retrievePackets() -{ - while (1) { - AVPacket *packet = av_packet_alloc(); - int ret = avcodec_receive_packet(codec, packet); - if (ret < 0) { - av_packet_unref(packet); - if (ret != AVERROR(EOF)) - break; - if (ret != AVERROR(EAGAIN)) { - char errStr[1024]; - av_strerror(ret, errStr, 1024); - qCDebug(qLcFFmpegEncoder) << "receive packet" << ret << errStr; - } - break; - } - - // qCDebug(qLcFFmpegEncoder) << "writing video packet" << packet->size << packet->pts << timeStamp(packet->pts, stream->time_base) << packet->stream_index; - packet->stream_index = stream->id; - encoder->muxer->addPacket(packet); - } -} - -void AudioEncoder::loop() -{ - QAudioBuffer buffer = takeBuffer(); - if (!buffer.isValid() || paused.loadAcquire()) - return; - -// qCDebug(qLcFFmpegEncoder) << "new audio buffer" << buffer.byteCount() << buffer.format() << buffer.frameCount() << codec->frame_size; - retrievePackets(); - - AVFrame *frame = av_frame_alloc(); - frame->format = codec->sample_fmt; - frame->channel_layout = codec->channel_layout; - frame->channels = codec->channels; - frame->sample_rate = codec->sample_rate; - frame->nb_samples = buffer.frameCount(); - if (frame->nb_samples) - av_frame_get_buffer(frame, 0); - - if (resampler) { - const uint8_t *data = buffer.constData<uint8_t>(); - swr_convert(resampler, frame->extended_data, frame->nb_samples, &data, frame->nb_samples); - } else { - memcpy(frame->buf[0]->data, buffer.constData<uint8_t>(), buffer.byteCount()); - } - - frame->pts = samplesWritten; - samplesWritten += buffer.frameCount(); - - qint64 time = format.durationForFrames(samplesWritten); - encoder->newTimeStamp(time/1000); - -// qCDebug(qLcFFmpegEncoder) << "sending audio frame" << buffer.byteCount() << frame->pts << ((double)buffer.frameCount()/frame->sample_rate); - int ret = avcodec_send_frame(codec, frame); - if (ret < 0) { - char errStr[1024]; - av_strerror(ret, errStr, 1024); -// qCDebug(qLcFFmpegEncoder) << "error sending frame" << ret << errStr; - } -} - -VideoEncoder::VideoEncoder(Encoder *encoder, QPlatformCamera *camera, const QMediaEncoderSettings &settings) - : m_encoderSettings(settings) - , m_camera(camera) -{ - this->encoder = encoder; - - setObjectName(QLatin1String("VideoEncoder")); - qCDebug(qLcFFmpegEncoder) << "VideoEncoder" << settings.videoCodec(); - - auto format = m_camera->cameraFormat(); - auto *hwAccel = static_cast<const QFFmpeg::HWAccel *>(camera->ffmpegHWAccel()); - AVPixelFormat swFormat = QFFmpegVideoBuffer::toAVPixelFormat(format.pixelFormat()); - AVPixelFormat pixelFormat = hwAccel ? hwAccel->hwFormat() : swFormat; - frameEncoder = new VideoFrameEncoder(settings, format.resolution(), format.maxFrameRate(), pixelFormat, swFormat); - frameEncoder->initWithFormatContext(encoder->formatContext); -} - -VideoEncoder::~VideoEncoder() -{ - delete frameEncoder; -} - -void VideoEncoder::addFrame(const QVideoFrame &frame) -{ - QMutexLocker locker(&queueMutex); - videoFrameQueue.enqueue(frame); - wake(); -} - -QVideoFrame VideoEncoder::takeFrame() -{ - QMutexLocker locker(&queueMutex); - if (videoFrameQueue.isEmpty()) - return QVideoFrame(); - return videoFrameQueue.dequeue(); -} - -void VideoEncoder::retrievePackets() -{ - if (!frameEncoder) - return; - while (AVPacket *packet = frameEncoder->retrievePacket()) - encoder->muxer->addPacket(packet); -} - -void VideoEncoder::init() -{ - qCDebug(qLcFFmpegEncoder) << "VideoEncoder::init started video device thread."; - bool ok = frameEncoder->open(); - if (!ok) - encoder->error(QMediaRecorder::ResourceError, "Could not initialize encoder"); -} - -void VideoEncoder::cleanup() -{ - while (!videoFrameQueue.isEmpty()) - loop(); - if (frameEncoder) { - while (frameEncoder->sendFrame(nullptr) == AVERROR(EAGAIN)) - retrievePackets(); - retrievePackets(); - } -} - -bool VideoEncoder::shouldWait() const -{ - QMutexLocker locker(&queueMutex); - return videoFrameQueue.isEmpty(); -} - -struct QVideoFrameHolder -{ - QVideoFrame f; - QImage i; -}; - -static void freeQVideoFrame(void *opaque, uint8_t *) -{ - delete reinterpret_cast<QVideoFrameHolder *>(opaque); -} - -void VideoEncoder::loop() -{ - if (paused.loadAcquire()) - return; - - retrievePackets(); - - auto frame = takeFrame(); - if (!frame.isValid()) - return; - - if (frameEncoder->isNull()) - return; - -// qCDebug(qLcFFmpegEncoder) << "new video buffer" << frame.startTime(); - - AVFrame *avFrame = nullptr; - - auto *videoBuffer = dynamic_cast<QFFmpegVideoBuffer *>(frame.videoBuffer()); - if (videoBuffer) { - // ffmpeg video buffer, let's use the native AVFrame stored in there - auto *hwFrame = videoBuffer->getHWFrame(); - if (hwFrame && hwFrame->format == frameEncoder->sourceFormat()) - avFrame = av_frame_clone(hwFrame); - } - - if (!avFrame) { - frame.map(QVideoFrame::ReadOnly); - auto size = frame.size(); - avFrame = av_frame_alloc(); - avFrame->format = frameEncoder->sourceFormat(); - avFrame->width = size.width(); - avFrame->height = size.height(); - av_frame_get_buffer(avFrame, 0); - - for (int i = 0; i < 4; ++i) { - avFrame->data[i] = const_cast<uint8_t *>(frame.bits(i)); - avFrame->linesize[i] = frame.bytesPerLine(i); - } - - QImage img; - if (frame.pixelFormat() == QVideoFrameFormat::Format_Jpeg) { - // the QImage is cached inside the video frame, so we can take the pointer to the image data here - img = frame.toImage(); - avFrame->data[0] = (uint8_t *)img.bits(); - avFrame->linesize[0] = img.bytesPerLine(); - } - - Q_ASSERT(avFrame->data[0]); - // ensure the video frame and it's data is alive as long as it's being used in the encoder - avFrame->opaque_ref = av_buffer_create(nullptr, 0, freeQVideoFrame, new QVideoFrameHolder{frame, img}, 0); - } - - if (baseTime.loadAcquire() < 0) { - baseTime.storeRelease(frame.startTime() - lastFrameTime); -// qCDebug(qLcFFmpegEncoder) << ">>>> adjusting base time to" << baseTime.loadAcquire() << frame.startTime() << lastFrameTime; - } - - qint64 time = frame.startTime() - baseTime.loadAcquire(); - lastFrameTime = frame.endTime() - baseTime.loadAcquire(); - avFrame->pts = frameEncoder->getPts(time); - - encoder->newTimeStamp(time/1000); - -// qCDebug(qLcFFmpegEncoder) << ">>> sending frame" << avFrame->pts << time; - int ret = frameEncoder->sendFrame(avFrame); - if (ret < 0) { - qCDebug(qLcFFmpegEncoder) << "error sending frame" << ret << err2str(ret); - encoder->error(QMediaRecorder::ResourceError, err2str(ret)); - } -} - -} - -QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencoder_p.h b/src/plugins/multimedia/ffmpeg/qffmpegencoder_p.h deleted file mode 100644 index cc4503c74..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegencoder_p.h +++ /dev/null @@ -1,233 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef QFFMPEGENCODER_P_H -#define QFFMPEGENCODER_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "qffmpegthread_p.h" -#include "qffmpeg_p.h" -#include "qffmpeghwaccel_p.h" - -#include <private/qplatformmediarecorder_p.h> -#include <qaudioformat.h> -#include <qaudiobuffer.h> - -#include <qqueue.h> - -QT_BEGIN_NAMESPACE - -class QFFmpegAudioInput; -class QVideoFrame; -class QPlatformCamera; - -namespace QFFmpeg -{ - -class Encoder; -class Muxer; -class AudioEncoder; -class VideoEncoder; -class VideoFrameEncoder; - -class EncodingFinalizer : public QThread -{ -public: - EncodingFinalizer(Encoder *e) - : encoder(e) - {} - void run() override; - - Encoder *encoder = nullptr; -}; - -class Encoder : public QObject -{ - Q_OBJECT -public: - Encoder(const QMediaEncoderSettings &settings, const QUrl &url); - ~Encoder(); - - void addAudioInput(QFFmpegAudioInput *input); - void addVideoSource(QPlatformCamera *source); - - void start(); - void finalize(); - - void setPaused(bool p); - - void setMetaData(const QMediaMetaData &metaData); - -public Q_SLOTS: - void newAudioBuffer(const QAudioBuffer &buffer); - void newVideoFrame(const QVideoFrame &frame); - void newTimeStamp(qint64 time); - -Q_SIGNALS: - void durationChanged(qint64 duration); - void error(QMediaRecorder::Error code, const QString &description); - void finalizationDone(); - -public: - - QMediaEncoderSettings settings; - QMediaMetaData metaData; - AVFormatContext *formatContext = nullptr; - Muxer *muxer = nullptr; - bool isRecording = false; - - AudioEncoder *audioEncode = nullptr; - VideoEncoder *videoEncode = nullptr; - - QMutex timeMutex; - qint64 timeRecorded = 0; -}; - - -class Muxer : public Thread -{ - mutable QMutex queueMutex; - QQueue<AVPacket *> packetQueue; -public: - Muxer(Encoder *encoder); - - void addPacket(AVPacket *); - -private: - AVPacket *takePacket(); - - void init() override; - void cleanup() override; - bool shouldWait() const override; - void loop() override; - - Encoder *encoder; -}; - -class EncoderThread : public Thread -{ -public: - virtual void setPaused(bool b) - { - paused.storeRelease(b); - } - -protected: - QAtomicInteger<bool> paused = false; - Encoder *encoder = nullptr; -}; - -class AudioEncoder : public EncoderThread -{ - mutable QMutex queueMutex; - QQueue<QAudioBuffer> audioBufferQueue; -public: - AudioEncoder(Encoder *encoder, QFFmpegAudioInput *input, const QMediaEncoderSettings &settings); - - void addBuffer(const QAudioBuffer &buffer); - - QFFmpegAudioInput *audioInput() const { return input; } - -private: - QAudioBuffer takeBuffer(); - void retrievePackets(); - - void init() override; - void cleanup() override; - bool shouldWait() const override; - void loop() override; - - AVStream *stream = nullptr; - AVCodecContext *codec = nullptr; - QFFmpegAudioInput *input; - QAudioFormat format; - - SwrContext *resampler = nullptr; - qint64 samplesWritten = 0; -}; - - -class VideoEncoder : public EncoderThread -{ - mutable QMutex queueMutex; - QQueue<QVideoFrame> videoFrameQueue; -public: - VideoEncoder(Encoder *encoder, QPlatformCamera *camera, const QMediaEncoderSettings &settings); - ~VideoEncoder(); - - void addFrame(const QVideoFrame &frame); - - void setPaused(bool b) override - { - EncoderThread::setPaused(b); - if (b) - baseTime.storeRelease(-1); - } - -private: - QVideoFrame takeFrame(); - void retrievePackets(); - - void init() override; - void cleanup() override; - bool shouldWait() const override; - void loop() override; - - QMediaEncoderSettings m_encoderSettings; - QPlatformCamera *m_camera = nullptr; - VideoFrameEncoder *frameEncoder = nullptr; - - QAtomicInteger<qint64> baseTime = -1; - qint64 lastFrameTime = 0; -}; - -} - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencoderoptions_p.h b/src/plugins/multimedia/ffmpeg/qffmpegencoderoptions_p.h deleted file mode 100644 index f6e0907c6..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegencoderoptions_p.h +++ /dev/null @@ -1,68 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef QFFMPEGENCODEROPTIONS_P_H -#define QFFMPEGENCODEROPTIONS_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "qffmpeghwaccel_p.h" -#include "qvideoframeformat.h" -#include "private/qplatformmediarecorder_p.h" - -QT_BEGIN_NAMESPACE - -namespace QFFmpeg { - -void applyVideoEncoderOptions(const QMediaEncoderSettings &settings, const QByteArray &codecName, AVCodecContext *codec, AVDictionary **opts); -void applyAudioEncoderOptions(const QMediaEncoderSettings &settings, const QByteArray &codecName, AVCodecContext *codec, AVDictionary **opts); - -} - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencodingformatcontext.cpp b/src/plugins/multimedia/ffmpeg/qffmpegencodingformatcontext.cpp new file mode 100644 index 000000000..7117d0c02 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegencodingformatcontext.cpp @@ -0,0 +1,116 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegencodingformatcontext_p.h" +#include "qffmpegmediaformatinfo_p.h" +#include "qffmpegioutils_p.h" +#include "qfile.h" +#include "QtCore/qloggingcategory.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +Q_STATIC_LOGGING_CATEGORY(qLcEncodingFormatContext, "qt.multimedia.ffmpeg.encodingformatcontext"); + +namespace { +// In the example https://ffmpeg.org/doxygen/trunk/avio_read_callback_8c-example.html, +// BufferSize = 4096 is suggested, however, it might be not optimal. To be investigated. +constexpr size_t DefaultBufferSize = 4096; +} // namespace + +EncodingFormatContext::EncodingFormatContext(QMediaFormat::FileFormat fileFormat) + : m_avFormatContext(avformat_alloc_context()) +{ + const AVOutputFormat *avFormat = QFFmpegMediaFormatInfo::outputFormatForFileFormat(fileFormat); + m_avFormatContext->oformat = const_cast<AVOutputFormat *>(avFormat); // constness varies +} + +EncodingFormatContext::~EncodingFormatContext() +{ + closeAVIO(); + + avformat_free_context(m_avFormatContext); +} + +void EncodingFormatContext::openAVIO(const QString &filePath) +{ + Q_ASSERT(!isAVIOOpen()); + Q_ASSERT(!filePath.isEmpty()); + + const QByteArray filePathUtf8 = filePath.toUtf8(); + + std::unique_ptr<char, decltype(&av_free)> url( + reinterpret_cast<char *>(av_malloc(filePathUtf8.size() + 1)), &av_free); + memcpy(url.get(), filePathUtf8.constData(), filePathUtf8.size() + 1); + + // Initialize the AVIOContext for accessing the resource indicated by the url + auto result = avio_open2(&m_avFormatContext->pb, url.get(), AVIO_FLAG_WRITE, nullptr, nullptr); + + qCDebug(qLcEncodingFormatContext) + << "opened by file path:" << url.get() << ", result:" << result; + + Q_ASSERT(m_avFormatContext->url == nullptr); + if (isAVIOOpen()) + m_avFormatContext->url = url.release(); + else + openAVIOWithQFile(filePath); +} + +void EncodingFormatContext::openAVIOWithQFile(const QString &filePath) +{ + // QTBUG-123082, To be investigated: + // - should we use the logic with QFile for all file paths? + // - does avio_open2 handle network protocols that QFile doesn't? + // - which buffer size should we set to opening with QFile to ensure the best performance? + + auto file = std::make_unique<QFile>(filePath); + + if (!file->open(QFile::WriteOnly)) { + qCDebug(qLcEncodingFormatContext) << "Cannot open QFile" << filePath; + return; + } + + openAVIO(file.get()); + + if (isAVIOOpen()) + m_outputFile = std::move(file); +} + +void EncodingFormatContext::openAVIO(QIODevice *device) +{ + Q_ASSERT(!isAVIOOpen()); + Q_ASSERT(device); + + if (!device->isWritable()) + return; + + auto buffer = static_cast<uint8_t *>(av_malloc(DefaultBufferSize)); + m_avFormatContext->pb = avio_alloc_context(buffer, DefaultBufferSize, 1, device, nullptr, + &writeQIODevice, &seekQIODevice); +} + +void EncodingFormatContext::closeAVIO() +{ + // Close the AVIOContext and release any file handles + if (isAVIOOpen()) { + if (m_avFormatContext->url && *m_avFormatContext->url != '\0') { + auto closeResult = avio_closep(&m_avFormatContext->pb); + Q_ASSERT(closeResult == 0); + } else { + av_free(std::exchange(m_avFormatContext->pb->buffer, nullptr)); + avio_context_free(&m_avFormatContext->pb); + } + + // delete url even though it might be delete by avformat_free_context to + // ensure consistency in openAVIO/closeAVIO. + av_freep(&m_avFormatContext->url); + m_outputFile.reset(); + } else { + Q_ASSERT(!m_outputFile); + } +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencodingformatcontext_p.h b/src/plugins/multimedia/ffmpeg/qffmpegencodingformatcontext_p.h new file mode 100644 index 000000000..69d0cd873 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegencodingformatcontext_p.h @@ -0,0 +1,60 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGENCODINGFORMATCONTEXT_P_H +#define QFFMPEGENCODINGFORMATCONTEXT_P_H + +#include "qffmpegdefs_p.h" +#include "qmediaformat.h" + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +QT_BEGIN_NAMESPACE + +class QIODevice; +class QFile; + +namespace QFFmpeg { + +class EncodingFormatContext +{ +public: + explicit EncodingFormatContext(QMediaFormat::FileFormat fileFormat); + ~EncodingFormatContext(); + + void openAVIO(const QString &filePath); + + void openAVIO(QIODevice *device); + + bool isAVIOOpen() const { return m_avFormatContext->pb != nullptr; } + + void closeAVIO(); + + AVFormatContext *avFormatContext() { return m_avFormatContext; } + + const AVFormatContext *avFormatContext() const { return m_avFormatContext; } + +private: + Q_DISABLE_COPY_MOVE(EncodingFormatContext) + + void openAVIOWithQFile(const QString &filePath); + +private: + AVFormatContext *m_avFormatContext; + std::unique_ptr<QFile> m_outputFile; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGENCODINGFORMATCONTEXT_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp index e28a89ad8..24e5614ce 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel.cpp @@ -1,41 +1,7 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "libavutil/version.h" #include "qffmpeghwaccel_p.h" #if QT_CONFIG(vaapi) @@ -46,161 +12,350 @@ #endif #if QT_CONFIG(wmf) #include "qffmpeghwaccel_d3d11_p.h" +#include <QtCore/private/qsystemlibrary_p.h> + +#endif +#ifdef Q_OS_ANDROID +# include "qffmpeghwaccel_mediacodec_p.h" #endif #include "qffmpeg_p.h" #include "qffmpegvideobuffer_p.h" - -#include <private/qrhi_p.h> -#include <qdebug.h> +#include "qscopedvaluerollback.h" +#include "QtCore/qfile.h" + +#include <rhi/qrhi.h> +#include <qloggingcategory.h> +#include <unordered_set> +#ifdef Q_OS_LINUX +#include <QLibrary> +#endif /* Infrastructure for HW acceleration goes into this file. */ QT_BEGIN_NAMESPACE -namespace QFFmpeg { +Q_STATIC_LOGGING_CATEGORY(qLHWAccel, "qt.multimedia.ffmpeg.hwaccel"); +extern bool thread_local FFmpegLogsEnabledInThread; -// HW context initialization +namespace QFFmpeg { -// preferred order of HW accelerators to use -static const AVHWDeviceType preferredHardwareAccelerators[] = { -// Linux/Unix -#if defined(Q_OS_LINUX) +static const std::initializer_list<AVHWDeviceType> preferredHardwareAccelerators = { +#if defined(Q_OS_ANDROID) + AV_HWDEVICE_TYPE_MEDIACODEC, +#elif defined(Q_OS_LINUX) + AV_HWDEVICE_TYPE_CUDA, AV_HWDEVICE_TYPE_VAAPI, -// AV_HWDEVICE_TYPE_DRM, + + // TODO: investigate VDPAU advantages. + // nvenc/nvdec codecs use AV_HWDEVICE_TYPE_CUDA by default, but they can also use VDPAU + // if it's included into the ffmpeg build and vdpau drivers are installed. + // AV_HWDEVICE_TYPE_VDPAU #elif defined (Q_OS_WIN) AV_HWDEVICE_TYPE_D3D11VA, #elif defined (Q_OS_DARWIN) AV_HWDEVICE_TYPE_VIDEOTOOLBOX, -#elif defined (Q_OS_ANDROID) - AV_HWDEVICE_TYPE_MEDIACODEC, #endif - AV_HWDEVICE_TYPE_NONE }; -static AVBufferRef *loadHWContext(const AVHWDeviceType type) +static AVBufferUPtr loadHWContext(AVHWDeviceType type) { AVBufferRef *hwContext = nullptr; + qCDebug(qLHWAccel) << " Checking HW context:" << av_hwdevice_get_type_name(type); int ret = av_hwdevice_ctx_create(&hwContext, type, nullptr, nullptr, 0); - qDebug() << " Checking HW context:" << av_hwdevice_get_type_name(type); + if (ret == 0) { - qDebug() << " Using above hw context."; - return hwContext; + qCDebug(qLHWAccel) << " Using above hw context."; + return AVBufferUPtr(hwContext); } - qDebug() << " Could not create hw context:" << ret << strerror(-ret); + qCDebug(qLHWAccel) << " Could not create hw context:" << ret << strerror(-ret); return nullptr; } -static AVBufferRef *hardwareContextForCodec(const AVCodec *codec) +// FFmpeg might crash on loading non-existing hw devices. +// Let's roughly precheck drivers/libraries. +static bool precheckDriver(AVHWDeviceType type) { - qDebug() << "Checking HW acceleration for decoder" << codec->name; - - // First try our preferred accelerators. Those are the ones where we can - // set up a zero copy pipeline - auto *preferred = preferredHardwareAccelerators; - while (*preferred != AV_HWDEVICE_TYPE_NONE) { - for (int i = 0;; ++i) { - const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i); - if (!config) - break; - if (config->device_type == *preferred) { - auto *hwContext = loadHWContext(config->device_type); - if (hwContext) - return hwContext; - break; - } - } - ++preferred; + // precheckings might need some improvements +#if defined(Q_OS_LINUX) + if (type == AV_HWDEVICE_TYPE_CUDA) { + if (!QFile::exists(QLatin1String("/proc/driver/nvidia/version"))) + return false; + + // QTBUG-122199 + // CUDA backend requires libnvcuvid in libavcodec + QLibrary lib("libnvcuvid.so"); + if (!lib.load()) + return false; + lib.unload(); + return true; } +#elif defined(Q_OS_WINDOWS) + if (type == AV_HWDEVICE_TYPE_D3D11VA) + return QSystemLibrary(QLatin1String("d3d11.dll")).load(); - // Ok, let's see if we can get any HW acceleration at all. It'll still involve one buffer copy, - // as we can't move the data into RHI textures without a CPU copy - for (int i = 0;; ++i) { - const AVCodecHWConfig *config = avcodec_get_hw_config(codec, i); - if (!config) - break; +#if QT_FFMPEG_HAS_D3D12VA + if (type == AV_HWDEVICE_TYPE_D3D12VA) + return QSystemLibrary(QLatin1String("d3d12.dll")).load(); +#endif - auto *hwContext = loadHWContext(config->device_type); - if (hwContext) - return hwContext; - } - qDebug() << " No HW accelerators found, using SW decoding."; - return nullptr; + if (type == AV_HWDEVICE_TYPE_DXVA2) + return QSystemLibrary(QLatin1String("d3d9.dll")).load(); + // TODO: check nvenc/nvdec and revisit the checking + if (type == AV_HWDEVICE_TYPE_CUDA) + return QSystemLibrary(QLatin1String("nvml.dll")).load(); +#else + Q_UNUSED(type); +#endif + + return true; } -// Used for the AVCodecContext::get_format callback -AVPixelFormat getFormat(AVCodecContext *s, const AVPixelFormat *fmt) +static bool checkHwType(AVHWDeviceType type) { - // First check HW accelerated codecs, the HW device context must be set - if (s->hw_device_ctx && s->codec->hw_configs) { - auto *device_ctx = (AVHWDeviceContext*)s->hw_device_ctx->data; - for (int i = 0; const AVCodecHWConfig *config = avcodec_get_hw_config(s->codec, i); i++) { - if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) - continue; - if (device_ctx->type != config->device_type) - continue; - for (int n = 0; fmt[n] != AV_PIX_FMT_NONE; n++) { - if (config->pix_fmt == fmt[n]) { -#if QT_CONFIG(wmf) - if (fmt[n] == AV_PIX_FMT_D3D11) - QFFmpeg::D3D11TextureConverter::SetupDecoderTextures(s); -#endif - return fmt[n]; - } - } - } + const auto deviceName = av_hwdevice_get_type_name(type); + if (!deviceName) { + qWarning() << "Internal FFmpeg error, unknow hw type:" << type; + return false; } - // prefer video formats we can handle directly - for (int n = 0; fmt[n] != AV_PIX_FMT_NONE; n++) { - bool needsConversion = true; - QFFmpegVideoBuffer::toQtPixelFormat(fmt[n], &needsConversion); - if (!needsConversion) - return fmt[n]; + if (!precheckDriver(type)) { + qCDebug(qLHWAccel) << "Drivers for hw device" << deviceName << "is not installed"; + return false; } - // take the native format, this will involve one additional format conversion on the CPU side - return *fmt; + if (type == AV_HWDEVICE_TYPE_MEDIACODEC || + type == AV_HWDEVICE_TYPE_VIDEOTOOLBOX || + type == AV_HWDEVICE_TYPE_D3D11VA || +#if QT_FFMPEG_HAS_D3D12VA + type == AV_HWDEVICE_TYPE_D3D12VA || +#endif + type == AV_HWDEVICE_TYPE_DXVA2) + return true; // Don't waste time; it's expected to work fine of the precheck is OK + + + QScopedValueRollback rollback(FFmpegLogsEnabledInThread); + FFmpegLogsEnabledInThread = false; + + return loadHWContext(type) != nullptr; } -TextureConverter::Data::~Data() +static const std::vector<AVHWDeviceType> &deviceTypes() { - delete backend; + static const auto types = []() { + qCDebug(qLHWAccel) << "Check device types"; + QElapsedTimer timer; + timer.start(); + + // gather hw pix formats + std::unordered_set<AVPixelFormat> hwPixFormats; + void *opaque = nullptr; + while (auto codec = av_codec_iterate(&opaque)) { + findAVPixelFormat(codec, [&](AVPixelFormat format) { + if (isHwPixelFormat(format)) + hwPixFormats.insert(format); + return false; + }); + } + + // create a device types list + std::vector<AVHWDeviceType> result; + AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE; + while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE) + if (hwPixFormats.count(pixelFormatForHwDevice(type)) && checkHwType(type)) + result.push_back(type); + result.shrink_to_fit(); + + // reorder the list accordingly preferredHardwareAccelerators + auto it = result.begin(); + for (const auto preffered : preferredHardwareAccelerators) { + auto found = std::find(it, result.end(), preffered); + if (found != result.end()) + std::rotate(it++, found, std::next(found)); + } + + using namespace std::chrono; + qCDebug(qLHWAccel) << "Device types checked. Spent time:" << duration_cast<microseconds>(timer.durationElapsed()); + + return result; + }(); + + return types; } +static std::vector<AVHWDeviceType> deviceTypes(const char *envVarName) +{ + const auto definedDeviceTypes = qgetenv(envVarName); + + if (definedDeviceTypes.isNull()) + return deviceTypes(); + + std::vector<AVHWDeviceType> result; + const auto definedDeviceTypesString = QString::fromUtf8(definedDeviceTypes).toLower(); + for (const auto &deviceType : definedDeviceTypesString.split(',')) { + if (!deviceType.isEmpty()) { + const auto foundType = av_hwdevice_find_type_by_name(deviceType.toUtf8().data()); + if (foundType == AV_HWDEVICE_TYPE_NONE) + qWarning() << "Unknown hw device type" << deviceType; + else + result.emplace_back(foundType); + } + } + result.shrink_to_fit(); + return result; +} -HWAccel::Data::~Data() +template<typename CodecFinder> +std::pair<const AVCodec *, std::unique_ptr<HWAccel>> +findCodecWithHwAccel(AVCodecID id, const std::vector<AVHWDeviceType> &deviceTypes, + CodecFinder codecFinder, + const std::function<bool(const HWAccel &)> &hwAccelPredicate) { - if (hwDeviceContext) - av_buffer_unref(&hwDeviceContext); - if (hwFramesContext) - av_buffer_unref(&hwFramesContext); + for (auto type : deviceTypes) { + const auto codec = codecFinder(id, type, {}); + + if (!codec) + continue; + + qCDebug(qLHWAccel) << "Found potential codec" << codec->name << "for hw accel" << type + << "; Checking the hw device..."; + + auto hwAccel = QFFmpeg::HWAccel::create(type); + + if (!hwAccel) + continue; + + if (hwAccelPredicate && !hwAccelPredicate(*hwAccel)) { + qCDebug(qLHWAccel) << "HW device is available but doesn't suit due to restrictions"; + continue; + } + + qCDebug(qLHWAccel) << "HW device is OK"; + + return { codec, std::move(hwAccel) }; + } + + qCDebug(qLHWAccel) << "No hw acceleration found for codec id" << id; + + return { nullptr, nullptr }; } +static bool isNoConversionFormat(AVPixelFormat f) +{ + bool needsConversion = true; + QFFmpegVideoBuffer::toQtPixelFormat(f, &needsConversion); + return !needsConversion; +}; + +namespace { -HWAccel::HWAccel(const AVCodec *codec) +bool hwTextureConversionEnabled() { - if (codec->type != AVMEDIA_TYPE_VIDEO) - return; - auto *ctx = hardwareContextForCodec(codec); - if (!ctx) - return; - d = new Data; - d->hwDeviceContext = ctx; + + // HW textures conversions are not stable in specific cases, dependent on the hardware and OS. + // We need the env var for testing with no textures conversion on the user's side. + static const int disableHwConversion = + qEnvironmentVariableIntValue("QT_DISABLE_HW_TEXTURES_CONVERSION"); + + return !disableHwConversion; } -HWAccel::HWAccel(AVHWDeviceType deviceType) +void setupDecoder(const AVPixelFormat format, AVCodecContext *const codecContext) { - auto *ctx = loadHWContext(deviceType); - if (!ctx) + if (!hwTextureConversionEnabled()) return; - d = new Data; - d->hwDeviceContext = ctx; + +#if QT_CONFIG(wmf) + if (format == AV_PIX_FMT_D3D11) + QFFmpeg::D3D11TextureConverter::SetupDecoderTextures(codecContext); +#elif defined Q_OS_ANDROID + if (format == AV_PIX_FMT_MEDIACODEC) + QFFmpeg::MediaCodecTextureConverter::setupDecoderSurface(codecContext); +#else + Q_UNUSED(codecContext); + Q_UNUSED(format); +#endif +} + +} // namespace + +// Used for the AVCodecContext::get_format callback +AVPixelFormat getFormat(AVCodecContext *codecContext, const AVPixelFormat *suggestedFormats) +{ + // First check HW accelerated codecs, the HW device context must be set + if (codecContext->hw_device_ctx) { + auto *device_ctx = (AVHWDeviceContext *)codecContext->hw_device_ctx->data; + std::pair formatAndScore(AV_PIX_FMT_NONE, NotSuitableAVScore); + + // to be rewritten via findBestAVFormat + for (int i = 0; + const AVCodecHWConfig *config = avcodec_get_hw_config(codecContext->codec, i); i++) { + if (!(config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX)) + continue; + + if (device_ctx->type != config->device_type) + continue; + + const bool isDeprecated = (config->methods & AV_CODEC_HW_CONFIG_METHOD_AD_HOC) != 0; + const bool shouldCheckCodecFormats = config->pix_fmt == AV_PIX_FMT_NONE; + + auto scoresGettor = [&](AVPixelFormat format) { + // check in supported codec->pix_fmts; + // no reason to use findAVPixelFormat as we're already in the hw_config loop + if (shouldCheckCodecFormats && !hasAVFormat(codecContext->codec->pix_fmts, format)) + return NotSuitableAVScore; + + if (!shouldCheckCodecFormats && config->pix_fmt != format) + return NotSuitableAVScore; + + auto result = DefaultAVScore; + + if (isDeprecated) + result -= 10000; + if (isHwPixelFormat(format)) + result += 10; + + return result; + }; + + const auto found = findBestAVValue(suggestedFormats, scoresGettor); + + if (found.second > formatAndScore.second) + formatAndScore = found; + } + + const auto &format = formatAndScore.first; + if (format != AV_PIX_FMT_NONE) { + setupDecoder(format, codecContext); + qCDebug(qLHWAccel) << "Selected format" << format << "for hw" << device_ctx->type; + return format; + } + } + + // prefer video formats we can handle directly + const auto noConversionFormat = findAVFormat(suggestedFormats, &isNoConversionFormat); + if (noConversionFormat != AV_PIX_FMT_NONE) { + qCDebug(qLHWAccel) << "Selected format with no conversion" << noConversionFormat; + return noConversionFormat; + } + + qCDebug(qLHWAccel) << "Selected format with conversion" << *suggestedFormats; + + // take the native format, this will involve one additional format conversion on the CPU side + return *suggestedFormats; } HWAccel::~HWAccel() = default; +std::unique_ptr<HWAccel> HWAccel::create(AVHWDeviceType deviceType) +{ + if (auto ctx = loadHWContext(deviceType)) + return std::unique_ptr<HWAccel>(new HWAccel(std::move(ctx))); + else + return {}; +} + AVPixelFormat HWAccel::format(AVFrame *frame) { if (!frame->hw_frames_ctx) @@ -211,130 +366,85 @@ AVPixelFormat HWAccel::format(AVFrame *frame) return AVPixelFormat(hwFramesContext->sw_format); } -const AVHWDeviceType *HWAccel::preferredDeviceTypes() +const std::vector<AVHWDeviceType> &HWAccel::encodingDeviceTypes() { - return preferredHardwareAccelerators; + static const auto &result = deviceTypes("QT_FFMPEG_ENCODING_HW_DEVICE_TYPES"); + return result; +} + +const std::vector<AVHWDeviceType> &HWAccel::decodingDeviceTypes() +{ + static const auto &result = deviceTypes("QT_FFMPEG_DECODING_HW_DEVICE_TYPES"); + return result; } AVHWDeviceContext *HWAccel::hwDeviceContext() const { - if (!d || !d->hwDeviceContext) - return nullptr; - return (AVHWDeviceContext *)d->hwDeviceContext->data; + return m_hwDeviceContext ? (AVHWDeviceContext *)m_hwDeviceContext->data : nullptr; } AVPixelFormat HWAccel::hwFormat() const { - switch (deviceType()) { - case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: - return AV_PIX_FMT_VIDEOTOOLBOX; - case AV_HWDEVICE_TYPE_VAAPI: - return AV_PIX_FMT_VAAPI; - default: - return AV_PIX_FMT_NONE; - } + return pixelFormatForHwDevice(deviceType()); } -const AVCodec *HWAccel::hardwareEncoderForCodecId(AVCodecID id) const +const AVHWFramesConstraints *HWAccel::constraints() const { - const char *codec = nullptr; - switch (deviceType()) { -#ifdef Q_OS_DARWIN - case AV_HWDEVICE_TYPE_VIDEOTOOLBOX: - switch (id) { - case AV_CODEC_ID_H264: - codec = "h264_videotoolbox"; - break; - case AV_CODEC_ID_HEVC: - codec = "hevc_videotoolbox"; - break; - case AV_CODEC_ID_PRORES: - codec = "prores_videotoolbox"; - break; - case AV_CODEC_ID_VP9: - codec = "vp9_videotoolbox"; - break; - default: - break; - } - break; -#endif - case AV_HWDEVICE_TYPE_VAAPI: - switch (id) { - case AV_CODEC_ID_H264: - codec = "h264_vaapi"; - break; - case AV_CODEC_ID_HEVC: - codec = "hevc_vaapi"; - break; - case AV_CODEC_ID_MJPEG: - codec = "mjpeg_vaapi"; - break; - case AV_CODEC_ID_MPEG2VIDEO: - codec = "mpeg2_vaapi"; - break; - case AV_CODEC_ID_VP8: - codec = "vp8_vaapi"; - break; - case AV_CODEC_ID_VP9: - codec = "vp9_vaapi"; - break; - default: - break; - } - break; - default: - break; - } - if (!codec) - return nullptr; - const AVCodec *c = avcodec_find_encoder_by_name(codec); - qDebug() << "searching for HW codec" << codec << "got" << c; - return c; + std::call_once(m_constraintsOnceFlag, [this]() { + if (auto context = hwDeviceContextAsBuffer()) + m_constraints.reset(av_hwdevice_get_hwframe_constraints(context, nullptr)); + }); + + return m_constraints.get(); } -HWAccel HWAccel::findHardwareAccelForCodecID(AVCodecID id) +std::pair<const AVCodec *, std::unique_ptr<HWAccel>> +HWAccel::findEncoderWithHwAccel(AVCodecID id, const std::function<bool(const HWAccel &)>& hwAccelPredicate) { - auto *accels = preferredHardwareAccelerators; - while (*accels != AV_HWDEVICE_TYPE_NONE) { - auto accel = HWAccel(*accels); - if (accel.hardwareEncoderForCodecId(id) != nullptr) - return accel; - ++accels; - } - return {}; + auto finder = qOverload<AVCodecID, const std::optional<AVHWDeviceType> &, + const std::optional<PixelOrSampleFormat> &>(&QFFmpeg::findAVEncoder); + return findCodecWithHwAccel(id, encodingDeviceTypes(), finder, hwAccelPredicate); +} + +std::pair<const AVCodec *, std::unique_ptr<HWAccel>> +HWAccel::findDecoderWithHwAccel(AVCodecID id, const std::function<bool(const HWAccel &)>& hwAccelPredicate) +{ + return findCodecWithHwAccel(id, decodingDeviceTypes(), &QFFmpeg::findAVDecoder, + hwAccelPredicate); } AVHWDeviceType HWAccel::deviceType() const { - if (!d || !d->hwDeviceContext) - return AV_HWDEVICE_TYPE_NONE; - return hwDeviceContext()->type; + return m_hwDeviceContext ? hwDeviceContext()->type : AV_HWDEVICE_TYPE_NONE; } void HWAccel::createFramesContext(AVPixelFormat swFormat, const QSize &size) { - if (!d || !d->hwDeviceContext) + if (m_hwFramesContext) { + qWarning() << "Frames context has been already created!"; return; - d->hwFramesContext = av_hwframe_ctx_alloc(d->hwDeviceContext); - auto *c = (AVHWFramesContext *)d->hwFramesContext->data; + } + + if (!m_hwDeviceContext) + return; + + m_hwFramesContext.reset(av_hwframe_ctx_alloc(m_hwDeviceContext.get())); + auto *c = (AVHWFramesContext *)m_hwFramesContext->data; c->format = hwFormat(); c->sw_format = swFormat; c->width = size.width(); c->height = size.height(); - qDebug() << "init frames context"; - int err = av_hwframe_ctx_init(d->hwFramesContext); + qCDebug(qLHWAccel) << "init frames context"; + int err = av_hwframe_ctx_init(m_hwFramesContext.get()); if (err < 0) qWarning() << "failed to init HW frame context" << err << err2str(err); else - qDebug() << "Initialized frames context" << size << c->format << c->sw_format; + qCDebug(qLHWAccel) << "Initialized frames context" << size << c->format << c->sw_format; } AVHWFramesContext *HWAccel::hwFramesContext() const { - if (!d || !d->hwFramesContext) - return nullptr; - return (AVHWFramesContext *)d->hwFramesContext->data; + return m_hwFramesContext ? (AVHWFramesContext *)m_hwFramesContext->data : nullptr; } @@ -358,20 +468,29 @@ void TextureConverter::updateBackend(AVPixelFormat fmt) d->backend = nullptr; if (!d->rhi) return; + + if (!hwTextureConversionEnabled()) + return; + switch (fmt) { #if QT_CONFIG(vaapi) case AV_PIX_FMT_VAAPI: - d->backend = new VAAPITextureConverter(d->rhi); + d->backend = std::make_unique<VAAPITextureConverter>(d->rhi); break; #endif #ifdef Q_OS_DARWIN case AV_PIX_FMT_VIDEOTOOLBOX: - d->backend = new VideoToolBoxTextureConverter(d->rhi); + d->backend = std::make_unique<VideoToolBoxTextureConverter>(d->rhi); break; #endif #if QT_CONFIG(wmf) case AV_PIX_FMT_D3D11: - d->backend = new D3D11TextureConverter(d->rhi); + d->backend = std::make_unique<D3D11TextureConverter>(d->rhi); + break; +#endif +#ifdef Q_OS_ANDROID + case AV_PIX_FMT_MEDIACODEC: + d->backend = std::make_unique<MediaCodecTextureConverter>(d->rhi); break; #endif default: @@ -380,9 +499,13 @@ void TextureConverter::updateBackend(AVPixelFormat fmt) d->format = fmt; } -std::unique_ptr<QRhiTexture> TextureSet::texture(int /*plane*/) +AVFrameUPtr copyFromHwPool(AVFrameUPtr frame) { - return {}; +#if QT_CONFIG(wmf) + return copyFromHwPoolD3D11(std::move(frame)); +#else + return frame; +#endif } } // namespace QFFmpeg diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp index 350545f8a..a44ceef7f 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11.cpp @@ -1,161 +1,326 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpeghwaccel_d3d11_p.h" +#include "playbackengine/qffmpegstreamdecoder_p.h" #include <qvideoframeformat.h> #include "qffmpegvideobuffer_p.h" - #include <private/qvideotexturehelper_p.h> -#include <private/qrhi_p.h> -#include <private/qrhid3d11_p.h> +#include <private/qcomptr_p.h> +#include <private/quniquehandle_p.h> + +#include <rhi/qrhi.h> #include <qopenglfunctions.h> #include <qdebug.h> #include <qloggingcategory.h> #include <libavutil/hwcontext_d3d11va.h> +#include <d3d11_1.h> +#include <dxgi1_2.h> QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcMediaFFmpegHWAccel, "qt.multimedia.hwaccel") +namespace { + +Q_LOGGING_CATEGORY(qLcMediaFFmpegHWAccel, "qt.multimedia.hwaccel"); + +ComPtr<ID3D11Device1> GetD3DDevice(QRhi *rhi) +{ + const auto native = static_cast<const QRhiD3D11NativeHandles *>(rhi->nativeHandles()); + if (!native) + return {}; + + const ComPtr<ID3D11Device> rhiDevice = static_cast<ID3D11Device *>(native->dev); + + ComPtr<ID3D11Device1> dev1; + if (rhiDevice.As(&dev1) != S_OK) + return nullptr; + + return dev1; +} + +ComPtr<ID3D11Texture2D> getAvFrameTexture(const AVFrame *frame) +{ + return reinterpret_cast<ID3D11Texture2D *>(frame->data[0]); +} + +int getAvFramePoolIndex(const AVFrame *frame) +{ + return static_cast<int>(reinterpret_cast<intptr_t>(frame->data[1])); +} + +const AVD3D11VADeviceContext *getHwDeviceContext(const AVHWDeviceContext *ctx) +{ + return static_cast<AVD3D11VADeviceContext *>(ctx->hwctx); +} + +void freeTextureAndData(void *opaque, uint8_t *data) +{ + static_cast<ID3D11Texture2D *>(opaque)->Release(); + av_free(data); +} + +AVBufferRef *wrapTextureAsBuffer(const ComPtr<ID3D11Texture2D> &tex) +{ + AVD3D11FrameDescriptor *avFrameDesc = + static_cast<AVD3D11FrameDescriptor *>(av_mallocz(sizeof(AVD3D11FrameDescriptor))); + avFrameDesc->index = 0; + avFrameDesc->texture = tex.Get(); + + return av_buffer_create(reinterpret_cast<uint8_t *>(avFrameDesc), + sizeof(AVD3D11FrameDescriptor *), freeTextureAndData, tex.Get(), 0); +} + +ComPtr<ID3D11Texture2D> copyTexture(const AVD3D11VADeviceContext *hwDevCtx, const AVFrame *src) +{ + const int poolIndex = getAvFramePoolIndex(src); + const ComPtr<ID3D11Texture2D> poolTex = getAvFrameTexture(src); + + D3D11_TEXTURE2D_DESC texDesc{}; + poolTex->GetDesc(&texDesc); + texDesc.ArraySize = 1; + texDesc.MiscFlags = 0; + texDesc.BindFlags = 0; + + ComPtr<ID3D11Texture2D> destTex; + if (hwDevCtx->device->CreateTexture2D(&texDesc, nullptr, &destTex) != S_OK) { + qCCritical(qLcMediaFFmpegHWAccel) << "Unable to copy frame from decoder pool"; + return {}; + } + + hwDevCtx->device_context->CopySubresourceRegion(destTex.Get(), 0, 0, 0, 0, poolTex.Get(), + poolIndex, nullptr); + + return destTex; +} + +} // namespace namespace QFFmpeg { -class D3D11TextureSet : public TextureSet +bool TextureBridge::copyToSharedTex(ID3D11Device *dev, ID3D11DeviceContext *ctx, + const ComPtr<ID3D11Texture2D> &tex, UINT index, + const QSize &frameSize) { -public: - D3D11TextureSet(QRhi *rhi, QVideoFrameFormat::PixelFormat format, ID3D11Texture2D *tex, int index) - : m_rhi(rhi) - , m_format(format) - , m_tex(tex) - , m_index(index) - {} - - ~D3D11TextureSet() override { - if (m_tex) - m_tex->Release(); + if (!ensureSrcTex(dev, tex, frameSize)) + return false; + + // Flush to ensure that texture is fully updated before we share it. + ctx->Flush(); + + if (m_srcMutex->AcquireSync(m_srcKey, INFINITE) != S_OK) + return false; + + const UINT width = static_cast<UINT>(frameSize.width()); + const UINT height = static_cast<UINT>(frameSize.height()); + + // A crop box is needed because FFmpeg may have created textures + // that are bigger than the frame size to account for the decoder's + // surface alignment requirements. + const D3D11_BOX crop{ 0, 0, 0, width, height, 1 }; + ctx->CopySubresourceRegion(m_srcTex.Get(), 0, 0, 0, 0, tex.Get(), index, &crop); + + m_srcMutex->ReleaseSync(m_destKey); + return true; +} + +ComPtr<ID3D11Texture2D> TextureBridge::copyFromSharedTex(const ComPtr<ID3D11Device1> &dev, + const ComPtr<ID3D11DeviceContext> &ctx) +{ + if (!ensureDestTex(dev)) + return {}; + + if (m_destMutex->AcquireSync(m_destKey, INFINITE) != S_OK) + return {}; + + ctx->CopySubresourceRegion(m_outputTex.Get(), 0, 0, 0, 0, m_destTex.Get(), 0, nullptr); + + m_destMutex->ReleaseSync(m_srcKey); + + return m_outputTex; +} + +bool TextureBridge::ensureDestTex(const ComPtr<ID3D11Device1> &dev) +{ + if (m_destDevice != dev) { + // Destination device changed. Recreate texture. + m_destTex = nullptr; + m_destDevice = dev; } - std::unique_ptr<QRhiTexture> texture(int plane) override { - auto desc = QVideoTextureHelper::textureDescription(m_format); - if (!m_tex || !m_rhi || !desc || plane >= desc->nplanes) - return {}; + if (m_destTex) + return true; - D3D11_TEXTURE2D_DESC d3d11desc = {}; - m_tex->GetDesc(&d3d11desc); + if (m_destDevice->OpenSharedResource1(m_sharedHandle.get(), IID_PPV_ARGS(&m_destTex)) != S_OK) + return false; - QSize planeSize(desc->widthForPlane(int(d3d11desc.Width), plane), - desc->heightForPlane(int(d3d11desc.Height), plane)); + CD3D11_TEXTURE2D_DESC desc{}; + m_destTex->GetDesc(&desc); - std::unique_ptr<QRhiTexture> tex(m_rhi->newTextureArray(desc->textureFormat[plane], - int(d3d11desc.ArraySize), - planeSize, 1, {})); - if (tex) { - tex->setArrayRange(m_index, 1); - if (!tex->createFrom({quint64(m_tex), 0})) - tex.reset(); - } - return tex; + desc.MiscFlags = 0; + desc.BindFlags = D3D11_BIND_SHADER_RESOURCE; + + if (m_destDevice->CreateTexture2D(&desc, nullptr, m_outputTex.ReleaseAndGetAddressOf()) != S_OK) + return false; + + if (m_destTex.As(&m_destMutex) != S_OK) + return false; + + return true; +} + +bool TextureBridge::ensureSrcTex(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex, const QSize &frameSize) +{ + if (!isSrcInitialized(dev, tex, frameSize)) + return recreateSrc(dev, tex, frameSize); + + return true; +} + +bool TextureBridge::isSrcInitialized(const ID3D11Device *dev, + const ComPtr<ID3D11Texture2D> &tex, + const QSize &frameSize) const +{ + if (!m_srcTex) + return false; + + // Check if device has changed + ComPtr<ID3D11Device> texDevice; + m_srcTex->GetDevice(texDevice.GetAddressOf()); + if (dev != texDevice.Get()) + return false; + + // Check if shared texture has correct size and format + CD3D11_TEXTURE2D_DESC inputDesc{}; + tex->GetDesc(&inputDesc); + + CD3D11_TEXTURE2D_DESC currentDesc{}; + m_srcTex->GetDesc(¤tDesc); + + if (inputDesc.Format != currentDesc.Format) + return false; + + const UINT width = static_cast<UINT>(frameSize.width()); + const UINT height = static_cast<UINT>(frameSize.height()); + + if (currentDesc.Width != width || currentDesc.Height != height) + return false; + + return true; +} + +bool TextureBridge::recreateSrc(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex, const QSize &frameSize) +{ + m_sharedHandle.close(); + + CD3D11_TEXTURE2D_DESC desc{}; + tex->GetDesc(&desc); + + const UINT width = static_cast<UINT>(frameSize.width()); + const UINT height = static_cast<UINT>(frameSize.height()); + + CD3D11_TEXTURE2D_DESC texDesc{ desc.Format, width, height }; + texDesc.MipLevels = 1; + texDesc.MiscFlags = D3D11_RESOURCE_MISC_SHARED_KEYEDMUTEX | D3D11_RESOURCE_MISC_SHARED_NTHANDLE; + + if (dev->CreateTexture2D(&texDesc, nullptr, m_srcTex.ReleaseAndGetAddressOf()) != S_OK) + return false; + + ComPtr<IDXGIResource1> res; + if (m_srcTex.As(&res) != S_OK) + return false; + + const HRESULT hr = + res->CreateSharedHandle(nullptr, DXGI_SHARED_RESOURCE_READ, nullptr, &m_sharedHandle); + + if (hr != S_OK || !m_sharedHandle) + return false; + + if (m_srcTex.As(&m_srcMutex) != S_OK || !m_srcMutex) + return false; + + m_destTex = nullptr; + m_destMutex = nullptr; + return true; +} + +class D3D11TextureSet : public TextureSet +{ +public: + D3D11TextureSet(QRhi *rhi, ComPtr<ID3D11Texture2D> &&tex) + : m_owner{ rhi }, m_tex(std::move(tex)) + { + } + + qint64 textureHandle(QRhi *rhi, int /*plane*/) override + { + if (rhi != m_owner) + return 0u; + return reinterpret_cast<qint64>(m_tex.Get()); } private: - QRhi *m_rhi = nullptr; - QVideoFrameFormat::PixelFormat m_format; - ID3D11Texture2D *m_tex = nullptr; - int m_index = 0; + QRhi *m_owner = nullptr; + ComPtr<ID3D11Texture2D> m_tex; }; - D3D11TextureConverter::D3D11TextureConverter(QRhi *rhi) - : TextureConverterBackend(rhi) + : TextureConverterBackend(rhi), m_rhiDevice{ GetD3DDevice(rhi) } { + if (!m_rhiDevice) + return; + + m_rhiDevice->GetImmediateContext(m_rhiCtx.GetAddressOf()); } TextureSet *D3D11TextureConverter::getTextures(AVFrame *frame) { - if (!frame || !frame->hw_frames_ctx || frame->format != AV_PIX_FMT_D3D11) + if (!m_rhiDevice) return nullptr; - auto *fCtx = (AVHWFramesContext *)frame->hw_frames_ctx->data; - auto *ctx = fCtx->device_ctx; - if (!ctx || ctx->type != AV_HWDEVICE_TYPE_D3D11VA) + if (!frame || !frame->hw_frames_ctx || frame->format != AV_PIX_FMT_D3D11) return nullptr; - auto nh = static_cast<const QRhiD3D11NativeHandles *>(rhi->nativeHandles()); - if (!nh) - return nullptr; + const auto *ctx = avFrameDeviceContext(frame); - auto dev = reinterpret_cast<ID3D11Device *>(nh->dev); - if (!dev) + if (!ctx || ctx->type != AV_HWDEVICE_TYPE_D3D11VA) return nullptr; - auto ffmpegTex = (ID3D11Texture2D *)frame->data[0]; - int index = (intptr_t)frame->data[1]; - - IDXGIResource *dxgiResource = nullptr; - HRESULT hr = ffmpegTex->QueryInterface(__uuidof(IDXGIResource), (void **)&dxgiResource); - if (FAILED(hr)) { - qCDebug(qLcMediaFFmpegHWAccel) << "Failed to obtain resource handle from FFMpeg texture" << hr; - return nullptr; - } - HANDLE shared = nullptr; - hr = dxgiResource->GetSharedHandle(&shared); - dxgiResource->Release(); - if (FAILED(hr)) { - qCDebug(qLcMediaFFmpegHWAccel) << "Failed to obtain shared handle for FFmpeg texture" << hr; - return nullptr; - } + const ComPtr<ID3D11Texture2D> ffmpegTex = getAvFrameTexture(frame); + const int index = getAvFramePoolIndex(frame); if (rhi->backend() == QRhi::D3D11) { - ID3D11Texture2D *sharedTex = nullptr; - hr = dev->OpenSharedResource(shared, __uuidof(ID3D11Texture2D), (void **)(&sharedTex)); - if (FAILED(hr)) { - qCDebug(qLcMediaFFmpegHWAccel) << "Failed to share FFmpeg texture" << hr; - return nullptr; + { + const auto *avDeviceCtx = getHwDeviceContext(ctx); + + if (!avDeviceCtx) + return nullptr; + + // Lock the FFmpeg device context while we copy from FFmpeg's + // frame pool into a shared texture because the underlying ID3D11DeviceContext + // is not thread safe. + avDeviceCtx->lock(avDeviceCtx->lock_ctx); + QScopeGuard autoUnlock([&] { avDeviceCtx->unlock(avDeviceCtx->lock_ctx); }); + + // Populate the shared texture with one slice from the frame pool, cropping away + // extra surface alignment areas that FFmpeg adds to the textures + QSize frameSize{ frame->width, frame->height }; + if (!m_bridge.copyToSharedTex(avDeviceCtx->device, avDeviceCtx->device_context, + ffmpegTex, index, frameSize)) { + return nullptr; + } } - QVideoFrameFormat::PixelFormat format = QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat(fCtx->sw_format)); - return new D3D11TextureSet(rhi, format, sharedTex, index); - } else if (rhi->backend() == QRhi::OpenGLES2) { + // Get a copy of the texture on the RHI device + ComPtr<ID3D11Texture2D> output = m_bridge.copyFromSharedTex(m_rhiDevice, m_rhiCtx); + + if (!output) + return nullptr; + return new D3D11TextureSet(rhi, std::move(output)); } return nullptr; @@ -163,17 +328,15 @@ TextureSet *D3D11TextureConverter::getTextures(AVFrame *frame) void D3D11TextureConverter::SetupDecoderTextures(AVCodecContext *s) { - int ret = avcodec_get_hw_frames_parameters(s, - s->hw_device_ctx, - AV_PIX_FMT_D3D11, + int ret = avcodec_get_hw_frames_parameters(s, s->hw_device_ctx, AV_PIX_FMT_D3D11, &s->hw_frames_ctx); if (ret < 0) { qCDebug(qLcMediaFFmpegHWAccel) << "Failed to allocate HW frames context" << ret; return; } - auto *frames_ctx = (AVHWFramesContext *)s->hw_frames_ctx->data; - auto *hwctx = (AVD3D11VAFramesContext *)frames_ctx->hwctx; + const auto *frames_ctx = reinterpret_cast<const AVHWFramesContext *>(s->hw_frames_ctx->data); + auto *hwctx = static_cast<AVD3D11VAFramesContext *>(frames_ctx->hwctx); hwctx->MiscFlags = D3D11_RESOURCE_MISC_SHARED; hwctx->BindFlags = D3D11_BIND_DECODER | D3D11_BIND_SHADER_RESOURCE; ret = av_hwframe_ctx_init(s->hw_frames_ctx); @@ -183,6 +346,41 @@ void D3D11TextureConverter::SetupDecoderTextures(AVCodecContext *s) } } +AVFrameUPtr copyFromHwPoolD3D11(AVFrameUPtr src) +{ + if (!src || !src->hw_frames_ctx || src->format != AV_PIX_FMT_D3D11) + return src; + + const AVHWDeviceContext *avDevCtx = avFrameDeviceContext(src.get()); + if (!avDevCtx || avDevCtx->type != AV_HWDEVICE_TYPE_D3D11VA) + return src; + + AVFrameUPtr dest = makeAVFrame(); + if (av_frame_copy_props(dest.get(), src.get()) != 0) { + qCCritical(qLcMediaFFmpegHWAccel) << "Unable to copy frame from decoder pool"; + return src; + } + + const AVD3D11VADeviceContext *hwDevCtx = getHwDeviceContext(avDevCtx); + ComPtr<ID3D11Texture2D> destTex; + { + hwDevCtx->lock(hwDevCtx->lock_ctx); + destTex = copyTexture(hwDevCtx, src.get()); + hwDevCtx->unlock(hwDevCtx->lock_ctx); + } + + dest->buf[0] = wrapTextureAsBuffer(destTex); + dest->data[0] = reinterpret_cast<uint8_t *>(destTex.Detach()); + dest->data[1] = reinterpret_cast<uint8_t *>(0); // This texture is not a texture array + + dest->width = src->width; + dest->height = src->height; + dest->format = src->format; + dest->hw_frames_ctx = av_buffer_ref(src->hw_frames_ctx); + + return dest; } +} // namespace QFFmpeg + QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h index 383b7168c..395016c69 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_d3d11_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGHWACCEL_D3D11_P_H #define QFFMPEGHWACCEL_D3D11_P_H @@ -51,6 +15,12 @@ // #include "qffmpeghwaccel_p.h" +#include <private/quniquehandle_p.h> +#include <private/qcomptr_p.h> +#include <qt_windows.h> + +#include <d3d11.h> +#include <d3d11_1.h> #if QT_CONFIG(wmf) @@ -60,6 +30,56 @@ class QRhi; namespace QFFmpeg { +struct SharedTextureHandleTraits +{ + using Type = HANDLE; + static Type invalidValue() { return nullptr; } + static bool close(Type handle) { return CloseHandle(handle) != 0; } +}; + +using SharedTextureHandle = QUniqueHandle<SharedTextureHandleTraits>; + +/*! \internal Utility class for synchronized transfer of a texture between two D3D devices + * + * This class is used to copy a texture from one device to another device. This + * is implemented using a shared texture, along with keyed mutexes to synchronize + * access to the texture. + * + * This is needed because we need to copy data from FFmpeg to RHI. FFmpeg and RHI + * uses different D3D devices. + */ +class TextureBridge final +{ +public: + /** Copy a texture slice at position 'index' belonging to device 'dev' + * into a shared texture, limiting the texture size to the frame size */ + bool copyToSharedTex(ID3D11Device *dev, ID3D11DeviceContext *ctx, + const ComPtr<ID3D11Texture2D> &tex, UINT index, const QSize &frameSize); + + /** Obtain a copy of the texture on a second device 'dev' */ + ComPtr<ID3D11Texture2D> copyFromSharedTex(const ComPtr<ID3D11Device1> &dev, + const ComPtr<ID3D11DeviceContext> &ctx); + +private: + bool ensureDestTex(const ComPtr<ID3D11Device1> &dev); + bool ensureSrcTex(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex, const QSize &frameSize); + bool isSrcInitialized(const ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex, const QSize &frameSize) const; + bool recreateSrc(ID3D11Device *dev, const ComPtr<ID3D11Texture2D> &tex, const QSize &frameSize); + + SharedTextureHandle m_sharedHandle{}; + + const UINT m_srcKey = 0; + ComPtr<ID3D11Texture2D> m_srcTex; + ComPtr<IDXGIKeyedMutex> m_srcMutex; + + const UINT m_destKey = 1; + ComPtr<ID3D11Device1> m_destDevice; + ComPtr<ID3D11Texture2D> m_destTex; + ComPtr<IDXGIKeyedMutex> m_destMutex; + + ComPtr<ID3D11Texture2D> m_outputTex; +}; + class D3D11TextureConverter : public TextureConverterBackend { public: @@ -68,9 +88,16 @@ public: TextureSet *getTextures(AVFrame *frame) override; static void SetupDecoderTextures(AVCodecContext *s); + +private: + ComPtr<ID3D11Device1> m_rhiDevice; + ComPtr<ID3D11DeviceContext> m_rhiCtx; + TextureBridge m_bridge; }; -} +AVFrameUPtr copyFromHwPoolD3D11(AVFrameUPtr src); + +} // namespace QFFmpeg QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp new file mode 100644 index 000000000..e9dd8705a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec.cpp @@ -0,0 +1,120 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpeghwaccel_mediacodec_p.h" + +#include "androidsurfacetexture_p.h" +#include <rhi/qrhi.h> + +extern "C" { +#include <libavcodec/mediacodec.h> +#include <libavutil/hwcontext_mediacodec.h> +} + +#if !defined(Q_OS_ANDROID) +# error "Configuration error" +#endif + +namespace QFFmpeg { + +class MediaCodecTextureSet : public TextureSet +{ +public: + MediaCodecTextureSet(qint64 textureHandle) : handle(textureHandle) { } + + qint64 textureHandle(QRhi *, int plane) override { return (plane == 0) ? handle : 0; } + +private: + qint64 handle; +}; + +namespace { + +void deleteSurface(AVHWDeviceContext *ctx) +{ + AndroidSurfaceTexture* s = reinterpret_cast<AndroidSurfaceTexture *>(ctx->user_opaque); + delete s; +} + +AndroidSurfaceTexture* getTextureSurface(AVFrame *frame) +{ + if (!frame || !frame->hw_frames_ctx) + return nullptr; + + auto *frameContext = reinterpret_cast<AVHWFramesContext *>(frame->hw_frames_ctx->data); + + if (!frameContext || !frameContext->device_ctx) + return nullptr; + + AVHWDeviceContext *deviceContext = frameContext->device_ctx; + + return reinterpret_cast<AndroidSurfaceTexture *>(deviceContext->user_opaque); +} +} // namespace + +void MediaCodecTextureConverter::setupDecoderSurface(AVCodecContext *avCodecContext) +{ + std::unique_ptr<AndroidSurfaceTexture> androidSurfaceTexture(new AndroidSurfaceTexture(0)); + AVMediaCodecContext *mediacodecContext = av_mediacodec_alloc_context(); + av_mediacodec_default_init(avCodecContext, mediacodecContext, androidSurfaceTexture->surface()); + + if (!avCodecContext->hw_device_ctx || !avCodecContext->hw_device_ctx->data) + return; + + AVHWDeviceContext *deviceContext = + reinterpret_cast<AVHWDeviceContext *>(avCodecContext->hw_device_ctx->data); + + if (!deviceContext->hwctx) + return; + + AVMediaCodecDeviceContext *mediaDeviceContext = + reinterpret_cast<AVMediaCodecDeviceContext *>(deviceContext->hwctx); + + if (!mediaDeviceContext) + return; + + mediaDeviceContext->surface = androidSurfaceTexture->surface(); + + Q_ASSERT(deviceContext->user_opaque == nullptr); + deviceContext->user_opaque = androidSurfaceTexture.release(); + deviceContext->free = deleteSurface; +} + +TextureSet *MediaCodecTextureConverter::getTextures(AVFrame *frame) +{ + AndroidSurfaceTexture * androidSurfaceTexture = getTextureSurface(frame); + + if (!androidSurfaceTexture || !androidSurfaceTexture->isValid()) + return {}; + + if (!externalTexture || m_currentSurfaceIndex != androidSurfaceTexture->index()) { + m_currentSurfaceIndex = androidSurfaceTexture->index(); + androidSurfaceTexture->detachFromGLContext(); + externalTexture = std::unique_ptr<QRhiTexture>( + rhi->newTexture(QRhiTexture::Format::RGBA8, { frame->width, frame->height }, 1, + QRhiTexture::ExternalOES)); + + if (!externalTexture->create()) { + qWarning() << "Failed to create the external texture!"; + return {}; + } + + quint64 textureHandle = externalTexture->nativeTexture().object; + androidSurfaceTexture->attachToGLContext(textureHandle); + } + + // release a MediaCodec buffer and render it to the surface + AVMediaCodecBuffer *buffer = (AVMediaCodecBuffer *)frame->data[3]; + + if (!buffer) { + qWarning() << "Received a frame without AVMediaCodecBuffer."; + } else if (av_mediacodec_release_buffer(buffer, 1) < 0) { + qWarning() << "Failed to render buffer to surface."; + return {}; + } + + androidSurfaceTexture->updateTexImage(); + + return new MediaCodecTextureSet(externalTexture->nativeTexture().object); +} +} diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h new file mode 100644 index 000000000..79d33d03e --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_mediacodec_p.h @@ -0,0 +1,36 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGHWACCEL_MEDIACODEC_P_H +#define QFFMPEGHWACCEL_MEDIACODEC_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeghwaccel_p.h" +#include <memory> + +namespace QFFmpeg { +struct Frame; + +class MediaCodecTextureConverter : public TextureConverterBackend +{ +public: + MediaCodecTextureConverter(QRhi *rhi) : TextureConverterBackend(rhi){}; + TextureSet *getTextures(AVFrame *frame) override; + + static void setupDecoderSurface(AVCodecContext *s); +private: + std::unique_ptr<QRhiTexture> externalTexture; + quint64 m_currentSurfaceIndex = 0; +}; +} +#endif // QFFMPEGHWACCEL_MEDIACODEC_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h index c25faecfd..5207e76c7 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGHWACCEL_P_H #define QFFMPEGHWACCEL_P_H @@ -52,8 +16,12 @@ #include "qffmpeg_p.h" #include "qvideoframeformat.h" +#include "qabstractvideobuffer.h" + #include <qshareddata.h> #include <memory> +#include <functional> +#include <mutex> QT_BEGIN_NAMESPACE @@ -72,8 +40,7 @@ class TextureSet { public: // ### Should add QVideoFrameFormat::PixelFormat here virtual ~TextureSet() {} - virtual qint64 textureHandle(int /*plane*/) { return 0; } - virtual std::unique_ptr<QRhiTexture> texture(int plane); + virtual qint64 textureHandle(QRhi *, int /*plane*/) { return 0; } }; class TextureConverterBackend @@ -93,11 +60,10 @@ class TextureConverter class Data final { public: - ~Data(); QAtomicInt ref = 0; QRhi *rhi = nullptr; AVPixelFormat format = AV_PIX_FMT_NONE; - TextureConverterBackend *backend = nullptr; + std::unique_ptr<TextureConverterBackend> backend; }; public: TextureConverter(QRhi *rhi = nullptr); @@ -118,40 +84,47 @@ private: class HWAccel { - struct Data { - ~Data(); - QAtomicInt ref = 0; - AVBufferRef *hwDeviceContext = nullptr; - AVBufferRef *hwFramesContext = nullptr; - }; + AVBufferUPtr m_hwDeviceContext; + AVBufferUPtr m_hwFramesContext; + + mutable std::once_flag m_constraintsOnceFlag; + mutable AVHWFramesConstraintsUPtr m_constraints; public: - HWAccel() = default; - explicit HWAccel(AVHWDeviceType deviceType); - explicit HWAccel(const AVCodec *codec); ~HWAccel(); - bool isNull() const { return !d || !d->hwDeviceContext; } + static std::unique_ptr<HWAccel> create(AVHWDeviceType deviceType); + + static std::pair<const AVCodec *, std::unique_ptr<HWAccel>> + findEncoderWithHwAccel(AVCodecID id, + const std::function<bool(const HWAccel &)>& hwAccelPredicate = nullptr); + + static std::pair<const AVCodec *, std::unique_ptr<HWAccel>> + findDecoderWithHwAccel(AVCodecID id, + const std::function<bool(const HWAccel &)>& hwAccelPredicate = nullptr); AVHWDeviceType deviceType() const; - AVBufferRef *hwDeviceContextAsBuffer() const { return d ? d->hwDeviceContext : nullptr; } + AVBufferRef *hwDeviceContextAsBuffer() const { return m_hwDeviceContext.get(); } AVHWDeviceContext *hwDeviceContext() const; AVPixelFormat hwFormat() const; - - const AVCodec *hardwareEncoderForCodecId(AVCodecID id) const; - static HWAccel findHardwareAccelForCodecID(AVCodecID id); + const AVHWFramesConstraints *constraints() const; void createFramesContext(AVPixelFormat swFormat, const QSize &size); - AVBufferRef *hwFramesContextAsBuffer() const { return d ? d->hwFramesContext : nullptr; } + AVBufferRef *hwFramesContextAsBuffer() const { return m_hwFramesContext.get(); } AVHWFramesContext *hwFramesContext() const; static AVPixelFormat format(AVFrame *frame); - static const AVHWDeviceType *preferredDeviceTypes(); + static const std::vector<AVHWDeviceType> &encodingDeviceTypes(); + + static const std::vector<AVHWDeviceType> &decodingDeviceTypes(); + private: - QExplicitlySharedDataPointer<Data> d; + HWAccel(AVBufferUPtr hwDeviceContext) : m_hwDeviceContext(std::move(hwDeviceContext)) { } }; +AVFrameUPtr copyFromHwPool(AVFrameUPtr frame); + } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp index 09e06c64a..069a04f52 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpeghwaccel_vaapi_p.h" @@ -49,8 +13,7 @@ #include "qffmpegvideobuffer_p.h" #include "private/qvideotexturehelper_p.h" -#include <private/qrhi_p.h> -#include <private/qrhigles2_p.h> +#include <rhi/qrhi.h> #include <qguiapplication.h> #include <qpa/qplatformnativeinterface.h> @@ -94,7 +57,11 @@ extern "C" { #include <unistd.h> -#include <qdebug.h> +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLHWAccelVAAPI, "qt.multimedia.ffmpeg.hwaccelvaapi"); namespace QFFmpeg { @@ -110,7 +77,7 @@ static const quint32 *fourccFromPixelFormat(const QVideoFrameFormat::PixelFormat const quint32 rg16_fourcc = DRM_FORMAT_RG1616; #endif -// qDebug() << "Getting DRM fourcc for pixel format" << format; +// qCDebug(qLHWAccelVAAPI) << "Getting DRM fourcc for pixel format" << format; switch (format) { case QVideoFrameFormat::Format_Invalid: @@ -187,7 +154,7 @@ class VAAPITextureSet : public TextureSet { public: ~VAAPITextureSet(); - qint64 textureHandle(int plane) override { + qint64 textureHandle(QRhi *, int plane) override { return textures[plane]; } @@ -201,7 +168,7 @@ public: VAAPITextureConverter::VAAPITextureConverter(QRhi *rhi) : TextureConverterBackend(nullptr) { - qDebug() << ">>>> Creating VAAPI HW accelerator"; + qCDebug(qLHWAccelVAAPI) << ">>>> Creating VAAPI HW accelerator"; if (!rhi || rhi->backend() != QRhi::OpenGLES2) { qWarning() << "VAAPITextureConverter: No rhi or non openGL based RHI"; @@ -212,21 +179,21 @@ VAAPITextureConverter::VAAPITextureConverter(QRhi *rhi) auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles()); glContext = nativeHandles->context; if (!glContext) { - qDebug() << " no GL context, disabling"; + qCDebug(qLHWAccelVAAPI) << " no GL context, disabling"; return; } const QString platform = QGuiApplication::platformName(); QPlatformNativeInterface *pni = QGuiApplication::platformNativeInterface(); - eglDisplay = pni->nativeResourceForIntegration("egldisplay"); - qDebug() << " platform is" << platform << eglDisplay; + eglDisplay = pni->nativeResourceForIntegration(QByteArrayLiteral("egldisplay")); + qCDebug(qLHWAccelVAAPI) << " platform is" << platform << eglDisplay; if (!eglDisplay) { - qDebug() << " no egl display, disabling"; + qCDebug(qLHWAccelVAAPI) << " no egl display, disabling"; return; } eglImageTargetTexture2D = eglGetProcAddress("glEGLImageTargetTexture2DOES"); if (!eglDisplay) { - qDebug() << " no eglImageTargetTexture2D, disabling"; + qCDebug(qLHWAccelVAAPI) << " no eglImageTargetTexture2D, disabling"; return; } @@ -241,30 +208,29 @@ VAAPITextureConverter::~VAAPITextureConverter() //#define VA_EXPORT_USE_LAYERS TextureSet *VAAPITextureConverter::getTextures(AVFrame *frame) { -// qDebug() << "VAAPIAccel::getTextures"; +// qCDebug(qLHWAccelVAAPI) << "VAAPIAccel::getTextures"; if (frame->format != AV_PIX_FMT_VAAPI || !eglDisplay) { - qDebug() << "format/egl error" << frame->format << eglDisplay; + qCDebug(qLHWAccelVAAPI) << "format/egl error" << frame->format << eglDisplay; return nullptr; } if (!frame->hw_frames_ctx) return nullptr; - auto *fCtx = (AVHWFramesContext *)frame->hw_frames_ctx->data; - auto *ctx = fCtx->device_ctx; + auto *ctx = avFrameDeviceContext(frame); if (!ctx) return nullptr; auto *vaCtx = (AVVAAPIDeviceContext *)ctx->hwctx; auto vaDisplay = vaCtx->display; if (!vaDisplay) { - qDebug() << " no VADisplay, disabling"; + qCDebug(qLHWAccelVAAPI) << " no VADisplay, disabling"; return nullptr; } VASurfaceID vaSurface = (uintptr_t)frame->data[3]; - VADRMPRIMESurfaceDescriptor prime; + VADRMPRIMESurfaceDescriptor prime = {}; if (vaExportSurfaceHandle(vaDisplay, vaSurface, VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2, VA_EXPORT_SURFACE_READ_ONLY | @@ -278,10 +244,17 @@ TextureSet *VAAPITextureConverter::getTextures(AVFrame *frame) qWarning() << "vaExportSurfaceHandle failed"; return nullptr; } + + // Make sure all fd's in 'prime' are closed when we return from this function + QScopeGuard closeObjectsGuard([&prime]() { + for (uint32_t i = 0; i < prime.num_objects; ++i) + close(prime.objects[i].fd); + }); + // ### Check that prime.fourcc is what we expect vaSyncSurface(vaDisplay, vaSurface); -// qDebug() << "VAAPIAccel: vaSufraceDesc: width/height" << prime.width << prime.height << "num objects" +// qCDebug(qLHWAccelVAAPI) << "VAAPIAccel: vaSufraceDesc: width/height" << prime.width << prime.height << "num objects" // << prime.num_objects << "num layers" << prime.num_layers; QOpenGLFunctions functions(glContext); @@ -303,7 +276,7 @@ TextureSet *VAAPITextureConverter::getTextures(AVFrame *frame) } Q_ASSERT(nPlanes == desc->nplanes); nPlanes = desc->nplanes; -// qDebug() << "VAAPIAccel: nPlanes" << nPlanes; +// qCDebug(qLHWAccelVAAPI) << "VAAPIAccel: nPlanes" << nPlanes; rhi->makeThreadLocalNativeContextCurrent(); @@ -334,22 +307,30 @@ TextureSet *VAAPITextureConverter::getTextures(AVFrame *frame) }; images[i] = eglCreateImage(eglDisplay, EGL_NO_CONTEXT, EGL_LINUX_DMA_BUF_EXT, nullptr, img_attr); if (!images[i]) { - qWarning() << "eglCreateImage failed for plane" << i << Qt::hex << eglGetError(); - return nullptr; + const GLenum error = eglGetError(); + if (error == EGL_BAD_MATCH) { + qWarning() << "eglCreateImage failed for plane" << i << "with error code EGL_BAD_MATCH, " + "disabling hardware acceleration. This could indicate an EGL implementation issue." + "\nVAAPI driver: " << vaQueryVendorString(vaDisplay) + << "\nEGL vendor:" << eglQueryString(eglDisplay, EGL_VENDOR); + this->rhi = nullptr; // Disabling texture conversion here to fix QTBUG-112312 + return nullptr; + } + if (error) { + qWarning() << "eglCreateImage failed for plane" << i << "with error code" << error; + return nullptr; + } } functions.glActiveTexture(GL_TEXTURE0 + i); functions.glBindTexture(GL_TEXTURE_2D, glTextures[i]); PFNGLEGLIMAGETARGETTEXTURE2DOESPROC eglImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)this->eglImageTargetTexture2D; eglImageTargetTexture2D(GL_TEXTURE_2D, images[i]); - if (glGetError()) { - qWarning() << "eglImageTargetTexture2D failed"; - } + GLenum error = glGetError(); + if (error) + qWarning() << "eglImageTargetTexture2D failed with error code" << error; } - for (int i = 0; i < (int)prime.num_objects; ++i) - close(prime.objects[i].fd); - for (int i = 0; i < nPlanes; ++i) { functions.glActiveTexture(GL_TEXTURE0 + i); functions.glBindTexture(GL_TEXTURE_2D, 0); @@ -363,7 +344,7 @@ TextureSet *VAAPITextureConverter::getTextures(AVFrame *frame) for (int i = 0; i < 4; ++i) textureSet->textures[i] = glTextures[i]; -// qDebug() << "VAAPIAccel: got textures" << textures[0] << textures[1] << textures[2] << textures[3]; +// qCDebug(qLHWAccelVAAPI) << "VAAPIAccel: got textures" << textures[0] << textures[1] << textures[2] << textures[3]; return textureSet; } diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi_p.h index f769ca2cf..03084cc72 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_vaapi_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGHWACCEL_VAAPI_P_H #define QFFMPEGHWACCEL_VAAPI_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox.mm b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox.mm index d77eab142..948f7fc23 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox.mm +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox.mm @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpeghwaccel_videotoolbox_p.h" @@ -45,22 +9,27 @@ #include <qvideoframeformat.h> #include <qffmpegvideobuffer_p.h> +#include <qloggingcategory.h> #include "private/qvideotexturehelper_p.h" -#include <private/qrhi_p.h> -#include <private/qrhimetal_p.h> -#include <private/qrhigles2_p.h> +#include <rhi/qrhi.h> #include <CoreVideo/CVMetalTexture.h> #include <CoreVideo/CVMetalTextureCache.h> #include <qopenglcontext.h> - +#ifdef Q_OS_MACOS #import <AppKit/AppKit.h> +#endif +#ifdef Q_OS_IOS +#import <OpenGLES/EAGL.h> +#endif #import <Metal/Metal.h> QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLcVideotoolbox, "qt.multimedia.ffmpeg.videotoolbox") + namespace QFFmpeg { @@ -70,7 +39,7 @@ class VideoToolBoxTextureSet : public TextureSet { public: ~VideoToolBoxTextureSet(); - qint64 textureHandle(int plane) override; + qint64 textureHandle(QRhi *, int plane) override; QRhi *rhi = nullptr; CVMetalTextureRef cvMetalTexture[3] = {}; @@ -91,7 +60,6 @@ VideoToolBoxTextureConverter::VideoToolBoxTextureConverter(QRhi *rhi) return; if (rhi->backend() == QRhi::Metal) { - qDebug() << " using metal backend"; const auto *metal = static_cast<const QRhiMetalNativeHandles *>(rhi->nativeHandles()); // Create a Metal Core Video texture cache from the pixel buffer. @@ -202,13 +170,13 @@ TextureSet *VideoToolBoxTextureConverter::getTextures(AVFrame *frame) bool needsConversion = false; QVideoFrameFormat::PixelFormat pixelFormat = QFFmpegVideoBuffer::toQtPixelFormat(HWAccel::format(frame), &needsConversion); if (needsConversion) { - qDebug() << "XXXXXXXXXXXX pixel format needs conversion" << pixelFormat << HWAccel::format(frame); + // qDebug() << "XXXXXXXXXXXX pixel format needs conversion" << pixelFormat << HWAccel::format(frame); return nullptr; } CVPixelBufferRef buffer = (CVPixelBufferRef)frame->data[3]; - VideoToolBoxTextureSet *textureSet = new VideoToolBoxTextureSet; + auto textureSet = std::make_unique<VideoToolBoxTextureSet>(); textureSet->m_buffer = buffer; textureSet->rhi = rhi; CVPixelBufferRetain(buffer); @@ -251,8 +219,10 @@ TextureSet *VideoToolBoxTextureConverter::getTextures(AVFrame *frame) buffer, nil, &textureSet->cvOpenGLTexture); - if (cvret != kCVReturnSuccess) - qWarning() << "OpenGL texture creation failed" << cvret; + if (cvret != kCVReturnSuccess) { + qCWarning(qLcVideotoolbox) << "OpenGL texture creation failed" << cvret; + return nullptr; + } Q_ASSERT(CVOpenGLTextureGetTarget(textureSet->cvOpenGLTexture) == GL_TEXTURE_RECTANGLE); #endif @@ -272,13 +242,15 @@ TextureSet *VideoToolBoxTextureConverter::getTextures(AVFrame *frame) GL_UNSIGNED_BYTE, 0, &textureSet->cvOpenGLESTexture); - if (cvret != kCVReturnSuccess) - qWarning() << "OpenGL ES texture creation failed" << cvret; + if (cvret != kCVReturnSuccess) { + qCWarning(qLcVideotoolbox) << "OpenGL ES texture creation failed" << cvret; + return nullptr; + } #endif #endif } - return textureSet; + return textureSet.release(); } VideoToolBoxTextureSet::~VideoToolBoxTextureSet() @@ -296,7 +268,7 @@ VideoToolBoxTextureSet::~VideoToolBoxTextureSet() CVPixelBufferRelease(m_buffer); } -qint64 VideoToolBoxTextureSet::textureHandle(int plane) +qint64 VideoToolBoxTextureSet::textureHandle(QRhi *, int plane) { if (rhi->backend() == QRhi::Metal) return cvMetalTexture[plane] ? qint64(CVMetalTextureGetTexture(cvMetalTexture[plane])) : 0; diff --git a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox_p.h b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox_p.h index 46ef9f6d0..44fa32dd2 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpeghwaccel_videotoolbox_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGHWACCEL_VIDEOTOOLBOX_P_H #define QFFMPEGHWACCEL_VIDEOTOOLBOX_P_H @@ -59,7 +23,11 @@ #include <CoreVideo/CVImageBuffer.h> #include <CoreVideo/CVMetalTexture.h> +#if defined(Q_OS_MACOS) #include <CoreVideo/CVOpenGLTextureCache.h> +#elif defined(Q_OS_IOS) +#include <CoreVideo/CVOpenGLESTextureCache.h> +#endif QT_BEGIN_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp index fe8df22b7..a9fc8f7af 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegimagecapture_p.h" #include <private/qplatformmediaformatinfo_p.h> @@ -53,11 +17,15 @@ QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcImageCapture, "qt.multimedia.imageCapture") +// Probably, might be increased. To be investigated and tested on Android implementation +static constexpr int MaxPendingImagesCount = 1; + +Q_STATIC_LOGGING_CATEGORY(qLcImageCapture, "qt.multimedia.imageCapture") QFFmpegImageCapture::QFFmpegImageCapture(QImageCapture *parent) : QPlatformImageCapture(parent) { + qRegisterMetaType<QVideoFrame>(); } QFFmpegImageCapture::~QFFmpegImageCapture() @@ -115,7 +83,7 @@ int QFFmpegImageCapture::doCapture(const QString &fileName) qCDebug(qLcImageCapture) << "error 1"; return -1; } - if (!m_camera) { + if (!m_videoSource) { //emit error in the next event loop, //so application can associate it with returned request id. QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, @@ -126,7 +94,7 @@ int QFFmpegImageCapture::doCapture(const QString &fileName) qCDebug(qLcImageCapture) << "error 2"; return -1; } - if (passImage) { + if (m_pendingImages.size() >= MaxPendingImagesCount) { //emit error in the next event loop, //so application can associate it with returned request id. QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, @@ -137,13 +105,12 @@ int QFFmpegImageCapture::doCapture(const QString &fileName) qCDebug(qLcImageCapture) << "error 3"; return -1; } - m_lastId++; - pendingImages.enqueue({m_lastId, fileName, QMediaMetaData{}}); - // let one image pass the pipeline - passImage = true; + m_lastId++; + m_pendingImages.enqueue({ m_lastId, fileName, QMediaMetaData{} }); updateReadyForCapture(); + return m_lastId; } @@ -154,48 +121,39 @@ void QFFmpegImageCapture::setCaptureSession(QPlatformMediaCaptureSession *sessio return; if (m_session) { - disconnect(m_session, nullptr, this, nullptr); + m_session->disconnect(this); m_lastId = 0; - pendingImages.clear(); - passImage = false; - cameraActive = false; + m_pendingImages.clear(); } m_session = captureSession; + if (m_session) - connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, &QFFmpegImageCapture::onCameraChanged); + connect(m_session, &QFFmpegMediaCaptureSession::primaryActiveVideoSourceChanged, this, + &QFFmpegImageCapture::onVideoSourceChanged); - onCameraChanged(); - updateReadyForCapture(); + onVideoSourceChanged(); } void QFFmpegImageCapture::updateReadyForCapture() { - bool ready = m_session && !passImage && cameraActive; - if (ready == m_isReadyForCapture) - return; - m_isReadyForCapture = ready; - emit readyForCaptureChanged(m_isReadyForCapture); -} + const bool ready = m_session && m_pendingImages.size() < MaxPendingImagesCount && m_videoSource + && m_videoSource->isActive(); -void QFFmpegImageCapture::cameraActiveChanged(bool active) -{ - qCDebug(qLcImageCapture) << "cameraActiveChanged" << cameraActive << active; - if (cameraActive == active) - return; - cameraActive = active; - qCDebug(qLcImageCapture) << "isReady" << isReadyForCapture(); - updateReadyForCapture(); + qCDebug(qLcImageCapture) << "updateReadyForCapture" << ready; + + if (std::exchange(m_isReadyForCapture, ready) != ready) + emit readyForCaptureChanged(ready); } void QFFmpegImageCapture::newVideoFrame(const QVideoFrame &frame) { - if (!passImage) + if (m_pendingImages.empty()) return; - passImage = false; - Q_ASSERT(!pendingImages.isEmpty()); - auto pending = pendingImages.dequeue(); + auto pending = m_pendingImages.dequeue(); + + qCDebug(qLcImageCapture) << "Taking image" << pending.id; emit imageExposed(pending.id); // ### Add metadata from the AVFrame @@ -253,27 +211,33 @@ void QFFmpegImageCapture::newVideoFrame(const QVideoFrame &frame) emit error(pending.id, err, writer.errorString()); } } + updateReadyForCapture(); } -void QFFmpegImageCapture::onCameraChanged() +void QFFmpegImageCapture::setupVideoSourceConnections() { - auto *camera = m_session ? m_session->camera() : nullptr; - if (m_camera == camera) - return; + connect(m_videoSource, &QPlatformCamera::newVideoFrame, this, + &QFFmpegImageCapture::newVideoFrame); +} + +QPlatformVideoSource *QFFmpegImageCapture::videoSource() const +{ + return m_videoSource; +} - if (m_camera) - disconnect(m_camera); +void QFFmpegImageCapture::onVideoSourceChanged() +{ + if (m_videoSource) + m_videoSource->disconnect(this); - m_camera = camera; + m_videoSource = m_session ? m_session->primaryActiveVideoSource() : nullptr; - if (camera) { - cameraActiveChanged(camera->isActive()); - connect(camera, &QPlatformCamera::activeChanged, this, &QFFmpegImageCapture::cameraActiveChanged); - connect(camera, &QPlatformCamera::newVideoFrame, this, &QFFmpegImageCapture::newVideoFrame); - } else { - cameraActiveChanged(false); - } + // TODO: optimize, setup the connection only when the capture is ready + if (m_videoSource) + setupVideoSourceConnections(); + + updateReadyForCapture(); } QImageEncoderSettings QFFmpegImageCapture::imageSettings() const @@ -303,3 +267,5 @@ void QFFmpegImageCapture::setImageSettings(const QImageEncoderSettings &settings } QT_END_NAMESPACE + +#include "moc_qffmpegimagecapture_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h index 91dacb9d2..d8174ae05 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegimagecapture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGIMAGECAPTURE_H @@ -55,12 +19,12 @@ #include <private/qplatformimagecapture_p.h> #include "qffmpegmediacapturesession_p.h" +#include <QtCore/qpointer.h> #include <qqueue.h> QT_BEGIN_NAMESPACE class QFFmpegImageCapture : public QPlatformImageCapture - { Q_OBJECT public: @@ -76,20 +40,21 @@ public: void setCaptureSession(QPlatformMediaCaptureSession *session); +protected: + virtual int doCapture(const QString &fileName); + virtual void setupVideoSourceConnections(); + QPlatformVideoSource *videoSource() const; void updateReadyForCapture(); -public Q_SLOTS: - void cameraActiveChanged(bool active); +protected Q_SLOTS: void newVideoFrame(const QVideoFrame &frame); - void onCameraChanged(); + void onVideoSourceChanged(); private: - int doCapture(const QString &fileName); - QFFmpegMediaCaptureSession *m_session = nullptr; + QPointer<QPlatformVideoSource> m_videoSource; int m_lastId = 0; QImageEncoderSettings m_settings; - QPlatformCamera *m_camera = nullptr; struct PendingImage { int id; @@ -97,9 +62,7 @@ private: QMediaMetaData metaData; }; - QQueue<PendingImage> pendingImages; - bool passImage = false; - bool cameraActive = false; + QQueue<PendingImage> m_pendingImages; bool m_isReadyForCapture = false; }; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegioutils.cpp b/src/plugins/multimedia/ffmpeg/qffmpegioutils.cpp new file mode 100644 index 000000000..cbef88f2b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegioutils.cpp @@ -0,0 +1,55 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegioutils_p.h" +#include "qiodevice.h" +#include "qffmpegdefs_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +int readQIODevice(void *opaque, uint8_t *buf, int buf_size) +{ + auto *dev = static_cast<QIODevice *>(opaque); + Q_ASSERT(dev); + + if (dev->atEnd()) + return AVERROR_EOF; + return dev->read(reinterpret_cast<char *>(buf), buf_size); +} + +int writeQIODevice(void *opaque, AvioWriteBufferType buf, int buf_size) +{ + auto dev = static_cast<QIODevice *>(opaque); + Q_ASSERT(dev); + + return dev->write(reinterpret_cast<const char *>(buf), buf_size); +} + +int64_t seekQIODevice(void *opaque, int64_t offset, int whence) +{ + QIODevice *dev = static_cast<QIODevice *>(opaque); + Q_ASSERT(dev); + + if (dev->isSequential()) + return AVERROR(EINVAL); + + if (whence & AVSEEK_SIZE) + return dev->size(); + + whence &= ~AVSEEK_FORCE; + + if (whence == SEEK_CUR) + offset += dev->pos(); + else if (whence == SEEK_END) + offset += dev->size(); + + if (!dev->seek(offset)) + return AVERROR(EINVAL); + return offset; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegioutils_p.h b/src/plugins/multimedia/ffmpeg/qffmpegioutils_p.h new file mode 100644 index 000000000..7f00990f6 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegioutils_p.h @@ -0,0 +1,40 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGIOUTILS_P_H +#define QFFMPEGIOUTILS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qtmultimediaglobal.h" +#include "qffmpegdefs_p.h" + +#include <type_traits> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +int readQIODevice(void *opaque, uint8_t *buf, int buf_size); + +using AvioWriteBufferType = + std::conditional_t<QT_FFMPEG_AVIO_WRITE_CONST, const uint8_t *, uint8_t *>; + +int writeQIODevice(void *opaque, AvioWriteBufferType buf, int buf_size); + +int64_t seekQIODevice(void *opaque, int64_t offset, int whence); + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGIOUTILS_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp index f9001d30b..545464ce8 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession.cpp @@ -1,66 +1,46 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegmediacapturesession_p.h" #include "private/qplatformaudioinput_p.h" #include "private/qplatformaudiooutput_p.h" +#include "private/qplatformsurfacecapture_p.h" +#include "private/qplatformaudiobufferinput_p.h" +#include "private/qplatformvideoframeinput_p.h" +#include "private/qplatformcamera_p.h" + #include "qffmpegimagecapture_p.h" #include "qffmpegmediarecorder_p.h" -#include "private/qplatformcamera_p.h" #include "qvideosink.h" +#include "qffmpegaudioinput_p.h" +#include "qaudiosink.h" +#include "qaudiobuffer.h" +#include "qaudiooutput.h" #include <qloggingcategory.h> QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcMediaCapture, "qt.multimedia.capture") +Q_STATIC_LOGGING_CATEGORY(qLcFFmpegMediaCaptureSession, "qt.multimedia.ffmpeg.mediacapturesession") +static int preferredAudioSinkBufferSize(const QFFmpegAudioInput &input) +{ + // Heuristic params to avoid jittering + // TODO: investigate the reason of jittering and probably reduce the factor + constexpr int BufferSizeFactor = 2; + constexpr int BufferSizeExceeding = 4096; + return input.bufferSize() * BufferSizeFactor + BufferSizeExceeding; +} QFFmpegMediaCaptureSession::QFFmpegMediaCaptureSession() { + connect(this, &QFFmpegMediaCaptureSession::primaryActiveVideoSourceChanged, this, + &QFFmpegMediaCaptureSession::updateVideoFrameConnection); } -QFFmpegMediaCaptureSession::~QFFmpegMediaCaptureSession() -{ -} +QFFmpegMediaCaptureSession::~QFFmpegMediaCaptureSession() = default; QPlatformCamera *QFFmpegMediaCaptureSession::camera() { @@ -69,21 +49,41 @@ QPlatformCamera *QFFmpegMediaCaptureSession::camera() void QFFmpegMediaCaptureSession::setCamera(QPlatformCamera *camera) { - if (m_camera == camera) - return; - if (m_camera) { - m_camera->disconnect(this); - m_camera->setCaptureSession(nullptr); - } + if (setVideoSource(m_camera, camera)) + emit cameraChanged(); +} - m_camera = camera; +QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::screenCapture() +{ + return m_screenCapture; +} - if (m_camera) { - connect(m_camera, &QPlatformCamera::newVideoFrame, this, &QFFmpegMediaCaptureSession::newVideoFrame); - m_camera->setCaptureSession(this); - } +void QFFmpegMediaCaptureSession::setScreenCapture(QPlatformSurfaceCapture *screenCapture) +{ + if (setVideoSource(m_screenCapture, screenCapture)) + emit screenCaptureChanged(); +} + +QPlatformSurfaceCapture *QFFmpegMediaCaptureSession::windowCapture() +{ + return m_windowCapture; +} + +void QFFmpegMediaCaptureSession::setWindowCapture(QPlatformSurfaceCapture *windowCapture) +{ + if (setVideoSource(m_windowCapture, windowCapture)) + emit windowCaptureChanged(); +} - emit cameraChanged(); +QPlatformVideoFrameInput *QFFmpegMediaCaptureSession::videoFrameInput() +{ + return m_videoFrameInput; +} + +void QFFmpegMediaCaptureSession::setVideoFrameInput(QPlatformVideoFrameInput *input) +{ + if (setVideoSource(m_videoFrameInput, input)) + emit videoFrameInputChanged(); } QPlatformImageCapture *QFFmpegMediaCaptureSession::imageCapture() @@ -129,33 +129,190 @@ QPlatformMediaRecorder *QFFmpegMediaCaptureSession::mediaRecorder() void QFFmpegMediaCaptureSession::setAudioInput(QPlatformAudioInput *input) { - if (m_audioInput == input) + qCDebug(qLcFFmpegMediaCaptureSession) + << "set audio input:" << (input ? input->device.description() : "null"); + + auto ffmpegAudioInput = dynamic_cast<QFFmpegAudioInput *>(input); + Q_ASSERT(!!input == !!ffmpegAudioInput); + + if (m_audioInput == ffmpegAudioInput) + return; + + if (m_audioInput) + m_audioInput->q->disconnect(this); + + m_audioInput = ffmpegAudioInput; + if (m_audioInput) + // TODO: implement the signal in QPlatformAudioInput and connect to it, QTBUG-112294 + connect(m_audioInput->q, &QAudioInput::deviceChanged, this, + &QFFmpegMediaCaptureSession::updateAudioSink); + + updateAudioSink(); +} + +void QFFmpegMediaCaptureSession::setAudioBufferInput(QPlatformAudioBufferInput *input) +{ + // TODO: implement binding to audio sink like setAudioInput does + m_audioBufferInput = input; +} + +void QFFmpegMediaCaptureSession::updateAudioSink() +{ + if (m_audioSink) { + m_audioSink->reset(); + m_audioSink.reset(); + } + + if (!m_audioInput || !m_audioOutput) return; - m_audioInput = input; + auto format = m_audioInput->device.preferredFormat(); + + if (!m_audioOutput->device.isFormatSupported(format)) + qWarning() << "Audio source format" << format << "is not compatible with the audio output"; + + m_audioSink = std::make_unique<QAudioSink>(m_audioOutput->device, format); + + m_audioBufferSize = preferredAudioSinkBufferSize(*m_audioInput); + m_audioSink->setBufferSize(m_audioBufferSize); + + qCDebug(qLcFFmpegMediaCaptureSession) + << "Create audiosink, format:" << format << "bufferSize:" << m_audioSink->bufferSize() + << "output device:" << m_audioOutput->device.description(); + + m_audioIODevice = m_audioSink->start(); + if (m_audioIODevice) { + auto writeToDevice = [this](const QAudioBuffer &buffer) { + if (m_audioBufferSize < preferredAudioSinkBufferSize(*m_audioInput)) { + qCDebug(qLcFFmpegMediaCaptureSession) + << "Recreate audiosink due to small buffer size:" << m_audioBufferSize; + + updateAudioSink(); + } + + const auto written = + m_audioIODevice->write(buffer.data<const char>(), buffer.byteCount()); + + if (written < buffer.byteCount()) + qCWarning(qLcFFmpegMediaCaptureSession) + << "Not all bytes written:" << written << "vs" << buffer.byteCount(); + }; + connect(m_audioInput, &QFFmpegAudioInput::newAudioBuffer, m_audioSink.get(), writeToDevice); + } else { + qWarning() << "Failed to start audiosink push mode"; + } + + updateVolume(); +} + +void QFFmpegMediaCaptureSession::updateVolume() +{ + if (m_audioSink) + m_audioSink->setVolume(m_audioOutput->muted ? 0.f : m_audioOutput->volume); +} + +QPlatformAudioInput *QFFmpegMediaCaptureSession::audioInput() const +{ + return m_audioInput; } void QFFmpegMediaCaptureSession::setVideoPreview(QVideoSink *sink) { - if (m_videoSink == sink) + if (std::exchange(m_videoSink, sink) == sink) return; - m_videoSink = sink; + updateVideoFrameConnection(); } void QFFmpegMediaCaptureSession::setAudioOutput(QPlatformAudioOutput *output) { + qCDebug(qLcFFmpegMediaCaptureSession) + << "set audio output:" << (output ? output->device.description() : "null"); + if (m_audioOutput == output) return; + if (m_audioOutput) + m_audioOutput->q->disconnect(this); + m_audioOutput = output; + if (m_audioOutput) { + // TODO: implement the signals in QPlatformAudioOutput and connect to them, QTBUG-112294 + connect(m_audioOutput->q, &QAudioOutput::deviceChanged, this, + &QFFmpegMediaCaptureSession::updateAudioSink); + connect(m_audioOutput->q, &QAudioOutput::volumeChanged, this, + &QFFmpegMediaCaptureSession::updateVolume); + connect(m_audioOutput->q, &QAudioOutput::mutedChanged, this, + &QFFmpegMediaCaptureSession::updateVolume); + } + + updateAudioSink(); +} + +void QFFmpegMediaCaptureSession::updateVideoFrameConnection() +{ + disconnect(m_videoFrameConnection); + + if (m_primaryActiveVideoSource && m_videoSink) { + // deliver frames directly to video sink; + // AutoConnection type might be a pessimization due to an extra queuing + // TODO: investigate and integrate direct connection + m_videoFrameConnection = + connect(m_primaryActiveVideoSource, &QPlatformVideoSource::newVideoFrame, + m_videoSink, &QVideoSink::setVideoFrame); + } +} + +void QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource() +{ + auto sources = activeVideoSources(); + auto source = sources.empty() ? nullptr : sources.front(); + if (std::exchange(m_primaryActiveVideoSource, source) != source) + emit primaryActiveVideoSourceChanged(); +} + +template<typename VideoSource> +bool QFFmpegMediaCaptureSession::setVideoSource(QPointer<VideoSource> &source, + VideoSource *newSource) +{ + if (source == newSource) + return false; + + if (auto prevSource = std::exchange(source, newSource)) { + prevSource->setCaptureSession(nullptr); + prevSource->disconnect(this); + } + + if (source) { + source->setCaptureSession(this); + connect(source, &QPlatformVideoSource::activeChanged, this, + &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource); + connect(source, &QObject::destroyed, this, + &QFFmpegMediaCaptureSession::updatePrimaryActiveVideoSource, Qt::QueuedConnection); + } + + updatePrimaryActiveVideoSource(); + + return true; } -void QFFmpegMediaCaptureSession::newVideoFrame(const QVideoFrame &frame) +QPlatformVideoSource *QFFmpegMediaCaptureSession::primaryActiveVideoSource() { - if (m_videoSink) - m_videoSink->setVideoFrame(frame); + return m_primaryActiveVideoSource; } +std::vector<QPlatformAudioBufferInputBase *> QFFmpegMediaCaptureSession::activeAudioInputs() const +{ + std::vector<QPlatformAudioBufferInputBase *> result; + if (m_audioInput) + result.push_back(m_audioInput); + + if (m_audioBufferInput) + result.push_back(m_audioBufferInput); + + return result; +} QT_END_NAMESPACE + +#include "moc_qffmpegmediacapturesession_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h index 353cab3e7..25340dad5 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediacapturesession_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGMEDIACAPTURESESSION_H #define QFFMPEGMEDIACAPTURESESSION_H @@ -53,24 +17,43 @@ #include <private/qplatformmediacapture_p.h> #include <private/qplatformmediaintegration_p.h> +#include "qpointer.h" +#include "qiodevice.h" QT_BEGIN_NAMESPACE class QFFmpegMediaRecorder; class QFFmpegImageCapture; class QVideoFrame; +class QAudioSink; +class QFFmpegAudioInput; +class QAudioBuffer; +class QPlatformVideoSource; +class QPlatformAudioBufferInput; +class QPlatformAudioBufferInputBase; class QFFmpegMediaCaptureSession : public QPlatformMediaCaptureSession { Q_OBJECT public: + using VideoSources = std::vector<QPointer<QPlatformVideoSource>>; + QFFmpegMediaCaptureSession(); - virtual ~QFFmpegMediaCaptureSession(); + ~QFFmpegMediaCaptureSession() override; QPlatformCamera *camera() override; void setCamera(QPlatformCamera *camera) override; + QPlatformSurfaceCapture *screenCapture() override; + void setScreenCapture(QPlatformSurfaceCapture *) override; + + QPlatformSurfaceCapture *windowCapture() override; + void setWindowCapture(QPlatformSurfaceCapture *) override; + + QPlatformVideoFrameInput *videoFrameInput() override; + void setVideoFrameInput(QPlatformVideoFrameInput *) override; + QPlatformImageCapture *imageCapture() override; void setImageCapture(QPlatformImageCapture *imageCapture) override; @@ -78,21 +61,50 @@ public: void setMediaRecorder(QPlatformMediaRecorder *recorder) override; void setAudioInput(QPlatformAudioInput *input) override; - QPlatformAudioInput *audioInput() { return m_audioInput; } + QPlatformAudioInput *audioInput() const; + + void setAudioBufferInput(QPlatformAudioBufferInput *input) override; void setVideoPreview(QVideoSink *sink) override; void setAudioOutput(QPlatformAudioOutput *output) override; -public Q_SLOTS: - void newVideoFrame(const QVideoFrame &frame); + QPlatformVideoSource *primaryActiveVideoSource(); + + // it might be moved to the base class, but it needs QPlatformAudioInput + // to be QPlatformAudioBufferInputBase, which might not make sense + std::vector<QPlatformAudioBufferInputBase *> activeAudioInputs() const; + +private Q_SLOTS: + void updateAudioSink(); + void updateVolume(); + void updateVideoFrameConnection(); + void updatePrimaryActiveVideoSource(); + +Q_SIGNALS: + void primaryActiveVideoSourceChanged(); private: - QPlatformCamera *m_camera = nullptr; - QPlatformAudioInput *m_audioInput = nullptr; + template<typename VideoSource> + bool setVideoSource(QPointer<VideoSource> &source, VideoSource *newSource); + + QPointer<QPlatformCamera> m_camera; + QPointer<QPlatformSurfaceCapture> m_screenCapture; + QPointer<QPlatformSurfaceCapture> m_windowCapture; + QPointer<QPlatformVideoFrameInput> m_videoFrameInput; + QPointer<QPlatformVideoSource> m_primaryActiveVideoSource; + + QPointer<QFFmpegAudioInput> m_audioInput; + QPointer<QPlatformAudioBufferInput> m_audioBufferInput; + QFFmpegImageCapture *m_imageCapture = nullptr; QFFmpegMediaRecorder *m_mediaRecorder = nullptr; QPlatformAudioOutput *m_audioOutput = nullptr; QVideoSink *m_videoSink = nullptr; + std::unique_ptr<QAudioSink> m_audioSink; + QPointer<QIODevice> m_audioIODevice; + qsizetype m_audioBufferSize = 0; + + QMetaObject::Connection m_videoFrameConnection; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo.cpp index 0ff4cc68b..b57ffb4b3 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo.cpp @@ -1,49 +1,16 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegmediaformatinfo_p.h" -#include "qffmpeg_p.h" #include "qaudioformat.h" #include "qimagewriter.h" +#include <qloggingcategory.h> + QT_BEGIN_NAMESPACE +Q_STATIC_LOGGING_CATEGORY(qLcMediaFormatInfo, "qt.multimedia.ffmpeg.mediaformatinfo") + static struct { AVCodecID id; QMediaFormat::VideoCodec codec; @@ -162,7 +129,7 @@ static const AVOutputFormat *avFormatForFormat(QMediaFormat::FileFormat format) QFFmpegMediaFormatInfo::QFFmpegMediaFormatInfo() { - qDebug() << ">>>> listing codecs"; + qCDebug(qLcMediaFormatInfo) << ">>>> listing codecs"; QList<QMediaFormat::AudioCodec> audioEncoders; QList<QMediaFormat::AudioCodec> extraAudioDecoders; @@ -171,8 +138,8 @@ QFFmpegMediaFormatInfo::QFFmpegMediaFormatInfo() const AVCodecDescriptor *descriptor = nullptr; while ((descriptor = avcodec_descriptor_next(descriptor))) { - bool canEncode = (avcodec_find_encoder(descriptor->id) != nullptr); - bool canDecode = (avcodec_find_decoder(descriptor->id) != nullptr); + const bool canEncode = QFFmpeg::findAVEncoder(descriptor->id) != nullptr; + const bool canDecode = QFFmpeg::findAVDecoder(descriptor->id) != nullptr; auto videoCodec = videoCodecForAVCodecId(descriptor->id); auto audioCodec = audioCodecForAVCodecId(descriptor->id); if (descriptor->type == AVMEDIA_TYPE_VIDEO && videoCodec != QMediaFormat::VideoCodec::Unspecified) { @@ -183,9 +150,8 @@ QFFmpegMediaFormatInfo::QFFmpegMediaFormatInfo() if (!extraVideoDecoders.contains(videoCodec)) extraVideoDecoders.append(videoCodec); } - } - - else if (descriptor->type == AVMEDIA_TYPE_AUDIO && audioCodec != QMediaFormat::AudioCodec::Unspecified) { + } else if (descriptor->type == AVMEDIA_TYPE_AUDIO + && audioCodec != QMediaFormat::AudioCodec::Unspecified) { if (canEncode) { if (!audioEncoders.contains(audioCodec)) audioEncoders.append(audioCodec); @@ -197,14 +163,14 @@ QFFmpegMediaFormatInfo::QFFmpegMediaFormatInfo() } // get demuxers -// qDebug() << ">>>> Muxers"; +// qCDebug(qLcMediaFormatInfo) << ">>>> Muxers"; void *opaque = nullptr; const AVOutputFormat *outputFormat = nullptr; while ((outputFormat = av_muxer_iterate(&opaque))) { auto mediaFormat = formatForAVFormat(outputFormat); if (mediaFormat == QMediaFormat::UnspecifiedFormat) continue; -// qDebug() << " mux:" << outputFormat->name << outputFormat->long_name << outputFormat->mime_type << outputFormat->extensions << mediaFormat; +// qCDebug(qLcMediaFormatInfo) << " mux:" << outputFormat->name << outputFormat->long_name << outputFormat->mime_type << outputFormat->extensions << mediaFormat; CodecMap encoder; encoder.format = mediaFormat; @@ -214,7 +180,7 @@ QFFmpegMediaFormatInfo::QFFmpegMediaFormatInfo() // only add the codec if it can be used with this container if (avformat_query_codec(outputFormat, id, FF_COMPLIANCE_NORMAL) == 1) { // add codec for container -// qDebug() << " " << codec << Qt::hex << av_codec_get_tag(outputFormat->codec_tag, id); +// qCDebug(qLcMediaFormatInfo) << " " << codec << Qt::hex << av_codec_get_tag(outputFormat->codec_tag, id); encoder.audio.append(codec); } } @@ -223,7 +189,7 @@ QFFmpegMediaFormatInfo::QFFmpegMediaFormatInfo() // only add the codec if it can be used with this container if (avformat_query_codec(outputFormat, id, FF_COMPLIANCE_NORMAL) == 1) { // add codec for container -// qDebug() << " " << codec << Qt::hex << av_codec_get_tag(outputFormat->codec_tag, id); +// qCDebug(qLcMediaFormatInfo) << " " << codec << Qt::hex << av_codec_get_tag(outputFormat->codec_tag, id); encoder.video.append(codec); } } @@ -258,7 +224,16 @@ QFFmpegMediaFormatInfo::QFFmpegMediaFormatInfo() // can encode. That's a safe subset. decoders = encoders; -// qDebug() << "extraDecoders:" << extraAudioDecoders << extraVideoDecoders; +#ifdef Q_OS_WINDOWS + // MediaFoundation HVEC encoder fails when processing frames + for (auto &encoder : encoders) { + auto h265index = encoder.video.indexOf(QMediaFormat::VideoCodec::H265); + if (h265index >= 0) + encoder.video.removeAt(h265index); + } +#endif + +// qCDebug(qLcMediaFormatInfo) << "extraDecoders:" << extraAudioDecoders << extraVideoDecoders; // FFmpeg can currently only decode WMA and WMV, not encode if (extraAudioDecoders.contains(QMediaFormat::AudioCodec::WMA)) { decoders[QMediaFormat::WMA].audio.append(QMediaFormat::AudioCodec::WMA); @@ -524,11 +499,17 @@ QAudioFormat QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(AVCodecParam QAudioFormat format; format.setSampleFormat(sampleFormat(AVSampleFormat(codecpar->format))); format.setSampleRate(codecpar->sample_rate); - - auto channelLayout = codecpar->channel_layout; +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + uint64_t channelLayout = codecpar->channel_layout; if (!channelLayout) channelLayout = avChannelLayout(QAudioFormat::defaultChannelConfigForChannelCount(codecpar->channels)); - +#else + uint64_t channelLayout = 0; + if (codecpar->ch_layout.order == AV_CHANNEL_ORDER_NATIVE) + channelLayout = codecpar->ch_layout.u.mask; + else + channelLayout = avChannelLayout(QAudioFormat::defaultChannelConfigForChannelCount(codecpar->ch_layout.nb_channels)); +#endif format.setChannelConfig(channelConfigForAVLayout(channelLayout)); return format; } diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo_p.h index ed941d0bd..52fcf6f72 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaformatinfo_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFmpegMediaFormatInfo_H #define QFFmpegMediaFormatInfo_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp index 791b59d47..ba1fff3b3 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <QtMultimedia/private/qplatformmediaplugin_p.h> #include <qcameradevice.h> @@ -48,27 +12,56 @@ #include "qffmpegimagecapture_p.h" #include "qffmpegaudioinput_p.h" #include "qffmpegaudiodecoder_p.h" +#include "qffmpegresampler_p.h" +#include "qgrabwindowsurfacecapture_p.h" +#include "qffmpegconverter_p.h" #ifdef Q_OS_MACOS #include <VideoToolbox/VideoToolbox.h> + +#include "qcgcapturablewindows_p.h" +#include "qcgwindowcapture_p.h" +#include "qavfscreencapture_p.h" #endif #ifdef Q_OS_DARWIN #include "qavfcamera_p.h" + #elif defined(Q_OS_WINDOWS) #include "qwindowscamera_p.h" #include "qwindowsvideodevices_p.h" +#include "qffmpegscreencapture_dxgi_p.h" +#include "qwincapturablewindows_p.h" +#include "qgdiwindowcapture_p.h" #endif #ifdef Q_OS_ANDROID # include "jni.h" +# include "qandroidvideodevices_p.h" +# include "qandroidcamera_p.h" +# include "qandroidimagecapture_p.h" extern "C" { -# include <libavcodec/jni.h> +# include <libavutil/log.h> +# include <libavcodec/jni.h> } #endif #if QT_CONFIG(linux_v4l) #include "qv4l2camera_p.h" +#include "qv4l2cameradevices_p.h" +#endif + +#if QT_CONFIG(cpp_winrt) +#include "qffmpegwindowcapture_uwp_p.h" +#endif + +#if QT_CONFIG(xlib) +#include "qx11surfacecapture_p.h" +#include "qx11capturablewindows_p.h" +#endif + +#if QT_CONFIG(eglfs) +#include "qeglfsscreencapture_p.h" #endif QT_BEGIN_NAMESPACE @@ -85,62 +78,140 @@ public: QPlatformMediaIntegration* create(const QString &name) override { - if (name == QLatin1String("ffmpeg")) + if (name == u"ffmpeg") return new QFFmpegMediaIntegration; return nullptr; } }; -QFFmpegMediaIntegration::QFFmpegMediaIntegration() +bool thread_local FFmpegLogsEnabledInThread = true; +static bool UseCustomFFmpegLogger = false; + +static void qffmpegLogCallback(void *ptr, int level, const char *fmt, va_list vl) { - m_formatsInfo = new QFFmpegMediaFormatInfo(); + if (!FFmpegLogsEnabledInThread) + return; -#if QT_CONFIG(linux_v4l) - m_videoDevices = new QV4L2CameraDevices(this); + if (!UseCustomFFmpegLogger) + return av_log_default_callback(ptr, level, fmt, vl); + + // filter logs above the chosen level and AV_LOG_QUIET (negative level) + if (level < 0 || level > av_log_get_level()) + return; + + QString message = QStringLiteral("FFmpeg log: %1").arg(QString::vasprintf(fmt, vl)); + if (message.endsWith("\n")) + message.removeLast(); + + if (level == AV_LOG_DEBUG || level == AV_LOG_TRACE) + qDebug() << message; + else if (level == AV_LOG_VERBOSE || level == AV_LOG_INFO) + qInfo() << message; + else if (level == AV_LOG_WARNING) + qWarning() << message; + else if (level == AV_LOG_ERROR || level == AV_LOG_FATAL || level == AV_LOG_PANIC) + qCritical() << message; +} + +static void setupFFmpegLogger() +{ + if (qEnvironmentVariableIsSet("QT_FFMPEG_DEBUG")) { + av_log_set_level(AV_LOG_DEBUG); + UseCustomFFmpegLogger = true; + } + + av_log_set_callback(&qffmpegLogCallback); +} + +static QPlatformSurfaceCapture *createScreenCaptureByBackend(QString backend) +{ + if (backend == u"grabwindow") + return new QGrabWindowSurfaceCapture(QPlatformSurfaceCapture::ScreenSource{}); + +#if QT_CONFIG(eglfs) + if (backend == u"eglfs") + return new QEglfsScreenCapture; #endif -#ifdef Q_OS_DARWIN - m_videoDevices = new QAVFVideoDevices(this); + +#if QT_CONFIG(xlib) + if (backend == u"x11") + return new QX11SurfaceCapture(QPlatformSurfaceCapture::ScreenSource{}); #elif defined(Q_OS_WINDOWS) - m_videoDevices = new QWindowsVideoDevices(this); + if (backend == u"dxgi") + return new QFFmpegScreenCaptureDxgi; +#elif defined(Q_OS_MACOS) + if (backend == u"avf") + return new QAVFScreenCapture; #endif + return nullptr; +} -#ifndef QT_NO_DEBUG - qDebug() << "Available HW decoding frameworks:"; - AVHWDeviceType type = AV_HWDEVICE_TYPE_NONE; - while ((type = av_hwdevice_iterate_types(type)) != AV_HWDEVICE_TYPE_NONE) - qDebug() << " " << av_hwdevice_get_type_name(type); +static QPlatformSurfaceCapture *createWindowCaptureByBackend(QString backend) +{ + if (backend == u"grabwindow") + return new QGrabWindowSurfaceCapture(QPlatformSurfaceCapture::WindowSource{}); + +#if QT_CONFIG(xlib) + if (backend == u"x11") + return new QX11SurfaceCapture(QPlatformSurfaceCapture::WindowSource{}); +#elif defined(Q_OS_WINDOWS) + if (backend == u"gdi") + return new QGdiWindowCapture; +#if QT_CONFIG(cpp_winrt) + if (backend == u"uwp") + return new QFFmpegWindowCaptureUwp; #endif +#elif defined(Q_OS_MACOS) + if (backend == u"cg") + return new QCGWindowCapture; +#endif + return nullptr; } -QFFmpegMediaIntegration::~QFFmpegMediaIntegration() +QFFmpegMediaIntegration::QFFmpegMediaIntegration() + : QPlatformMediaIntegration(QLatin1String("ffmpeg")) { - delete m_formatsInfo; + setupFFmpegLogger(); + +#ifndef QT_NO_DEBUG + qDebug() << "Available HW decoding frameworks:"; + for (auto type : QFFmpeg::HWAccel::decodingDeviceTypes()) + qDebug() << " " << av_hwdevice_get_type_name(type); + + qDebug() << "Available HW encoding frameworks:"; + for (auto type : QFFmpeg::HWAccel::encodingDeviceTypes()) + qDebug() << " " << av_hwdevice_get_type_name(type); +#endif } -QPlatformMediaFormatInfo *QFFmpegMediaIntegration::formatInfo() +QMaybe<QPlatformAudioDecoder *> QFFmpegMediaIntegration::createAudioDecoder(QAudioDecoder *decoder) { - return m_formatsInfo; + return new QFFmpegAudioDecoder(decoder); } -QPlatformAudioDecoder *QFFmpegMediaIntegration::createAudioDecoder(QAudioDecoder *decoder) +QMaybe<std::unique_ptr<QPlatformAudioResampler>> +QFFmpegMediaIntegration::createAudioResampler(const QAudioFormat &inputFormat, + const QAudioFormat &outputFormat) { - return new QFFmpegAudioDecoder(decoder); + return { std::make_unique<QFFmpegResampler>(inputFormat, outputFormat) }; } -QPlatformMediaCaptureSession *QFFmpegMediaIntegration::createCaptureSession() +QMaybe<QPlatformMediaCaptureSession *> QFFmpegMediaIntegration::createCaptureSession() { return new QFFmpegMediaCaptureSession(); } -QPlatformMediaPlayer *QFFmpegMediaIntegration::createPlayer(QMediaPlayer *player) +QMaybe<QPlatformMediaPlayer *> QFFmpegMediaIntegration::createPlayer(QMediaPlayer *player) { return new QFFmpegMediaPlayer(player); } -QPlatformCamera *QFFmpegMediaIntegration::createCamera(QCamera *camera) +QMaybe<QPlatformCamera *> QFFmpegMediaIntegration::createCamera(QCamera *camera) { #ifdef Q_OS_DARWIN return new QAVFCamera(camera); +#elif defined(Q_OS_ANDROID) + return new QAndroidCamera(camera); #elif QT_CONFIG(linux_v4l) return new QV4L2Camera(camera); #elif defined(Q_OS_WINDOWS) @@ -151,27 +222,131 @@ QPlatformCamera *QFFmpegMediaIntegration::createCamera(QCamera *camera) #endif } -QPlatformMediaRecorder *QFFmpegMediaIntegration::createRecorder(QMediaRecorder *recorder) +QPlatformSurfaceCapture *QFFmpegMediaIntegration::createScreenCapture(QScreenCapture *) +{ + static const QString screenCaptureBackend = qgetenv("QT_SCREEN_CAPTURE_BACKEND").toLower(); + + if (!screenCaptureBackend.isEmpty()) { + if (auto screenCapture = createScreenCaptureByBackend(screenCaptureBackend)) + return screenCapture; + + qWarning() << "Not supported QT_SCREEN_CAPTURE_BACKEND:" << screenCaptureBackend; + } + +#if QT_CONFIG(xlib) + if (QX11SurfaceCapture::isSupported()) + return new QX11SurfaceCapture(QPlatformSurfaceCapture::ScreenSource{}); +#endif + +#if QT_CONFIG(eglfs) + if (QEglfsScreenCapture::isSupported()) + return new QEglfsScreenCapture; +#endif + +#if defined(Q_OS_WINDOWS) + return new QFFmpegScreenCaptureDxgi; +#elif defined(Q_OS_MACOS) // TODO: probably use it for iOS as well + return new QAVFScreenCapture; +#else + return new QGrabWindowSurfaceCapture(QPlatformSurfaceCapture::ScreenSource{}); +#endif +} + +QPlatformSurfaceCapture *QFFmpegMediaIntegration::createWindowCapture(QWindowCapture *) +{ + static const QString windowCaptureBackend = qgetenv("QT_WINDOW_CAPTURE_BACKEND").toLower(); + + if (!windowCaptureBackend.isEmpty()) { + if (auto windowCapture = createWindowCaptureByBackend(windowCaptureBackend)) + return windowCapture; + + qWarning() << "Not supported QT_WINDOW_CAPTURE_BACKEND:" << windowCaptureBackend; + } + +#if QT_CONFIG(xlib) + if (QX11SurfaceCapture::isSupported()) + return new QX11SurfaceCapture(QPlatformSurfaceCapture::WindowSource{}); +#endif + +#if defined(Q_OS_WINDOWS) +# if QT_CONFIG(cpp_winrt) + if (QFFmpegWindowCaptureUwp::isSupported()) + return new QFFmpegWindowCaptureUwp; +# endif + + return new QGdiWindowCapture; +#elif defined(Q_OS_MACOS) // TODO: probably use it for iOS as well + return new QCGWindowCapture; +#else + return new QGrabWindowSurfaceCapture(QPlatformSurfaceCapture::WindowSource{}); +#endif +} + +QMaybe<QPlatformMediaRecorder *> QFFmpegMediaIntegration::createRecorder(QMediaRecorder *recorder) { return new QFFmpegMediaRecorder(recorder); } -QPlatformImageCapture *QFFmpegMediaIntegration::createImageCapture(QImageCapture *imageCapture) +QMaybe<QPlatformImageCapture *> QFFmpegMediaIntegration::createImageCapture(QImageCapture *imageCapture) { +#if defined(Q_OS_ANDROID) + return new QAndroidImageCapture(imageCapture); +#else return new QFFmpegImageCapture(imageCapture); +#endif } -QPlatformVideoSink *QFFmpegMediaIntegration::createVideoSink(QVideoSink *sink) +QMaybe<QPlatformVideoSink *> QFFmpegMediaIntegration::createVideoSink(QVideoSink *sink) { return new QFFmpegVideoSink(sink); } -QPlatformAudioInput *QFFmpegMediaIntegration::createAudioInput(QAudioInput *input) +QMaybe<QPlatformAudioInput *> QFFmpegMediaIntegration::createAudioInput(QAudioInput *input) { return new QFFmpegAudioInput(input); } +QVideoFrame QFFmpegMediaIntegration::convertVideoFrame(QVideoFrame &srcFrame, + const QVideoFrameFormat &destFormat) +{ + return convertFrame(srcFrame, destFormat); +} + +QPlatformMediaFormatInfo *QFFmpegMediaIntegration::createFormatInfo() +{ + return new QFFmpegMediaFormatInfo; +} + +QPlatformVideoDevices *QFFmpegMediaIntegration::createVideoDevices() +{ +#if defined(Q_OS_ANDROID) + return new QAndroidVideoDevices(this); +#elif QT_CONFIG(linux_v4l) + return new QV4L2CameraDevices(this); +#elif defined Q_OS_DARWIN + return new QAVFVideoDevices(this); +#elif defined(Q_OS_WINDOWS) + return new QWindowsVideoDevices(this); +#else + return nullptr; +#endif +} + +QPlatformCapturableWindows *QFFmpegMediaIntegration::createCapturableWindows() +{ +#if QT_CONFIG(xlib) + if (QX11SurfaceCapture::isSupported()) + return new QX11CapturableWindows; +#elif defined Q_OS_MACOS + return new QCGCapturableWindows; +#elif defined(Q_OS_WINDOWS) + return new QWinCapturableWindows; +#endif + return nullptr; +} + #ifdef Q_OS_ANDROID + Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void * /*reserved*/) { static bool initialized = false; @@ -188,6 +363,9 @@ Q_DECL_EXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void * /*reserved*/) if (av_jni_set_java_vm(vm, nullptr)) return JNI_ERR; + if (!QAndroidCamera::registerNativeMethods()) + return JNI_ERR; + return JNI_VERSION_1_6; } #endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h index 17840c4d1..473a5f044 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaintegration_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERINTEGRATION_H #define QGSTREAMERINTEGRATION_H @@ -61,24 +25,31 @@ class QFFmpegMediaIntegration : public QPlatformMediaIntegration { public: QFFmpegMediaIntegration(); - ~QFFmpegMediaIntegration(); - static QFFmpegMediaIntegration *instance() { return static_cast<QFFmpegMediaIntegration *>(QPlatformMediaIntegration::instance()); } - QPlatformMediaFormatInfo *formatInfo() override; + QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *decoder) override; + QMaybe<std::unique_ptr<QPlatformAudioResampler>> createAudioResampler(const QAudioFormat &inputFormat, const QAudioFormat &outputFormat) override; + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *player) override; + QMaybe<QPlatformCamera *> createCamera(QCamera *) override; + QPlatformSurfaceCapture *createScreenCapture(QScreenCapture *) override; + QPlatformSurfaceCapture *createWindowCapture(QWindowCapture *) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *) override; - QPlatformAudioDecoder *createAudioDecoder(QAudioDecoder *decoder) override; - QPlatformMediaCaptureSession *createCaptureSession() override; - QPlatformMediaPlayer *createPlayer(QMediaPlayer *player) override; - QPlatformCamera *createCamera(QCamera *) override; - QPlatformMediaRecorder *createRecorder(QMediaRecorder *) override; - QPlatformImageCapture *createImageCapture(QImageCapture *) override; + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *sink) override; - QPlatformVideoSink *createVideoSink(QVideoSink *sink) override; - - QPlatformAudioInput *createAudioInput(QAudioInput *input) override; + QMaybe<QPlatformAudioInput *> createAudioInput(QAudioInput *input) override; // QPlatformAudioOutput *createAudioOutput(QAudioOutput *) override; - QFFmpegMediaFormatInfo *m_formatsInfo = nullptr; + QVideoFrame convertVideoFrame(QVideoFrame &srcFrame, + const QVideoFrameFormat &destFormat) override; + +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; + + QPlatformVideoDevices *createVideoDevices() override; + + QPlatformCapturableWindows *createCapturableWindows() override; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediametadata.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediametadata.cpp index 0d84440f0..db3878c09 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediametadata.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediametadata.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegmediametadata_p.h" #include <QDebug> @@ -44,14 +8,21 @@ #include <qurl.h> #include <qlocale.h> +#include <qloggingcategory.h> + QT_BEGIN_NAMESPACE +Q_STATIC_LOGGING_CATEGORY(qLcMetaData, "qt.multimedia.ffmpeg.metadata") + namespace { -struct { +struct ffmpegTagToMetaDataKey +{ const char *tag; QMediaMetaData::Key key; -} ffmpegTagToMetaDataKey[] = { +}; + +constexpr ffmpegTagToMetaDataKey ffmpegTagToMetaDataKey[] = { { "title", QMediaMetaData::Title }, { "comment", QMediaMetaData::Comment }, { "description", QMediaMetaData::Description }, @@ -80,7 +51,7 @@ struct { static QMediaMetaData::Key tagToKey(const char *tag) { - auto *map = ffmpegTagToMetaDataKey; + const auto *map = ffmpegTagToMetaDataKey; while (map->tag) { if (!strcmp(map->tag, tag)) return map->key; @@ -91,7 +62,7 @@ static QMediaMetaData::Key tagToKey(const char *tag) static const char *keyToTag(QMediaMetaData::Key key) { - auto *map = ffmpegTagToMetaDataKey; + const auto *map = ffmpegTagToMetaDataKey; while (map->tag) { if (map->key == key) return map->tag; @@ -103,12 +74,12 @@ static const char *keyToTag(QMediaMetaData::Key key) //internal void QFFmpegMetaData::addEntry(QMediaMetaData &metaData, AVDictionaryEntry *entry) { -// qDebug() << " checking:" << entry->key << entry->value; + qCDebug(qLcMetaData) << " checking:" << entry->key << entry->value; QByteArray tag(entry->key); QMediaMetaData::Key key = tagToKey(tag.toLower()); if (key == QMediaMetaData::Key(-1)) return; -// qDebug() << " adding" << key; + qCDebug(qLcMetaData) << " adding" << key; auto *map = &metaData; @@ -165,8 +136,6 @@ QMediaMetaData QFFmpegMetaData::fromAVMetaData(const AVDictionary *tags) QByteArray QFFmpegMetaData::value(const QMediaMetaData &metaData, QMediaMetaData::Key key) { -// qDebug() << " checking:" << entry->key << entry->value; - const int metaTypeId = keyType(key).id(); const QVariant val = metaData.value(key); switch (metaTypeId) { @@ -194,9 +163,8 @@ QByteArray QFFmpegMetaData::value(const QMediaMetaData &metaData, QMediaMetaData AVDictionary *QFFmpegMetaData::toAVMetaData(const QMediaMetaData &metaData) { - const QList<Key> keys = metaData.keys(); AVDictionary *dict = nullptr; - for (const auto &k : keys) { + for (const auto &&[k, v] : metaData.asKeyValueRange()) { const char *key = ::keyToTag(k); if (!key) continue; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediametadata_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediametadata_p.h index 33e14a89b..201287495 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediametadata_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediametadata_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGMEDIAMETADATA_H #define QFFMPEGMEDIAMETADATA_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp index a1614d08f..951144692 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer.cpp @@ -1,94 +1,133 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegmediaplayer_p.h" -#include "qffmpegdecoder_p.h" -#include "qffmpegmediaformatinfo_p.h" -#include "qlocale.h" -#include "qffmpeg_p.h" -#include "qffmpegmediametadata_p.h" -#include "qffmpegvideobuffer_p.h" #include "private/qplatformaudiooutput_p.h" #include "qvideosink.h" -#include "qaudiosink.h" #include "qaudiooutput.h" +#include "qaudiobufferoutput.h" -#include <qlocale.h> -#include <qthread.h> -#include <qatomic.h> -#include <qwaitcondition.h> -#include <qmutex.h> +#include "qffmpegplaybackengine_p.h" +#include <qiodevice.h> +#include <qvideosink.h> #include <qtimer.h> -#include <qqueue.h> +#include <QtConcurrent/QtConcurrent> #include <qloggingcategory.h> QT_BEGIN_NAMESPACE +namespace QFFmpeg { + +class CancelToken : public ICancelToken +{ +public: + + bool isCancelled() const override { return m_cancelled.load(std::memory_order_acquire); } + + void cancel() { m_cancelled.store(true, std::memory_order_release); } + +private: + std::atomic_bool m_cancelled = false; +}; + +} // namespace QFFmpeg + using namespace QFFmpeg; QFFmpegMediaPlayer::QFFmpegMediaPlayer(QMediaPlayer *player) : QPlatformMediaPlayer(player) { + m_positionUpdateTimer.setInterval(50); + m_positionUpdateTimer.setTimerType(Qt::PreciseTimer); + connect(&m_positionUpdateTimer, &QTimer::timeout, this, &QFFmpegMediaPlayer::updatePosition); } QFFmpegMediaPlayer::~QFFmpegMediaPlayer() { - delete decoder; -} + if (m_cancelToken) + m_cancelToken->cancel(); + + m_loadMedia.waitForFinished(); +}; qint64 QFFmpegMediaPlayer::duration() const { - return decoder ? decoder->m_duration/1000 : 0; + return m_playbackEngine ? m_playbackEngine->duration() / 1000 : 0; } void QFFmpegMediaPlayer::setPosition(qint64 position) { - if (decoder) - decoder->seek(position*1000); - if (state() == QMediaPlayer::StoppedState) - mediaStatusChanged(QMediaPlayer::LoadedMedia); + if (mediaStatus() == QMediaPlayer::LoadingMedia) + return; + + if (m_playbackEngine) { + m_playbackEngine->seek(position * 1000); + updatePosition(); + } + + mediaStatusChanged(QMediaPlayer::LoadedMedia); +} + +void QFFmpegMediaPlayer::updatePosition() +{ + positionChanged(m_playbackEngine ? m_playbackEngine->currentPosition() / 1000 : 0); +} + +void QFFmpegMediaPlayer::endOfStream() +{ + // stop update timer and report end position anyway + m_positionUpdateTimer.stop(); + QPointer currentPlaybackEngine(m_playbackEngine.get()); + positionChanged(duration()); + + // skip changing state and mediaStatus if playbackEngine has been recreated, + // e.g. if new media has been loaded as a response to positionChanged signal + if (currentPlaybackEngine) + stateChanged(QMediaPlayer::StoppedState); + if (currentPlaybackEngine) + mediaStatusChanged(QMediaPlayer::EndOfMedia); +} + +void QFFmpegMediaPlayer::onLoopChanged() +{ + // report about finish and start + // reporting both signals is a bit contraversial + // but it eshures the idea of notifications about + // imporatant position points. + // Also, it ensures more predictable flow for testing. + positionChanged(duration()); + positionChanged(0); + m_positionUpdateTimer.stop(); + m_positionUpdateTimer.start(); +} + +void QFFmpegMediaPlayer::onBuffered() +{ + if (mediaStatus() == QMediaPlayer::BufferingMedia) + mediaStatusChanged(QMediaPlayer::BufferedMedia); } float QFFmpegMediaPlayer::bufferProgress() const { - return 1.; + return m_bufferProgress; +} + +void QFFmpegMediaPlayer::mediaStatusChanged(QMediaPlayer::MediaStatus status) +{ + if (mediaStatus() == status) + return; + + const auto newBufferProgress = status == QMediaPlayer::BufferingMedia ? 0.25f // to be improved + : status == QMediaPlayer::BufferedMedia ? 1.f + : 0.f; + + if (!qFuzzyCompare(newBufferProgress, m_bufferProgress)) { + m_bufferProgress = newBufferProgress; + bufferProgressChanged(newBufferProgress); + } + + QPlatformMediaPlayer::mediaStatusChanged(status); } QMediaTimeRange QFFmpegMediaPlayer::availablePlaybackRanges() const @@ -103,11 +142,17 @@ qreal QFFmpegMediaPlayer::playbackRate() const void QFFmpegMediaPlayer::setPlaybackRate(qreal rate) { - if (m_playbackRate == rate) + const float effectiveRate = std::max(static_cast<float>(rate), 0.0f); + + if (qFuzzyCompare(m_playbackRate, effectiveRate)) return; - m_playbackRate = rate; - if (decoder) - decoder->setPlaybackRate(rate); + + m_playbackRate = effectiveRate; + + if (m_playbackEngine) + m_playbackEngine->setPlaybackRate(effectiveRate); + + playbackRateChanged(effectiveRate); } QUrl QFFmpegMediaPlayer::media() const @@ -120,95 +165,206 @@ const QIODevice *QFFmpegMediaPlayer::mediaStream() const return m_device; } +void QFFmpegMediaPlayer::handleIncorrectMedia(QMediaPlayer::MediaStatus status) +{ + seekableChanged(false); + audioAvailableChanged(false); + videoAvailableChanged(false); + metaDataChanged(); + mediaStatusChanged(status); + m_playbackEngine = nullptr; +}; + void QFFmpegMediaPlayer::setMedia(const QUrl &media, QIODevice *stream) { + // Wait for previous unfinished load attempts. + if (m_cancelToken) + m_cancelToken->cancel(); + + m_loadMedia.waitForFinished(); + m_url = media; m_device = stream; - if (decoder) - delete decoder; - decoder = nullptr; - - positionChanged(0); + m_playbackEngine = nullptr; if (media.isEmpty() && !stream) { - seekableChanged(false); - audioAvailableChanged(false); - videoAvailableChanged(false); - metaDataChanged(); - mediaStatusChanged(QMediaPlayer::NoMedia); + handleIncorrectMedia(QMediaPlayer::NoMedia); return; } mediaStatusChanged(QMediaPlayer::LoadingMedia); - decoder = new Decoder(this); - decoder->setMedia(media, stream); - decoder->setAudioSink(m_audioOutput); - decoder->setVideoSink(m_videoSink); + m_requestedStatus = QMediaPlayer::StoppedState; + + m_cancelToken = std::make_shared<CancelToken>(); + + // Load media asynchronously to keep GUI thread responsive while loading media + m_loadMedia = QtConcurrent::run([this, media, stream, cancelToken = m_cancelToken] { + // On worker thread + const MediaDataHolder::Maybe mediaHolder = + MediaDataHolder::create(media, stream, cancelToken); + + // Transition back to calling thread using invokeMethod because + // QFuture continuations back on calling thread may deadlock (QTBUG-117918) + QMetaObject::invokeMethod(this, [this, mediaHolder, cancelToken] { + setMediaAsync(mediaHolder, cancelToken); + }); + }); +} + +void QFFmpegMediaPlayer::setMediaAsync(QFFmpeg::MediaDataHolder::Maybe mediaDataHolder, + const std::shared_ptr<QFFmpeg::CancelToken> &cancelToken) +{ + Q_ASSERT(mediaStatus() == QMediaPlayer::LoadingMedia); + + // If loading was cancelled, we do not emit any signals about failing + // to load media (or any other events). The rationale is that cancellation + // either happens during destruction, where the signals are no longer + // of interest, or it happens as a response to user requesting to load + // another media file. In the latter case, we don't want to risk popping + // up error dialogs or similar. + if (cancelToken->isCancelled()) { + return; + } + + if (!mediaDataHolder) { + const auto [code, description] = mediaDataHolder.error(); + error(code, description); + handleIncorrectMedia(QMediaPlayer::MediaStatus::InvalidMedia); + return; + } + + m_playbackEngine = std::make_unique<PlaybackEngine>(); + + connect(m_playbackEngine.get(), &PlaybackEngine::endOfStream, this, + &QFFmpegMediaPlayer::endOfStream); + connect(m_playbackEngine.get(), &PlaybackEngine::errorOccured, this, + &QFFmpegMediaPlayer::error); + connect(m_playbackEngine.get(), &PlaybackEngine::loopChanged, this, + &QFFmpegMediaPlayer::onLoopChanged); + connect(m_playbackEngine.get(), &PlaybackEngine::buffered, this, + &QFFmpegMediaPlayer::onBuffered); + + m_playbackEngine->setMedia(std::move(*mediaDataHolder.value())); + + m_playbackEngine->setAudioBufferOutput(m_audioBufferOutput); + m_playbackEngine->setAudioSink(m_audioOutput); + m_playbackEngine->setVideoSink(m_videoSink); + + m_playbackEngine->setLoops(loops()); + m_playbackEngine->setPlaybackRate(m_playbackRate); + + durationChanged(duration()); + tracksChanged(); metaDataChanged(); - seekableChanged(decoder->isSeekable()); + seekableChanged(m_playbackEngine->isSeekable()); + + audioAvailableChanged( + !m_playbackEngine->streamInfo(QPlatformMediaPlayer::AudioStream).isEmpty()); + videoAvailableChanged( + !m_playbackEngine->streamInfo(QPlatformMediaPlayer::VideoStream).isEmpty()); - audioAvailableChanged(!decoder->m_streamMap[QPlatformMediaPlayer::AudioStream].isEmpty()); - videoAvailableChanged(!decoder->m_streamMap[QPlatformMediaPlayer::VideoStream].isEmpty()); + mediaStatusChanged(QMediaPlayer::LoadedMedia); - QMetaObject::invokeMethod(this, "delayedLoadedStatus", Qt::QueuedConnection); + if (m_requestedStatus != QMediaPlayer::StoppedState) { + if (m_requestedStatus == QMediaPlayer::PlayingState) + play(); + else if (m_requestedStatus == QMediaPlayer::PausedState) + pause(); + } } void QFFmpegMediaPlayer::play() { - if (!decoder) + if (mediaStatus() == QMediaPlayer::LoadingMedia) { + m_requestedStatus = QMediaPlayer::PlayingState; + return; + } + + if (!m_playbackEngine) return; - if (mediaStatus() == QMediaPlayer::EndOfMedia && state() == QMediaPlayer::StoppedState) - decoder->seek(0); - decoder->play(); + if (mediaStatus() == QMediaPlayer::EndOfMedia && state() == QMediaPlayer::StoppedState) { + m_playbackEngine->seek(0); + positionChanged(0); + } + + runPlayback(); +} + +void QFFmpegMediaPlayer::runPlayback() +{ + m_playbackEngine->play(); + m_positionUpdateTimer.start(); stateChanged(QMediaPlayer::PlayingState); - mediaStatusChanged(QMediaPlayer::BufferedMedia); + + if (mediaStatus() == QMediaPlayer::LoadedMedia || mediaStatus() == QMediaPlayer::EndOfMedia) + mediaStatusChanged(QMediaPlayer::BufferingMedia); } void QFFmpegMediaPlayer::pause() { - if (!decoder) + if (mediaStatus() == QMediaPlayer::LoadingMedia) { + m_requestedStatus = QMediaPlayer::PausedState; + return; + } + + if (!m_playbackEngine) return; - if (mediaStatus() == QMediaPlayer::EndOfMedia && state() == QMediaPlayer::StoppedState) - decoder->seek(0); - decoder->pause(); + + if (mediaStatus() == QMediaPlayer::EndOfMedia && state() == QMediaPlayer::StoppedState) { + m_playbackEngine->seek(0); + positionChanged(0); + } + m_playbackEngine->pause(); + m_positionUpdateTimer.stop(); stateChanged(QMediaPlayer::PausedState); - mediaStatusChanged(QMediaPlayer::BufferedMedia); + + if (mediaStatus() == QMediaPlayer::LoadedMedia || mediaStatus() == QMediaPlayer::EndOfMedia) + mediaStatusChanged(QMediaPlayer::BufferingMedia); } void QFFmpegMediaPlayer::stop() { - if (!decoder) + if (mediaStatus() == QMediaPlayer::LoadingMedia) { + m_requestedStatus = QMediaPlayer::StoppedState; + return; + } + + if (!m_playbackEngine) return; - decoder->stop(); + + m_playbackEngine->stop(); + m_positionUpdateTimer.stop(); + m_playbackEngine->seek(0); + positionChanged(0); stateChanged(QMediaPlayer::StoppedState); mediaStatusChanged(QMediaPlayer::LoadedMedia); } void QFFmpegMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) { - if (m_audioOutput == output) - return; - m_audioOutput = output; - if (decoder) - decoder->setAudioSink(output); + if (m_playbackEngine) + m_playbackEngine->setAudioSink(output); +} + +void QFFmpegMediaPlayer::setAudioBufferOutput(QAudioBufferOutput *output) { + m_audioBufferOutput = output; + if (m_playbackEngine) + m_playbackEngine->setAudioBufferOutput(output); } QMediaMetaData QFFmpegMediaPlayer::metaData() const { - return decoder ? decoder->m_metaData : QMediaMetaData{}; + return m_playbackEngine ? m_playbackEngine->metaData() : QMediaMetaData{}; } void QFFmpegMediaPlayer::setVideoSink(QVideoSink *sink) { - if (m_videoSink == sink) - return; - m_videoSink = sink; - if (decoder) - decoder->setVideoSink(sink); + if (m_playbackEngine) + m_playbackEngine->setVideoSink(sink); } QVideoSink *QFFmpegMediaPlayer::videoSink() const @@ -218,25 +374,38 @@ QVideoSink *QFFmpegMediaPlayer::videoSink() const int QFFmpegMediaPlayer::trackCount(TrackType type) { - return decoder ? decoder->m_streamMap[type].count() : 0; + return m_playbackEngine ? m_playbackEngine->streamInfo(type).count() : 0; } QMediaMetaData QFFmpegMediaPlayer::trackMetaData(TrackType type, int streamNumber) { - if (!decoder || streamNumber < 0 || streamNumber >= decoder->m_streamMap[type].count()) + if (!m_playbackEngine || streamNumber < 0 + || streamNumber >= m_playbackEngine->streamInfo(type).count()) return {}; - return decoder->m_streamMap[type].at(streamNumber).metaData; + return m_playbackEngine->streamInfo(type).at(streamNumber).metaData; } int QFFmpegMediaPlayer::activeTrack(TrackType type) { - return decoder ? decoder->m_requestedStreams[type] : -1; + return m_playbackEngine ? m_playbackEngine->activeTrack(type) : -1; } void QFFmpegMediaPlayer::setActiveTrack(TrackType type, int streamNumber) { - if (decoder) - decoder->setActiveTrack(type, streamNumber); + if (m_playbackEngine) + m_playbackEngine->setActiveTrack(type, streamNumber); + else + qWarning() << "Cannot set active track without open source"; +} + +void QFFmpegMediaPlayer::setLoops(int loops) +{ + if (m_playbackEngine) + m_playbackEngine->setLoops(loops); + + QPlatformMediaPlayer::setLoops(loops); } QT_END_NAMESPACE + +#include "moc_qffmpegmediaplayer_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h index 74d95b67c..4ab5701da 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediaplayer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only // // W A R N I N G @@ -53,13 +17,20 @@ #include <private/qplatformmediaplayer_p.h> #include <qmediametadata.h> +#include <qtimer.h> +#include <qpointer.h> +#include <qfuture.h> #include "qffmpeg_p.h" +#include "playbackengine/qffmpegmediadataholder_p.h" QT_BEGIN_NAMESPACE namespace QFFmpeg { -class Decoder; +class CancelToken; + +class PlaybackEngine; } + class QPlatformAudioOutput; class QFFmpegMediaPlayer : public QObject, public QPlatformMediaPlayer @@ -88,10 +59,10 @@ public: void pause() override; void stop() override; -// bool streamPlaybackSupported() const { return false; } - void setAudioOutput(QPlatformAudioOutput *) override; + void setAudioBufferOutput(QAudioBufferOutput *) override; + QMediaMetaData metaData() const override; void setVideoSink(QVideoSink *sink) override; @@ -101,21 +72,44 @@ public: QMediaMetaData trackMetaData(TrackType type, int streamNumber) override; int activeTrack(TrackType) override; void setActiveTrack(TrackType, int streamNumber) override; + void setLoops(int loops) override; - Q_INVOKABLE void delayedLoadedStatus() { mediaStatusChanged(QMediaPlayer::LoadedMedia); } +private: + void runPlayback(); + void handleIncorrectMedia(QMediaPlayer::MediaStatus status); + void setMediaAsync(QFFmpeg::MediaDataHolder::Maybe mediaDataHolder, + const std::shared_ptr<QFFmpeg::CancelToken> &cancelToken); + + void mediaStatusChanged(QMediaPlayer::MediaStatus); + +private slots: + void updatePosition(); + void endOfStream(); + void error(int error, const QString &errorString) + { + QPlatformMediaPlayer::error(error, errorString); + } + void onLoopChanged(); + void onBuffered(); private: - friend class QFFmpeg::Decoder; + QTimer m_positionUpdateTimer; + QMediaPlayer::PlaybackState m_requestedStatus = QMediaPlayer::StoppedState; - QFFmpeg::Decoder *decoder = nullptr; - void checkStreams(); + using PlaybackEngine = QFFmpeg::PlaybackEngine; + std::unique_ptr<PlaybackEngine> m_playbackEngine; QPlatformAudioOutput *m_audioOutput = nullptr; - QVideoSink *m_videoSink = nullptr; + QPointer<QAudioBufferOutput> m_audioBufferOutput; + QPointer<QVideoSink> m_videoSink; QUrl m_url; - QIODevice *m_device = nullptr; + QPointer<QIODevice> m_device; float m_playbackRate = 1.; + float m_bufferProgress = 0.f; + QFuture<void> m_loadMedia; + std::shared_ptr<QFFmpeg::CancelToken> m_cancelToken; // For interrupting ongoing + // network connection attempt }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder.cpp index 5acb50c1b..f30350f83 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder.cpp @@ -1,71 +1,30 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegmediarecorder_p.h" #include "qaudiodevice.h" #include <private/qmediastoragelocation_p.h> #include <private/qplatformcamera_p.h> +#include <private/qplatformsurfacecapture_p.h> #include "qaudiosource.h" #include "qffmpegaudioinput_p.h" #include "qaudiobuffer.h" -#include "qffmpegencoder_p.h" -#include "qffmpegmediaformatinfo_p.h" +#include "recordingengine/qffmpegrecordingengine_p.h" +#include "qffmpegmediacapturesession_p.h" #include <qdebug.h> -#include <qeventloop.h> -#include <qstandardpaths.h> -#include <qmimetype.h> #include <qloggingcategory.h> -Q_LOGGING_CATEGORY(qLcMediaEncoder, "qt.multimedia.encoder") +Q_STATIC_LOGGING_CATEGORY(qLcMediaEncoder, "qt.multimedia.ffmpeg.encoder"); -QFFmpegMediaRecorder::QFFmpegMediaRecorder(QMediaRecorder *parent) - : QPlatformMediaRecorder(parent) -{ -} +QT_BEGIN_NAMESPACE -QFFmpegMediaRecorder::~QFFmpegMediaRecorder() +QFFmpegMediaRecorder::QFFmpegMediaRecorder(QMediaRecorder *parent) : QPlatformMediaRecorder(parent) { - if (encoder) - encoder->finalize(); } +QFFmpegMediaRecorder::~QFFmpegMediaRecorder() = default; + bool QFFmpegMediaRecorder::isLocationWritable(const QUrl &) const { return true; @@ -73,7 +32,7 @@ bool QFFmpegMediaRecorder::isLocationWritable(const QUrl &) const void QFFmpegMediaRecorder::handleSessionError(QMediaRecorder::Error code, const QString &description) { - error(code, description); + updateError(code, description); stop(); } @@ -82,45 +41,69 @@ void QFFmpegMediaRecorder::record(QMediaEncoderSettings &settings) if (!m_session || state() != QMediaRecorder::StoppedState) return; - const auto hasVideo = m_session->camera() && m_session->camera()->isActive(); - const auto hasAudio = m_session->audioInput() != nullptr; + auto videoSources = m_session->activeVideoSources(); + auto audioInputs = m_session->activeAudioInputs(); + const auto hasVideo = !videoSources.empty(); + const auto hasAudio = !audioInputs.empty(); if (!hasVideo && !hasAudio) { - error(QMediaRecorder::ResourceError, QMediaRecorder::tr("No camera or audio input")); + updateError(QMediaRecorder::ResourceError, QMediaRecorder::tr("No video or audio input")); return; } - const auto audioOnly = settings.videoCodec() == QMediaFormat::VideoCodec::Unspecified; + if (outputDevice() && !outputLocation().isEmpty()) + qCWarning(qLcMediaEncoder) + << "Both outputDevice and outputLocation has been set to QMediaRecorder"; + + if (outputDevice() && !outputDevice()->isWritable()) + qCWarning(qLcMediaEncoder) << "Output device has been set but not it's not writable"; + + QString actualLocation; + auto formatContext = std::make_unique<QFFmpeg::EncodingFormatContext>(settings.fileFormat()); + + if (outputDevice() && outputDevice()->isWritable()) { + formatContext->openAVIO(outputDevice()); + } else { + actualLocation = findActualLocation(settings); + qCDebug(qLcMediaEncoder) << "recording new media to" << actualLocation; + formatContext->openAVIO(actualLocation); + } + + qCDebug(qLcMediaEncoder) << "requested format:" << settings.fileFormat() + << settings.audioCodec(); - auto primaryLocation = audioOnly ? QStandardPaths::MusicLocation : QStandardPaths::MoviesLocation; - auto container = settings.mimeType().preferredSuffix(); - auto location = QMediaStorageLocation::generateFileName(outputLocation().toLocalFile(), primaryLocation, container); + if (!formatContext->isAVIOOpen()) { + updateError(QMediaRecorder::LocationNotWritable, + QMediaRecorder::tr("Cannot open the output location for writing")); + return; + } - QUrl actualSink = QUrl::fromLocalFile(QDir::currentPath()).resolved(location); - qCDebug(qLcMediaEncoder) << "recording new video to" << actualSink; - qDebug() << "requested format:" << settings.fileFormat() << settings.audioCodec(); + m_recordingEngine.reset(new RecordingEngine(settings, std::move(formatContext))); + m_recordingEngine->setMetaData(m_metaData); - Q_ASSERT(!actualSink.isEmpty()); + connect(m_recordingEngine.get(), &QFFmpeg::RecordingEngine::durationChanged, this, + &QFFmpegMediaRecorder::newDuration); + connect(m_recordingEngine.get(), &QFFmpeg::RecordingEngine::finalizationDone, this, + &QFFmpegMediaRecorder::finalizationDone); + connect(m_recordingEngine.get(), &QFFmpeg::RecordingEngine::sessionError, this, + &QFFmpegMediaRecorder::handleSessionError); - encoder = new QFFmpeg::Encoder(settings, actualSink); - encoder->setMetaData(m_metaData); - connect(encoder, &QFFmpeg::Encoder::durationChanged, this, &QFFmpegMediaRecorder::newDuration); - connect(encoder, &QFFmpeg::Encoder::finalizationDone, this, &QFFmpegMediaRecorder::finalizationDone); - connect(encoder, &QFFmpeg::Encoder::error, this, &QFFmpegMediaRecorder::handleSessionError); + updateAutoStop(); - auto *audioInput = m_session->audioInput(); - if (audioInput) - encoder->addAudioInput(static_cast<QFFmpegAudioInput *>(audioInput)); + auto handleStreamInitializationError = [this](QMediaRecorder::Error code, + const QString &description) { + qCWarning(qLcMediaEncoder) << "Stream initialization error:" << description; + updateError(code, description); + }; - auto *camera = m_session->camera(); - if (camera) - encoder->addVideoSource(camera); + connect(m_recordingEngine.get(), &QFFmpeg::RecordingEngine::streamInitializationError, this, + handleStreamInitializationError); durationChanged(0); stateChanged(QMediaRecorder::RecordingState); - actualLocationChanged(QUrl::fromLocalFile(location)); + actualLocationChanged(QUrl::fromLocalFile(actualLocation)); - encoder->start(); + m_recordingEngine->initialize(audioInputs, videoSources); } void QFFmpegMediaRecorder::pause() @@ -128,8 +111,8 @@ void QFFmpegMediaRecorder::pause() if (!m_session || state() != QMediaRecorder::RecordingState) return; - Q_ASSERT(encoder); - encoder->setPaused(true); + Q_ASSERT(m_recordingEngine); + m_recordingEngine->setPaused(true); stateChanged(QMediaRecorder::PausedState); } @@ -139,8 +122,8 @@ void QFFmpegMediaRecorder::resume() if (!m_session || state() != QMediaRecorder::PausedState) return; - Q_ASSERT(encoder); - encoder->setPaused(false); + Q_ASSERT(m_recordingEngine); + m_recordingEngine->setPaused(false); stateChanged(QMediaRecorder::RecordingState); } @@ -153,12 +136,8 @@ void QFFmpegMediaRecorder::stop() if (input) static_cast<QFFmpegAudioInput *>(input)->setRunning(false); qCDebug(qLcMediaEncoder) << "stop"; - // ### all of the below should be done asynchronous. finalize() should do it's work in a thread - // to avoid blocking the UI in case of slow codecs - if (encoder) { - encoder->finalize(); - encoder = nullptr; - } + + m_recordingEngine.reset(); } void QFFmpegMediaRecorder::finalizationDone() @@ -178,9 +157,9 @@ QMediaMetaData QFFmpegMediaRecorder::metaData() const return m_metaData; } -void QFFmpegMediaRecorder::setCaptureSession(QPlatformMediaCaptureSession *session) +void QFFmpegMediaRecorder::setCaptureSession(QFFmpegMediaCaptureSession *session) { - auto *captureSession = static_cast<QFFmpegMediaCaptureSession *>(session); + auto *captureSession = session; if (m_session == captureSession) return; @@ -191,3 +170,31 @@ void QFFmpegMediaRecorder::setCaptureSession(QPlatformMediaCaptureSession *sessi if (!m_session) return; } + +void QFFmpegMediaRecorder::updateAutoStop() +{ + const bool autoStop = mediaRecorder()->autoStop(); + if (!m_recordingEngine || m_recordingEngine->autoStop() == autoStop) + return; + + if (autoStop) + connect(m_recordingEngine.get(), &QFFmpeg::RecordingEngine::autoStopped, this, + &QFFmpegMediaRecorder::stop); + else + disconnect(m_recordingEngine.get(), &QFFmpeg::RecordingEngine::autoStopped, this, + &QFFmpegMediaRecorder::stop); + + m_recordingEngine->setAutoStop(autoStop); +} + +void QFFmpegMediaRecorder::RecordingEngineDeleter::operator()( + RecordingEngine *recordingEngine) const +{ + // ### all of the below should be done asynchronous. finalize() should do it's work in a thread + // to avoid blocking the UI in case of slow codecs + recordingEngine->finalize(); +} + +QT_END_NAMESPACE + +#include "moc_qffmpegmediarecorder_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder_p.h b/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder_p.h index 8ca9c046d..af3ee1509 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegmediarecorder_p.h @@ -1,42 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGMEDIARECODER_H #define QFFMPEGMEDIARECODER_H @@ -53,9 +16,6 @@ // #include <private/qplatformmediarecorder_p.h> -#include "qffmpegmediacapturesession_p.h" - -#include "qffmpeg_p.h" QT_BEGIN_NAMESPACE @@ -63,9 +23,10 @@ class QAudioSource; class QAudioSourceIO; class QAudioBuffer; class QMediaMetaData; +class QFFmpegMediaCaptureSession; namespace QFFmpeg { -class Encoder; +class RecordingEngine; } class QFFmpegMediaRecorder : public QObject, public QPlatformMediaRecorder @@ -85,7 +46,9 @@ public: void setMetaData(const QMediaMetaData &) override; QMediaMetaData metaData() const override; - void setCaptureSession(QPlatformMediaCaptureSession *session); + void setCaptureSession(QFFmpegMediaCaptureSession *session); + + void updateAutoStop() override; private Q_SLOTS: void newDuration(qint64 d) { durationChanged(d); } @@ -93,10 +56,16 @@ private Q_SLOTS: void handleSessionError(QMediaRecorder::Error code, const QString &description); private: + using RecordingEngine = QFFmpeg::RecordingEngine; + struct RecordingEngineDeleter + { + void operator()(RecordingEngine *) const; + }; + QFFmpegMediaCaptureSession *m_session = nullptr; QMediaMetaData m_metaData; - QFFmpeg::Encoder *encoder = nullptr; + std::unique_ptr<RecordingEngine, RecordingEngineDeleter> m_recordingEngine; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp new file mode 100644 index 000000000..74e58203a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine.cpp @@ -0,0 +1,649 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include "qffmpegplaybackengine_p.h" + +#include "qvideosink.h" +#include "qaudiooutput.h" +#include "private/qplatformaudiooutput_p.h" +#include "private/qplatformvideosink_p.h" +#include "private/qaudiobufferoutput_p.h" +#include "qiodevice.h" +#include "playbackengine/qffmpegdemuxer_p.h" +#include "playbackengine/qffmpegstreamdecoder_p.h" +#include "playbackengine/qffmpegsubtitlerenderer_p.h" +#include "playbackengine/qffmpegvideorenderer_p.h" +#include "playbackengine/qffmpegaudiorenderer_p.h" + +#include <qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +Q_STATIC_LOGGING_CATEGORY(qLcPlaybackEngine, "qt.multimedia.ffmpeg.playbackengine"); + +// The helper is needed since on some compilers std::unique_ptr +// doesn't have a default constructor in the case of sizeof(CustomDeleter) > 0 +template<typename Array> +inline Array defaultObjectsArray() +{ + using T = typename Array::value_type; + return { T{ {}, {} }, T{ {}, {} }, T{ {}, {} } }; +} + +// TODO: investigate what's better: profile and try network case +// Most likely, shouldPauseStreams = false is better because of: +// - packet and frame buffers are not big, the saturration of the is pretty fast. +// - after any pause a user has some preloaded buffers, so the playback is +// supposed to be more stable in cases with a weak processor or bad internet. +// - the code is simplier, usage is more convenient. +// +static constexpr bool shouldPauseStreams = false; + +PlaybackEngine::PlaybackEngine() + : m_demuxer({}, {}), + m_streams(defaultObjectsArray<decltype(m_streams)>()), + m_renderers(defaultObjectsArray<decltype(m_renderers)>()) +{ + qCDebug(qLcPlaybackEngine) << "Create PlaybackEngine"; + qRegisterMetaType<QFFmpeg::Packet>(); + qRegisterMetaType<QFFmpeg::Frame>(); +} + +PlaybackEngine::~PlaybackEngine() { + qCDebug(qLcPlaybackEngine) << "Delete PlaybackEngine"; + + finalizeOutputs(); + forEachExistingObject([](auto &object) { object.reset(); }); + deleteFreeThreads(); +} + +void PlaybackEngine::onRendererFinished() +{ + auto isAtEnd = [this](auto trackType) { + return !m_renderers[trackType] || m_renderers[trackType]->isAtEnd(); + }; + + if (!isAtEnd(QPlatformMediaPlayer::VideoStream)) + return; + + if (!isAtEnd(QPlatformMediaPlayer::AudioStream)) + return; + + if (!isAtEnd(QPlatformMediaPlayer::SubtitleStream) && !hasMediaStream()) + return; + + if (std::exchange(m_state, QMediaPlayer::StoppedState) == QMediaPlayer::StoppedState) + return; + + finilizeTime(duration()); + + forceUpdate(); + + qCDebug(qLcPlaybackEngine) << "Playback engine end of stream"; + + emit endOfStream(); +} + +void PlaybackEngine::onRendererLoopChanged(quint64 id, qint64 offset, int loopIndex) +{ + if (!hasRenderer(id)) + return; + + if (loopIndex > m_currentLoopOffset.index) { + m_currentLoopOffset = { offset, loopIndex }; + emit loopChanged(); + } else if (loopIndex == m_currentLoopOffset.index && offset != m_currentLoopOffset.pos) { + qWarning() << "Unexpected offset for loop" << loopIndex << ":" << offset << "vs" + << m_currentLoopOffset.pos; + m_currentLoopOffset.pos = offset; + } +} + +void PlaybackEngine::onRendererSynchronized(quint64 id, std::chrono::steady_clock::time_point tp, + qint64 pos) +{ + if (!hasRenderer(id)) + return; + + Q_ASSERT(m_renderers[QPlatformMediaPlayer::AudioStream] + && m_renderers[QPlatformMediaPlayer::AudioStream]->id() == id); + + m_timeController.sync(tp, pos); + + forEachExistingObject<Renderer>([&](auto &renderer) { + if (id != renderer->id()) + renderer->syncSoft(tp, pos); + }); +} + +void PlaybackEngine::setState(QMediaPlayer::PlaybackState state) { + if (!m_media.avContext()) + return; + + if (state == m_state) + return; + + const auto prevState = std::exchange(m_state, state); + + if (m_state == QMediaPlayer::StoppedState) { + finalizeOutputs(); + finilizeTime(0); + } + + if (prevState == QMediaPlayer::StoppedState || m_state == QMediaPlayer::StoppedState) + recreateObjects(); + + if (prevState == QMediaPlayer::StoppedState) + triggerStepIfNeeded(); + + updateObjectsPausedState(); +} + +void PlaybackEngine::updateObjectsPausedState() +{ + const auto paused = m_state != QMediaPlayer::PlayingState; + m_timeController.setPaused(paused); + + forEachExistingObject([&](auto &object) { + bool objectPaused = false; + + if constexpr (std::is_same_v<decltype(*object), Renderer &>) + objectPaused = paused; + else if constexpr (shouldPauseStreams) { + auto streamPaused = [](bool p, auto &r) { + const auto needMoreFrames = r && r->stepInProgress(); + return p && !needMoreFrames; + }; + + if constexpr (std::is_same_v<decltype(*object), StreamDecoder &>) + objectPaused = streamPaused(paused, renderer(object->trackType())); + else + objectPaused = std::accumulate(m_renderers.begin(), m_renderers.end(), paused, + streamPaused); + } + + object->setPaused(objectPaused); + }); +} + +void PlaybackEngine::ObjectDeleter::operator()(PlaybackEngineObject *object) const +{ + Q_ASSERT(engine); + if (!std::exchange(engine->m_threadsDirty, true)) + QMetaObject::invokeMethod(engine, &PlaybackEngine::deleteFreeThreads, Qt::QueuedConnection); + + object->kill(); +} + +void PlaybackEngine::registerObject(PlaybackEngineObject &object) +{ + connect(&object, &PlaybackEngineObject::error, this, &PlaybackEngine::errorOccured); + + auto threadName = objectThreadName(object); + auto &thread = m_threads[threadName]; + if (!thread) { + thread = std::make_unique<QThread>(); + thread->setObjectName(threadName); + thread->start(); + } + + Q_ASSERT(object.thread() != thread.get()); + object.moveToThread(thread.get()); +} + +PlaybackEngine::RendererPtr +PlaybackEngine::createRenderer(QPlatformMediaPlayer::TrackType trackType) +{ + switch (trackType) { + case QPlatformMediaPlayer::VideoStream: + return m_videoSink + ? createPlaybackEngineObject<VideoRenderer>(m_timeController, m_videoSink, m_media.rotation()) + : RendererPtr{ {}, {} }; + case QPlatformMediaPlayer::AudioStream: + return m_audioOutput || m_audioBufferOutput + ? createPlaybackEngineObject<AudioRenderer>(m_timeController, m_audioOutput, m_audioBufferOutput) + : RendererPtr{ {}, {} }; + case QPlatformMediaPlayer::SubtitleStream: + return m_videoSink + ? createPlaybackEngineObject<SubtitleRenderer>(m_timeController, m_videoSink) + : RendererPtr{ {}, {} }; + default: + return { {}, {} }; + } +} + +template<typename C, typename Action> +void PlaybackEngine::forEachExistingObject(Action &&action) +{ + auto handleNotNullObject = [&](auto &object) { + if constexpr (std::is_base_of_v<C, std::remove_reference_t<decltype(*object)>>) + if (object) + action(object); + }; + + handleNotNullObject(m_demuxer); + std::for_each(m_streams.begin(), m_streams.end(), handleNotNullObject); + std::for_each(m_renderers.begin(), m_renderers.end(), handleNotNullObject); +} + +template<typename Action> +void PlaybackEngine::forEachExistingObject(Action &&action) +{ + forEachExistingObject<PlaybackEngineObject>(std::forward<Action>(action)); +} + +void PlaybackEngine::seek(qint64 pos) +{ + pos = boundPosition(pos); + + m_timeController.setPaused(true); + m_timeController.sync(m_currentLoopOffset.pos + pos); + + forceUpdate(); +} + +void PlaybackEngine::setLoops(int loops) +{ + if (!isSeekable()) { + qWarning() << "Cannot set loops for non-seekable source"; + return; + } + + if (std::exchange(m_loops, loops) == loops) + return; + + qCDebug(qLcPlaybackEngine) << "set playback engine loops:" << loops << "prev loops:" << m_loops + << "index:" << m_currentLoopOffset.index; + + if (m_demuxer) + m_demuxer->setLoops(loops); +} + +void PlaybackEngine::triggerStepIfNeeded() +{ + if (m_state != QMediaPlayer::PausedState) + return; + + if (m_renderers[QPlatformMediaPlayer::VideoStream]) + m_renderers[QPlatformMediaPlayer::VideoStream]->doForceStep(); + + // TODO: maybe trigger SubtitleStream. + // If trigger it, we have to make seeking for the current subtitle frame more stable. + // Or set some timeout for seeking. +} + +QString PlaybackEngine::objectThreadName(const PlaybackEngineObject &object) +{ + QString result = object.metaObject()->className(); + if (auto stream = qobject_cast<const StreamDecoder *>(&object)) + result += QString::number(stream->trackType()); + + return result; +} + +void PlaybackEngine::setPlaybackRate(float rate) { + if (rate == playbackRate()) + return; + + m_timeController.setPlaybackRate(rate); + forEachExistingObject<Renderer>([rate](auto &renderer) { renderer->setPlaybackRate(rate); }); +} + +float PlaybackEngine::playbackRate() const { + return m_timeController.playbackRate(); +} + +void PlaybackEngine::recreateObjects() +{ + m_timeController.setPaused(true); + + forEachExistingObject([](auto &object) { object.reset(); }); + + createObjectsIfNeeded(); +} + +void PlaybackEngine::createObjectsIfNeeded() +{ + if (m_state == QMediaPlayer::StoppedState || !m_media.avContext()) + return; + + for (int i = 0; i < QPlatformMediaPlayer::NTrackTypes; ++i) + createStreamAndRenderer(static_cast<QPlatformMediaPlayer::TrackType>(i)); + + createDemuxer(); +} + +void PlaybackEngine::forceUpdate() +{ + recreateObjects(); + triggerStepIfNeeded(); + updateObjectsPausedState(); +} + +void PlaybackEngine::createStreamAndRenderer(QPlatformMediaPlayer::TrackType trackType) +{ + auto codec = codecForTrack(trackType); + + auto &renderer = m_renderers[trackType]; + + if (!codec) + return; + + if (!renderer) { + renderer = createRenderer(trackType); + + if (!renderer) + return; + + connect(renderer.get(), &Renderer::synchronized, this, + &PlaybackEngine::onRendererSynchronized); + + connect(renderer.get(), &Renderer::loopChanged, this, + &PlaybackEngine::onRendererLoopChanged); + + if constexpr (shouldPauseStreams) + connect(renderer.get(), &Renderer::forceStepDone, this, + &PlaybackEngine::updateObjectsPausedState); + + connect(renderer.get(), &PlaybackEngineObject::atEnd, this, + &PlaybackEngine::onRendererFinished); + } + + auto &stream = m_streams[trackType] = + createPlaybackEngineObject<StreamDecoder>(*codec, renderer->seekPosition()); + + Q_ASSERT(trackType == stream->trackType()); + + connect(stream.get(), &StreamDecoder::requestHandleFrame, renderer.get(), &Renderer::render); + connect(stream.get(), &PlaybackEngineObject::atEnd, renderer.get(), + &Renderer::onFinalFrameReceived); + connect(renderer.get(), &Renderer::frameProcessed, stream.get(), + &StreamDecoder::onFrameProcessed); +} + +std::optional<Codec> PlaybackEngine::codecForTrack(QPlatformMediaPlayer::TrackType trackType) +{ + const auto streamIndex = m_media.currentStreamIndex(trackType); + if (streamIndex < 0) + return {}; + + auto &result = m_codecs[trackType]; + + if (!result) { + qCDebug(qLcPlaybackEngine) + << "Create codec for stream:" << streamIndex << "trackType:" << trackType; + auto maybeCodec = + Codec::create(m_media.avContext()->streams[streamIndex], m_media.avContext()); + + if (!maybeCodec) { + emit errorOccured(QMediaPlayer::FormatError, + "Cannot create codec," + maybeCodec.error()); + return {}; + } + + result = maybeCodec.value(); + } + + return result; +} + +bool PlaybackEngine::hasMediaStream() const +{ + return m_renderers[QPlatformMediaPlayer::AudioStream] + || m_renderers[QPlatformMediaPlayer::VideoStream]; +} + +void PlaybackEngine::createDemuxer() +{ + std::array<int, QPlatformMediaPlayer::NTrackTypes> streamIndexes = { -1, -1, -1 }; + + bool hasStreams = false; + forEachExistingObject<StreamDecoder>([&](auto &stream) { + hasStreams = true; + const auto trackType = stream->trackType(); + streamIndexes[trackType] = m_media.currentStreamIndex(trackType); + }); + + if (!hasStreams) + return; + + const PositionWithOffset positionWithOffset{ currentPosition(false), m_currentLoopOffset }; + + m_demuxer = createPlaybackEngineObject<Demuxer>(m_media.avContext(), positionWithOffset, + streamIndexes, m_loops); + + connect(m_demuxer.get(), &Demuxer::packetsBuffered, this, &PlaybackEngine::buffered); + + forEachExistingObject<StreamDecoder>([&](auto &stream) { + connect(m_demuxer.get(), Demuxer::signalByTrackType(stream->trackType()), stream.get(), + &StreamDecoder::decode); + connect(m_demuxer.get(), &PlaybackEngineObject::atEnd, stream.get(), + &StreamDecoder::onFinalPacketReceived); + connect(stream.get(), &StreamDecoder::packetProcessed, m_demuxer.get(), + &Demuxer::onPacketProcessed); + }); + + if (!isSeekable() || duration() <= 0) { + // We need initial synchronization for such streams + forEachExistingObject([&](auto &object) { + using Type = std::remove_reference_t<decltype(*object)>; + if constexpr (!std::is_same_v<Type, Demuxer>) + connect(m_demuxer.get(), &Demuxer::firstPacketFound, object.get(), + &Type::setInitialPosition); + }); + + auto updateTimeController = [this](TimeController::TimePoint tp, qint64 pos) { + m_timeController.sync(tp, pos); + }; + + connect(m_demuxer.get(), &Demuxer::firstPacketFound, this, updateTimeController); + } +} + +void PlaybackEngine::deleteFreeThreads() { + m_threadsDirty = false; + auto freeThreads = std::move(m_threads); + + forEachExistingObject([&](auto &object) { + m_threads.insert(freeThreads.extract(objectThreadName(*object))); + }); + + for (auto &[name, thr] : freeThreads) + thr->quit(); + + for (auto &[name, thr] : freeThreads) + thr->wait(); +} + +void PlaybackEngine::setMedia(MediaDataHolder media) +{ + Q_ASSERT(!m_media.avContext()); // Playback engine does not support reloading media + Q_ASSERT(m_state == QMediaPlayer::StoppedState); + Q_ASSERT(m_threads.empty()); + + m_media = std::move(media); + updateVideoSinkSize(); +} + +void PlaybackEngine::setVideoSink(QVideoSink *sink) +{ + auto prev = std::exchange(m_videoSink, sink); + if (prev == sink) + return; + + updateVideoSinkSize(prev); + updateActiveVideoOutput(sink); + + if (!sink || !prev) { + // might need some improvements + forceUpdate(); + } +} + +void PlaybackEngine::setAudioSink(QPlatformAudioOutput *output) { + setAudioSink(output ? output->q : nullptr); +} + +void PlaybackEngine::setAudioSink(QAudioOutput *output) +{ + QAudioOutput *prev = std::exchange(m_audioOutput, output); + if (prev == output) + return; + + updateActiveAudioOutput(output); + + if (!output || !prev) { + // might need some improvements + forceUpdate(); + } +} + +void PlaybackEngine::setAudioBufferOutput(QAudioBufferOutput *output) +{ + QAudioBufferOutput *prev = std::exchange(m_audioBufferOutput, output); + if (prev == output) + return; + updateActiveAudioOutput(output); +} + +qint64 PlaybackEngine::currentPosition(bool topPos) const { + std::optional<qint64> pos; + + for (size_t i = 0; i < m_renderers.size(); ++i) { + const auto &renderer = m_renderers[i]; + if (!renderer) + continue; + + // skip subtitle stream for finding lower rendering position + if (!topPos && i == QPlatformMediaPlayer::SubtitleStream && hasMediaStream()) + continue; + + const auto rendererPos = renderer->lastPosition(); + pos = !pos ? rendererPos + : topPos ? std::max(*pos, rendererPos) + : std::min(*pos, rendererPos); + } + + if (!pos) + pos = m_timeController.currentPosition(); + + return boundPosition(*pos - m_currentLoopOffset.pos); +} + +qint64 PlaybackEngine::duration() const +{ + return m_media.duration(); +} + +bool PlaybackEngine::isSeekable() const { return m_media.isSeekable(); } + +const QList<MediaDataHolder::StreamInfo> & +PlaybackEngine::streamInfo(QPlatformMediaPlayer::TrackType trackType) const +{ + return m_media.streamInfo(trackType); +} + +const QMediaMetaData &PlaybackEngine::metaData() const +{ + return m_media.metaData(); +} + +int PlaybackEngine::activeTrack(QPlatformMediaPlayer::TrackType type) const +{ + return m_media.activeTrack(type); +} + +void PlaybackEngine::setActiveTrack(QPlatformMediaPlayer::TrackType trackType, int streamNumber) +{ + if (!m_media.setActiveTrack(trackType, streamNumber)) + return; + + m_codecs[trackType] = {}; + + m_renderers[trackType].reset(); + m_streams = defaultObjectsArray<decltype(m_streams)>(); + m_demuxer.reset(); + + updateVideoSinkSize(); + createObjectsIfNeeded(); + updateObjectsPausedState(); +} + +void PlaybackEngine::finilizeTime(qint64 pos) +{ + Q_ASSERT(pos >= 0 && pos <= duration()); + + m_timeController.setPaused(true); + m_timeController.sync(pos); + m_currentLoopOffset = {}; +} + +void PlaybackEngine::finalizeOutputs() +{ + if (m_audioBufferOutput) + updateActiveAudioOutput(static_cast<QAudioBufferOutput *>(nullptr)); + if (m_audioOutput) + updateActiveAudioOutput(static_cast<QAudioOutput *>(nullptr)); + updateActiveVideoOutput(nullptr, true); +} + +bool PlaybackEngine::hasRenderer(quint64 id) const +{ + return std::any_of(m_renderers.begin(), m_renderers.end(), + [id](auto &renderer) { return renderer && renderer->id() == id; }); +} + +template <typename AudioOutput> +void PlaybackEngine::updateActiveAudioOutput(AudioOutput *output) +{ + if (auto renderer = + qobject_cast<AudioRenderer *>(m_renderers[QPlatformMediaPlayer::AudioStream].get())) + renderer->setOutput(output); +} + +void PlaybackEngine::updateActiveVideoOutput(QVideoSink *sink, bool cleanOutput) +{ + if (auto renderer = qobject_cast<SubtitleRenderer *>( + m_renderers[QPlatformMediaPlayer::SubtitleStream].get())) + renderer->setOutput(sink, cleanOutput); + if (auto renderer = + qobject_cast<VideoRenderer *>(m_renderers[QPlatformMediaPlayer::VideoStream].get())) + renderer->setOutput(sink, cleanOutput); +} + +void PlaybackEngine::updateVideoSinkSize(QVideoSink *prevSink) +{ + auto platformVideoSink = m_videoSink ? m_videoSink->platformVideoSink() : nullptr; + if (!platformVideoSink) + return; + + if (prevSink && prevSink->platformVideoSink()) + platformVideoSink->setNativeSize(prevSink->platformVideoSink()->nativeSize()); + else { + const auto streamIndex = m_media.currentStreamIndex(QPlatformMediaPlayer::VideoStream); + if (streamIndex >= 0) { + const auto context = m_media.avContext(); + const auto stream = context->streams[streamIndex]; + const AVRational pixelAspectRatio = + av_guess_sample_aspect_ratio(context, stream, nullptr); + // auto size = metaData().value(QMediaMetaData::Resolution) + const QSize size = + qCalculateFrameSize({ stream->codecpar->width, stream->codecpar->height }, + { pixelAspectRatio.num, pixelAspectRatio.den }); + + platformVideoSink->setNativeSize(qRotatedFrameSize(size, m_media.rotation())); + } + } +} + +qint64 PlaybackEngine::boundPosition(qint64 position) const +{ + position = qMax(position, 0); + return duration() > 0 ? qMin(position, duration()) : position; +} +} + +QT_END_NAMESPACE + +#include "moc_qffmpegplaybackengine_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h new file mode 100644 index 000000000..50c94c955 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegplaybackengine_p.h @@ -0,0 +1,234 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGPLAYBACKENGINE_P_H +#define QFFMPEGPLAYBACKENGINE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +/* Playback engine design description. + * + * + * PLAYBACK ENGINE OBJECTS + * + * - Playback engine manages 7 objects inside, each one works in a separate thread. + * Each object inherits PlaybackEngineObject. The objects are: + * Demuxer + * Stream Decoders: audio, video, subtitles + * Renderers: audio, video, subtitles + * + * + * THREADS: + * + * - By default, each object works in a separate thread. It's easy to reconfigure + * to using several objects in thread. + * - New thread is allocated if a new object is created and the engine doesn't + * have free threads. If it does, the thread is to be reused. + * - If all objects for some thread are deleted, the thread becomes free and the engine + * postpones its termination. + * + * OBJECTS WEAK CONNECTIVITY + * + * - The objects know nothing about others and about PlaybackEngine. + * For any interactions the objects use slots/signals. + * + * - PlaybackEngine knows the objects object and is able to create/delete them and + * call their public methods. + * + */ + +#include "playbackengine/qffmpegplaybackenginedefs_p.h" +#include "playbackengine/qffmpegtimecontroller_p.h" +#include "playbackengine/qffmpegmediadataholder_p.h" +#include "playbackengine/qffmpegcodec_p.h" +#include "playbackengine/qffmpegpositionwithoffset_p.h" + +#include <QtCore/qpointer.h> + +#include <unordered_map> + +QT_BEGIN_NAMESPACE + +class QAudioSink; +class QVideoSink; +class QAudioOutput; +class QAudioBufferOutput; +class QFFmpegMediaPlayer; + +namespace QFFmpeg +{ + +class PlaybackEngine : public QObject +{ + Q_OBJECT +public: + PlaybackEngine(); + + ~PlaybackEngine() override; + + void setMedia(MediaDataHolder media); + + void setVideoSink(QVideoSink *sink); + + void setAudioSink(QAudioOutput *output); + + void setAudioSink(QPlatformAudioOutput *output); + + void setAudioBufferOutput(QAudioBufferOutput *output); + + void setState(QMediaPlayer::PlaybackState state); + + void play() { + setState(QMediaPlayer::PlayingState); + } + void pause() { + setState(QMediaPlayer::PausedState); + } + void stop() { + setState(QMediaPlayer::StoppedState); + } + + void seek(qint64 pos); + + void setLoops(int loopsCount); + + void setPlaybackRate(float rate); + + float playbackRate() const; + + void setActiveTrack(QPlatformMediaPlayer::TrackType type, int streamNumber); + + qint64 currentPosition(bool topPos = true) const; + + qint64 duration() const; + + bool isSeekable() const; + + const QList<MediaDataHolder::StreamInfo> & + streamInfo(QPlatformMediaPlayer::TrackType trackType) const; + + const QMediaMetaData &metaData() const; + + int activeTrack(QPlatformMediaPlayer::TrackType type) const; + +signals: + void endOfStream(); + void errorOccured(int, const QString &); + void loopChanged(); + void buffered(); + +protected: // objects managing + struct ObjectDeleter + { + void operator()(PlaybackEngineObject *) const; + + PlaybackEngine *engine = nullptr; + }; + + template<typename T> + using ObjectPtr = std::unique_ptr<T, ObjectDeleter>; + + using RendererPtr = ObjectPtr<Renderer>; + using StreamPtr = ObjectPtr<StreamDecoder>; + + template<typename T, typename... Args> + ObjectPtr<T> createPlaybackEngineObject(Args &&...args); + + virtual RendererPtr createRenderer(QPlatformMediaPlayer::TrackType trackType); + + template <typename AudioOutput> + void updateActiveAudioOutput(AudioOutput *output); + + void updateActiveVideoOutput(QVideoSink *sink, bool cleanOutput = false); + +private: + void createStreamAndRenderer(QPlatformMediaPlayer::TrackType trackType); + + void createDemuxer(); + + void registerObject(PlaybackEngineObject &object); + + template<typename C, typename Action> + void forEachExistingObject(Action &&action); + + template<typename Action> + void forEachExistingObject(Action &&action); + + void forceUpdate(); + + void recreateObjects(); + + void createObjectsIfNeeded(); + + void updateObjectsPausedState(); + + void deleteFreeThreads(); + + void onRendererSynchronized(quint64 id, std::chrono::steady_clock::time_point time, + qint64 trackTime); + + void onRendererFinished(); + + void onRendererLoopChanged(quint64 id, qint64 offset, int loopIndex); + + void triggerStepIfNeeded(); + + static QString objectThreadName(const PlaybackEngineObject &object); + + std::optional<Codec> codecForTrack(QPlatformMediaPlayer::TrackType trackType); + + bool hasMediaStream() const; + + void finilizeTime(qint64 pos); + + void finalizeOutputs(); + + bool hasRenderer(quint64 id) const; + + void updateVideoSinkSize(QVideoSink *prevSink = nullptr); + + qint64 boundPosition(qint64 position) const; + +private: + MediaDataHolder m_media; + + TimeController m_timeController; + + std::unordered_map<QString, std::unique_ptr<QThread>> m_threads; + bool m_threadsDirty = false; + + QPointer<QVideoSink> m_videoSink; + QPointer<QAudioOutput> m_audioOutput; + QPointer<QAudioBufferOutput> m_audioBufferOutput; + + QMediaPlayer::PlaybackState m_state = QMediaPlayer::StoppedState; + + ObjectPtr<Demuxer> m_demuxer; + std::array<StreamPtr, QPlatformMediaPlayer::NTrackTypes> m_streams; + std::array<RendererPtr, QPlatformMediaPlayer::NTrackTypes> m_renderers; + + std::array<std::optional<Codec>, QPlatformMediaPlayer::NTrackTypes> m_codecs; + int m_loops = QMediaPlayer::Once; + LoopOffset m_currentLoopOffset; +}; + +template<typename T, typename... Args> +PlaybackEngine::ObjectPtr<T> PlaybackEngine::createPlaybackEngineObject(Args &&...args) +{ + auto result = ObjectPtr<T>(new T(std::forward<Args>(args)...), { this }); + registerObject(*result); + return result; +} +} + +QT_END_NAMESPACE + +#endif // QFFMPEGPLAYBACKENGINE_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp b/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp index 911278f63..8a685b9fd 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegresampler.cpp @@ -1,117 +1,112 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegresampler_p.h" -#include "qffmpegdecoder_p.h" +#include "playbackengine/qffmpegcodec_p.h" #include "qffmpegmediaformatinfo_p.h" #include <qloggingcategory.h> -extern "C" { -#include <libavutil/opt.h> -} - -Q_LOGGING_CATEGORY(qLcResampler, "qt.multimedia.ffmpeg.resampler") +Q_STATIC_LOGGING_CATEGORY(qLcResampler, "qt.multimedia.ffmpeg.resampler") QT_BEGIN_NAMESPACE -namespace QFFmpeg +using namespace QFFmpeg; + +QFFmpegResampler::QFFmpegResampler(const QAudioFormat &inputFormat, const QAudioFormat &outputFormat) : + m_inputFormat(inputFormat), m_outputFormat(outputFormat) { + Q_ASSERT(inputFormat.isValid()); + Q_ASSERT(outputFormat.isValid()); + + m_resampler = + createResampleContext(AVAudioFormat(m_inputFormat), AVAudioFormat(m_outputFormat)); +} -Resampler::Resampler(const Codec *codec, const QAudioFormat &outputFormat) - : m_outputFormat(outputFormat) +QFFmpegResampler::QFFmpegResampler(const Codec *codec, const QAudioFormat &outputFormat, + qint64 startTime) + : m_outputFormat(outputFormat), m_startTime(startTime) { + Q_ASSERT(codec); + qCDebug(qLcResampler) << "createResampler"; const AVStream *audioStream = codec->stream(); - const auto *codecpar = audioStream->codecpar; if (!m_outputFormat.isValid()) // want the native format m_outputFormat = QFFmpegMediaFormatInfo::audioFormatFromCodecParameters(audioStream->codecpar); - QAudioFormat::ChannelConfig config = m_outputFormat.channelConfig(); - if (config == QAudioFormat::ChannelConfigUnknown) - config = QAudioFormat::defaultChannelConfigForChannelCount(m_outputFormat.channelCount()); - - auto inConfig = codecpar->channel_layout; - if (inConfig == 0) - inConfig = QFFmpegMediaFormatInfo::avChannelLayout(QAudioFormat::defaultChannelConfigForChannelCount(codecpar->channels)); - - qCDebug(qLcResampler) << "init resampler" << m_outputFormat.sampleRate() << config << codecpar->sample_rate; - resampler = swr_alloc_set_opts(nullptr, // we're allocating a new context - QFFmpegMediaFormatInfo::avChannelLayout(config), // out_ch_layout - QFFmpegMediaFormatInfo::avSampleFormat(m_outputFormat.sampleFormat()), // out_sample_fmt - m_outputFormat.sampleRate(), // out_sample_rate - inConfig, // in_ch_layout - AVSampleFormat(codecpar->format), // in_sample_fmt - codecpar->sample_rate, // in_sample_rate - 0, // log_offset - nullptr); - - // if we're not the master clock, we might need to handle clock adjustments, initialize for that - av_opt_set_double(resampler, "async", m_outputFormat.sampleRate()/50, 0); - - swr_init(resampler); + m_resampler = createResampleContext(AVAudioFormat(audioStream->codecpar), + AVAudioFormat(m_outputFormat)); } -Resampler::~Resampler() +QFFmpegResampler::~QFFmpegResampler() = default; + +QAudioBuffer QFFmpegResampler::resample(const char* data, size_t size) { - swr_free(&resampler); + if (!m_inputFormat.isValid()) + return {}; + + return resample(reinterpret_cast<const uint8_t **>(&data), + m_inputFormat.framesForBytes(static_cast<qint32>(size))); } -QAudioBuffer Resampler::resample(const AVFrame *frame) +QAudioBuffer QFFmpegResampler::resample(const AVFrame *frame) { - const int outSamples = swr_get_out_samples(resampler, frame->nb_samples); - QByteArray samples(m_outputFormat.bytesForFrames(outSamples), Qt::Uninitialized); - auto **in = const_cast<const uint8_t **>(frame->extended_data); + return resample(const_cast<const uint8_t **>(frame->extended_data), frame->nb_samples); +} + +QAudioBuffer QFFmpegResampler::resample(const uint8_t **inputData, int inputSamplesCount) +{ + const int maxOutSamples = adjustMaxOutSamples(inputSamplesCount); + + QByteArray samples(m_outputFormat.bytesForFrames(maxOutSamples), Qt::Uninitialized); auto *out = reinterpret_cast<uint8_t *>(samples.data()); - const int out_samples = swr_convert(resampler, &out, outSamples, - in, frame->nb_samples); - samples.resize(m_outputFormat.bytesForFrames(out_samples)); + const int outSamples = + swr_convert(m_resampler.get(), &out, maxOutSamples, inputData, inputSamplesCount); + + samples.resize(m_outputFormat.bytesForFrames(outSamples)); - qint64 startTime = m_outputFormat.durationForFrames(m_samplesProcessed); - m_samplesProcessed += out_samples; + const qint64 startTime = m_outputFormat.durationForFrames(m_samplesProcessed) + m_startTime; + m_samplesProcessed += outSamples; - qCDebug(qLcResampler) << " new frame" << startTime << "in_samples" << frame->nb_samples << out_samples << outSamples; - QAudioBuffer buffer(samples, m_outputFormat, startTime); - return buffer; + qCDebug(qLcResampler) << " new frame" << startTime << "in_samples" << inputSamplesCount + << outSamples << maxOutSamples; + return QAudioBuffer(samples, m_outputFormat, startTime); } +int QFFmpegResampler::adjustMaxOutSamples(int inputSamplesCount) +{ + int maxOutSamples = swr_get_out_samples(m_resampler.get(), inputSamplesCount); + + const auto remainingCompensationDistance = m_endCompensationSample - m_samplesProcessed; + + if (remainingCompensationDistance > 0 && maxOutSamples > remainingCompensationDistance) { + // If the remaining compensation distance less than output frame, + // the ffmpeg resampler bufferises the rest of frames that makes + // unexpected delays on large frames. + // The hack might cause some compensation bias on large frames, + // however it's not significant for our logic, in fact. + // TODO: probably, it will need some improvements + setSampleCompensation(0, 0); + maxOutSamples = swr_get_out_samples(m_resampler.get(), inputSamplesCount); + } + + return maxOutSamples; +} +void QFFmpegResampler::setSampleCompensation(qint32 delta, quint32 distance) +{ + const int res = swr_set_compensation(m_resampler.get(), delta, static_cast<int>(distance)); + if (res < 0) + qCWarning(qLcResampler) << "swr_set_compensation fail:" << res; + else { + m_sampleCompensationDelta = delta; + m_endCompensationSample = m_samplesProcessed + distance; + } +} + +qint32 QFFmpegResampler::activeSampleCompensationDelta() const +{ + return m_samplesProcessed < m_endCompensationSample ? m_sampleCompensationDelta : 0; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h b/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h index ddede2aab..530f40aa2 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegresampler_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGRESAMPLER_P_H #define QFFMPEGRESAMPLER_P_H @@ -52,31 +16,47 @@ #include "qaudiobuffer.h" #include "qffmpeg_p.h" +#include "private/qplatformaudioresampler_p.h" QT_BEGIN_NAMESPACE namespace QFFmpeg { +class Codec; +} -struct Codec; - -class Resampler +class QFFmpegResampler : public QPlatformAudioResampler { public: - Resampler(const Codec *codec, const QAudioFormat &outputFormat); - ~Resampler(); + QFFmpegResampler(const QAudioFormat &inputFormat, const QAudioFormat &outputFormat); + QFFmpegResampler(const QFFmpeg::Codec *codec, const QAudioFormat &outputFormat, + qint64 startTime = 0); + + ~QFFmpegResampler() override; + + QAudioBuffer resample(const char* data, size_t size) override; QAudioBuffer resample(const AVFrame *frame); + qint64 samplesProcessed() const { return m_samplesProcessed; } + void setSampleCompensation(qint32 delta, quint32 distance); + qint32 activeSampleCompensationDelta() const; private: + int adjustMaxOutSamples(int inputSamplesCount); + + QAudioBuffer resample(const uint8_t **inputData, int inputSamplesCount); + +private: + QAudioFormat m_inputFormat; QAudioFormat m_outputFormat; - SwrContext *resampler = nullptr; + qint64 m_startTime = 0; + QFFmpeg::SwrContextUPtr m_resampler; qint64 m_samplesProcessed = 0; + qint64 m_endCompensationSample = std::numeric_limits<qint64>::min(); + qint32 m_sampleCompensationDelta = 0; }; -} - QT_END_NAMESPACE #endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegscreencapture_dxgi.cpp b/src/plugins/multimedia/ffmpeg/qffmpegscreencapture_dxgi.cpp new file mode 100644 index 000000000..eb5bcfdf8 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegscreencapture_dxgi.cpp @@ -0,0 +1,464 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegscreencapture_dxgi_p.h" +#include "qffmpegsurfacecapturegrabber_p.h" +#include "qabstractvideobuffer.h" +#include <private/qmultimediautils_p.h> +#include <private/qwindowsmultimediautils_p.h> +#include <private/qvideoframe_p.h> +#include <qtgui/qscreen_platform.h> +#include "qvideoframe.h" + +#include <qloggingcategory.h> +#include <qwaitcondition.h> +#include <qmutex.h> + +#include "D3d11.h" +#include "dxgi1_2.h" + +#include <system_error> +#include <thread> +#include <chrono> + +#include <mutex> // std::scoped_lock + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLcScreenCaptureDxgi, "qt.multimedia.ffmpeg.screencapturedxgi") + +using namespace std::chrono; +using namespace QWindowsMultimediaUtils; +using namespace Qt::StringLiterals; + +namespace { + +// Convenience wrapper that combines an HRESULT +// status code with an optional textual description. +class ComStatus +{ +public: + ComStatus() = default; + ComStatus(HRESULT hr) : m_hr{ hr } { } + ComStatus(HRESULT hr, QAnyStringView msg) : m_hr{ hr }, m_msg{ msg.toString() } { } + + ComStatus(const ComStatus &) = default; + ComStatus(ComStatus &&) = default; + ComStatus &operator=(const ComStatus &) = default; + ComStatus &operator=(ComStatus &&) = default; + + explicit operator bool() const { return m_hr == S_OK; } + + HRESULT code() const { return m_hr; } + QString str() const + { + if (!m_msg) + return errorString(m_hr); + return *m_msg + " " + errorString(m_hr); + } + +private: + HRESULT m_hr = S_OK; + std::optional<QString> m_msg; +}; + +template <typename T> +using ComProduct = QMaybe<ComPtr<T>, ComStatus>; + +} + +class QD3D11TextureVideoBuffer : public QAbstractVideoBuffer +{ +public: + QD3D11TextureVideoBuffer(const ComPtr<ID3D11Device> &device, std::shared_ptr<QMutex> &mutex, + const ComPtr<ID3D11Texture2D> &texture, QSize size) + : m_device(device), m_texture(texture), m_ctxMutex(mutex), m_size(size) + {} + + ~QD3D11TextureVideoBuffer() { Q_ASSERT(m_mapMode == QtVideo::MapMode::NotMapped); } + + MapData map(QtVideo::MapMode mode) override + { + MapData mapData; + if (!m_ctx && mode == QtVideo::MapMode::ReadOnly) { + D3D11_TEXTURE2D_DESC texDesc = {}; + m_texture->GetDesc(&texDesc); + texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + texDesc.Usage = D3D11_USAGE_STAGING; + texDesc.MiscFlags = 0; + texDesc.BindFlags = 0; + + HRESULT hr = m_device->CreateTexture2D(&texDesc, nullptr, m_cpuTexture.GetAddressOf()); + if (FAILED(hr)) { + qCDebug(qLcScreenCaptureDxgi) << "Failed to create texture with CPU access" + << std::system_category().message(hr).c_str(); + qCDebug(qLcScreenCaptureDxgi) << m_device->GetDeviceRemovedReason(); + return {}; + } + + m_device->GetImmediateContext(m_ctx.GetAddressOf()); + m_ctxMutex->lock(); + m_ctx->CopyResource(m_cpuTexture.Get(), m_texture.Get()); + + D3D11_MAPPED_SUBRESOURCE resource = {}; + hr = m_ctx->Map(m_cpuTexture.Get(), 0, D3D11_MAP_READ, 0, &resource); + m_ctxMutex->unlock(); + if (FAILED(hr)) { + qCDebug(qLcScreenCaptureDxgi) << "Failed to map texture" << m_cpuTexture.Get() + << std::system_category().message(hr).c_str(); + return {}; + } + + m_mapMode = mode; + mapData.planeCount = 1; + mapData.bytesPerLine[0] = int(resource.RowPitch); + mapData.data[0] = reinterpret_cast<uchar*>(resource.pData); + mapData.dataSize[0] = m_size.height() * int(resource.RowPitch); + } + + return mapData; + } + + void unmap() override + { + if (m_mapMode == QtVideo::MapMode::NotMapped) + return; + if (m_ctx) { + m_ctxMutex->lock(); + m_ctx->Unmap(m_cpuTexture.Get(), 0); + m_ctxMutex->unlock(); + m_ctx.Reset(); + } + m_cpuTexture.Reset(); + m_mapMode = QtVideo::MapMode::NotMapped; + } + + QVideoFrameFormat format() const override { return {}; } + + QSize getSize() const + { + if (!m_texture) + return {}; + + D3D11_TEXTURE2D_DESC desc{}; + m_texture->GetDesc(&desc); + + return { static_cast<int>(desc.Width), static_cast<int>(desc.Height) }; + } + +private: + ComPtr<ID3D11Device> m_device; + ComPtr<ID3D11Texture2D> m_texture; + ComPtr<ID3D11Texture2D> m_cpuTexture; + ComPtr<ID3D11DeviceContext> m_ctx; + std::shared_ptr<QMutex> m_ctxMutex; + QSize m_size; + QtVideo::MapMode m_mapMode = QtVideo::MapMode::NotMapped; +}; + +namespace { +class DxgiDuplication +{ + struct DxgiScreen + { + ComPtr<IDXGIAdapter1> adapter; + ComPtr<IDXGIOutput> output; + }; + +public: + ~DxgiDuplication() + { + if (m_releaseFrame) + m_dup->ReleaseFrame(); + } + + ComStatus initialize(QScreen const *screen) + { + const QMaybe<DxgiScreen, ComStatus> dxgiScreen = findDxgiScreen(screen); + if (!dxgiScreen) + return dxgiScreen.error(); + + const ComPtr<IDXGIAdapter1> adapter = dxgiScreen->adapter; + + ComPtr<ID3D11Device> d3d11dev; + HRESULT hr = + D3D11CreateDevice(adapter.Get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr, 0, nullptr, 0, + D3D11_SDK_VERSION, d3d11dev.GetAddressOf(), nullptr, nullptr); + if (FAILED(hr)) + return { hr, "Failed to create ID3D11Device device"_L1 }; + + ComPtr<IDXGIOutput1> output; + hr = dxgiScreen->output.As(&output); + if (FAILED(hr)) + return { hr, "Failed to create IDXGIOutput1"_L1 }; + + ComPtr<IDXGIOutputDuplication> dup; + hr = output->DuplicateOutput(d3d11dev.Get(), dup.GetAddressOf()); + if (FAILED(hr)) + return { hr, "Failed to duplicate IDXGIOutput1"_L1 }; + + m_adapter = dxgiScreen->adapter; + m_output = output; + m_device = d3d11dev; + m_dup = dup; + return { S_OK }; + } + + bool valid() const { return m_dup != nullptr; } + + QSize getFrameSize() const + { + DXGI_OUTDUPL_DESC outputDesc = {}; + m_dup->GetDesc(&outputDesc); + + return { static_cast<int>(outputDesc.ModeDesc.Width), + static_cast<int>(outputDesc.ModeDesc.Height) }; + } + + QMaybe<std::unique_ptr<QD3D11TextureVideoBuffer>, ComStatus> getNextVideoFrame() + { + const ComProduct<ID3D11Texture2D> texture = getNextFrame(); + + if (!texture) + return texture.error(); + + return std::make_unique<QD3D11TextureVideoBuffer>(m_device, m_ctxMutex, *texture, + getFrameSize()); + } + +private: + ComProduct<ID3D11Texture2D> getNextFrame() + { + std::scoped_lock guard{ *m_ctxMutex }; + + if (m_releaseFrame) { + m_releaseFrame = false; + + HRESULT hr = m_dup->ReleaseFrame(); + + if (hr != S_OK) + return ComStatus{ hr, "Failed to release duplication frame."_L1 }; + } + + ComPtr<IDXGIResource> frame; + DXGI_OUTDUPL_FRAME_INFO info; + + HRESULT hr = m_dup->AcquireNextFrame(0, &info, frame.GetAddressOf()); + + if (hr != S_OK) + return { unexpect, hr, "Failed to grab the screen content"_L1 }; + + m_releaseFrame = true; + + ComPtr<ID3D11Texture2D> tex; + hr = frame.As(&tex); + if (hr != S_OK) + return { unexpect, hr, "Failed to obtain D3D11 texture"_L1 }; + + D3D11_TEXTURE2D_DESC texDesc = {}; + tex->GetDesc(&texDesc); + texDesc.MiscFlags = 0; + texDesc.BindFlags = 0; + + ComPtr<ID3D11Texture2D> texCopy; + hr = m_device->CreateTexture2D(&texDesc, nullptr, texCopy.GetAddressOf()); + if (hr != S_OK) + return { unexpect, hr, "Failed to create texture with CPU access"_L1 }; + + ComPtr<ID3D11DeviceContext> ctx; + m_device->GetImmediateContext(ctx.GetAddressOf()); + ctx->CopyResource(texCopy.Get(), tex.Get()); + + return texCopy; + } + + static QMaybe<DxgiScreen, ComStatus> findDxgiScreen(const QScreen *screen) + { + if (!screen) + return { unexpect, E_FAIL, "Cannot find nullptr screen"_L1 }; + + auto *winScreen = screen->nativeInterface<QNativeInterface::QWindowsScreen>(); + HMONITOR handle = winScreen ? winScreen->handle() : nullptr; + + ComPtr<IDXGIFactory1> factory; + HRESULT hr = CreateDXGIFactory1(IID_PPV_ARGS(&factory)); + if (FAILED(hr)) + return { unexpect, hr, "Failed to create IDXGIFactory"_L1 }; + + ComPtr<IDXGIAdapter1> adapter; + for (quint32 i = 0; factory->EnumAdapters1(i, adapter.ReleaseAndGetAddressOf()) == S_OK; i++) { + ComPtr<IDXGIOutput> output; + for (quint32 j = 0; adapter->EnumOutputs(j, output.ReleaseAndGetAddressOf()) == S_OK; ++j) { + DXGI_OUTPUT_DESC desc = {}; + output->GetDesc(&desc); + qCDebug(qLcScreenCaptureDxgi) << i << j << QString::fromWCharArray(desc.DeviceName); + auto match = handle ? handle == desc.Monitor + : QString::fromWCharArray(desc.DeviceName) == screen->name(); + if (match) + return DxgiScreen{ adapter, output }; + } + } + return { unexpect, DXGI_ERROR_NOT_FOUND, + "Could not find screen adapter "_L1 + screen->name() }; + } + + ComPtr<IDXGIAdapter1> m_adapter; + ComPtr<IDXGIOutput> m_output; + ComPtr<ID3D11Device> m_device; + ComPtr<IDXGIOutputDuplication> m_dup; + bool m_releaseFrame = false; + std::shared_ptr<QMutex> m_ctxMutex = std::make_shared<QMutex>(); +}; + +QSize getPhysicalSizePixels(const QScreen *screen) +{ + const auto *winScreen = screen->nativeInterface<QNativeInterface::QWindowsScreen>(); + if (!winScreen) + return {}; + + const HMONITOR handle = winScreen->handle(); + if (!handle) + return {}; + + MONITORINFO info{}; + info.cbSize = sizeof(info); + + if (!GetMonitorInfoW(handle, &info)) + return {}; + + return { info.rcMonitor.right - info.rcMonitor.left, + info.rcMonitor.bottom - info.rcMonitor.top }; +} + +QVideoFrameFormat getFrameFormat(QScreen* screen) +{ + const QSize screenSize = getPhysicalSizePixels(screen); + + QVideoFrameFormat format = { screenSize, QVideoFrameFormat::Format_BGRA8888 }; + format.setStreamFrameRate(static_cast<int>(screen->refreshRate())); + + return format; +} + +} // namespace + +class QFFmpegScreenCaptureDxgi::Grabber : public QFFmpegSurfaceCaptureGrabber +{ +public: + Grabber(QFFmpegScreenCaptureDxgi &screenCapture, QScreen *screen, + const QVideoFrameFormat &format) + : m_screen(screen) + , m_format(format) + { + setFrameRate(screen->refreshRate()); + addFrameCallback(screenCapture, &QFFmpegScreenCaptureDxgi::newVideoFrame); + connect(this, &Grabber::errorUpdated, &screenCapture, &QFFmpegScreenCaptureDxgi::updateError); + } + + ~Grabber() { + stop(); + } + + QVideoFrameFormat format() { + return m_format; + } + + QVideoFrame grabFrame() override + { + QVideoFrame frame; + if (!m_duplication.valid()) { + const ComStatus status = m_duplication.initialize(m_screen); + if (!status) { + if (status.code() == E_ACCESSDENIED) { + // May occur for some time after pushing Ctrl+Alt+Del. + updateError(QPlatformSurfaceCapture::NoError, status.str()); + qCWarning(qLcScreenCaptureDxgi) << status.str(); + } + return frame; + } + } + + auto maybeBuf = m_duplication.getNextVideoFrame(); + const ComStatus &status = maybeBuf.error(); + + if (status.code() == DXGI_ERROR_WAIT_TIMEOUT) { + // All is good, we just didn't get a new frame yet + updateError(QPlatformSurfaceCapture::NoError, status.str()); + } else if (status.code() == DXGI_ERROR_ACCESS_LOST) { + // Can happen for example when pushing Ctrl + Alt + Del + m_duplication = {}; + updateError(QPlatformSurfaceCapture::NoError, status.str()); + qCWarning(qLcScreenCaptureDxgi) << status.str(); + } else if (!status) { + updateError(QPlatformSurfaceCapture::CaptureFailed, status.str()); + qCWarning(qLcScreenCaptureDxgi) << status.str(); + } else if (maybeBuf) { + std::unique_ptr<QD3D11TextureVideoBuffer> buffer = std::move(*maybeBuf); + + const QSize bufSize = buffer->getSize(); + if (bufSize != m_format.frameSize()) + m_format.setFrameSize(bufSize); + + frame = QVideoFramePrivate::createFrame(std::move(buffer), format()); + } + + return frame; + } + + protected: + void initializeGrabbingContext() override + { + m_duplication = DxgiDuplication(); + const ComStatus status = m_duplication.initialize(m_screen); + if (!status) { + updateError(CaptureFailed, status.str()); + return; + } + + QFFmpegSurfaceCaptureGrabber::initializeGrabbingContext(); + } + +private: + const QScreen *m_screen = nullptr; + QVideoFrameFormat m_format; + DxgiDuplication m_duplication; +}; + +QFFmpegScreenCaptureDxgi::QFFmpegScreenCaptureDxgi() : QPlatformSurfaceCapture(ScreenSource{}) { } + +QFFmpegScreenCaptureDxgi::~QFFmpegScreenCaptureDxgi() = default; + +QVideoFrameFormat QFFmpegScreenCaptureDxgi::frameFormat() const +{ + if (m_grabber) + return m_grabber->format(); + return {}; +} + +bool QFFmpegScreenCaptureDxgi::setActiveInternal(bool active) +{ + if (static_cast<bool>(m_grabber) == active) + return true; + + if (m_grabber) { + m_grabber.reset(); + } else { + auto screen = source<ScreenSource>(); + + if (!checkScreenWithError(screen)) + return false; + + const QVideoFrameFormat format = getFrameFormat(screen); + if (!format.isValid()) { + updateError(NotFound, QLatin1String("Unable to determine screen size or format")); + return false; + } + + m_grabber.reset(new Grabber(*this, screen, format)); + m_grabber->start(); + } + + return true; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegscreencapture_dxgi_p.h b/src/plugins/multimedia/ffmpeg/qffmpegscreencapture_dxgi_p.h new file mode 100644 index 000000000..8f866b135 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegscreencapture_dxgi_p.h @@ -0,0 +1,44 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGSCREENCAPTURE_WINDOWS_H +#define QFFMPEGSCREENCAPTURE_WINDOWS_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qvideoframeformat.h" +#include <private/qcomptr_p.h> +#include <private/qplatformsurfacecapture_p.h> +#include <memory> + +QT_BEGIN_NAMESPACE + +class QFFmpegScreenCaptureDxgi : public QPlatformSurfaceCapture +{ +public: + explicit QFFmpegScreenCaptureDxgi(); + + ~QFFmpegScreenCaptureDxgi() override; + + QVideoFrameFormat frameFormat() const override; + +private: + bool setActiveInternal(bool active) override; + +private: + class Grabber; + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // QFFMPEGSCREENCAPTURE_WINDOWS_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegsurfacecapturegrabber.cpp b/src/plugins/multimedia/ffmpeg/qffmpegsurfacecapturegrabber.cpp new file mode 100644 index 000000000..38b48938f --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegsurfacecapturegrabber.cpp @@ -0,0 +1,202 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegsurfacecapturegrabber_p.h" + +#include <qelapsedtimer.h> +#include <qloggingcategory.h> +#include <qthread.h> +#include <qtimer.h> + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcScreenCaptureGrabber, "qt.multimedia.ffmpeg.surfacecapturegrabber"); + +namespace { + +class GrabbingProfiler +{ +public: + auto measure() + { + m_elapsedTimer.start(); + return qScopeGuard([&]() { + const auto nsecsElapsed = m_elapsedTimer.nsecsElapsed(); + ++m_number; + m_wholeTime += nsecsElapsed; + +#ifdef DUMP_SCREEN_CAPTURE_PROFILING + qDebug() << "screen grabbing time:" << nsecsElapsed << "avg:" << avgTime() + << "number:" << m_number; +#endif + }); + } + + qreal avgTime() const + { + return m_number ? m_wholeTime / (m_number * 1000000.) : 0.; + } + + qint64 number() const + { + return m_number; + } + +private: + QElapsedTimer m_elapsedTimer; + qint64 m_wholeTime = 0; + qint64 m_number = 0; +}; + +} // namespace + +struct QFFmpegSurfaceCaptureGrabber::GrabbingContext +{ + GrabbingProfiler profiler; + QTimer timer; + QElapsedTimer elapsedTimer; + qint64 lastFrameTime = 0; +}; + +class QFFmpegSurfaceCaptureGrabber::GrabbingThread : public QThread +{ +public: + GrabbingThread(QFFmpegSurfaceCaptureGrabber& grabber) + : m_grabber(grabber) + {} + +protected: + void run() override + { + m_grabber.initializeGrabbingContext(); + + if (!m_grabber.isGrabbingContextInitialized()) + return; + + exec(); + m_grabber.finalizeGrabbingContext(); + } + +private: + QFFmpegSurfaceCaptureGrabber& m_grabber; +}; + +QFFmpegSurfaceCaptureGrabber::QFFmpegSurfaceCaptureGrabber(ThreadPolicy threadPolicy) +{ + setFrameRate(DefaultScreenCaptureFrameRate); + + if (threadPolicy == CreateGrabbingThread) + m_thread = std::make_unique<GrabbingThread>(*this); +} + +void QFFmpegSurfaceCaptureGrabber::start() +{ + if (m_thread) + m_thread->start(); + else if (!isGrabbingContextInitialized()) + initializeGrabbingContext(); +} + +QFFmpegSurfaceCaptureGrabber::~QFFmpegSurfaceCaptureGrabber() = default; + +void QFFmpegSurfaceCaptureGrabber::setFrameRate(qreal rate) +{ + rate = qBound(MinScreenCaptureFrameRate, rate, MaxScreenCaptureFrameRate); + if (std::exchange(m_rate, rate) != rate) { + qCDebug(qLcScreenCaptureGrabber) << "Screen capture rate has been changed:" << m_rate; + + updateTimerInterval(); + } +} + +qreal QFFmpegSurfaceCaptureGrabber::frameRate() const +{ + return m_rate; +} + +void QFFmpegSurfaceCaptureGrabber::stop() +{ + if (m_thread) + { + m_thread->quit(); + m_thread->wait(); + } + else if (isGrabbingContextInitialized()) + { + finalizeGrabbingContext(); + } +} + +void QFFmpegSurfaceCaptureGrabber::updateError(QPlatformSurfaceCapture::Error error, + const QString &description) +{ + const auto prevError = std::exchange(m_prevError, error); + + if (error != QPlatformSurfaceCapture::NoError + || prevError != QPlatformSurfaceCapture::NoError) { + emit errorUpdated(error, description); + } + + updateTimerInterval(); +} + +void QFFmpegSurfaceCaptureGrabber::updateTimerInterval() +{ + const qreal rate = m_prevError && *m_prevError != QPlatformSurfaceCapture::NoError + ? MinScreenCaptureFrameRate + : m_rate; + const int interval = static_cast<int>(1000 / rate); + if (m_context && m_context->timer.interval() != interval) + m_context->timer.setInterval(interval); +} + +void QFFmpegSurfaceCaptureGrabber::initializeGrabbingContext() +{ + Q_ASSERT(!isGrabbingContextInitialized()); + qCDebug(qLcScreenCaptureGrabber) << "screen capture started"; + + m_context = std::make_unique<GrabbingContext>(); + m_context->timer.setTimerType(Qt::PreciseTimer); + updateTimerInterval(); + + m_context->elapsedTimer.start(); + + auto doGrab = [this]() { + auto measure = m_context->profiler.measure(); + + auto frame = grabFrame(); + + if (frame.isValid()) { + frame.setStartTime(m_context->lastFrameTime); + frame.setEndTime(m_context->elapsedTimer.nsecsElapsed() / 1000); + m_context->lastFrameTime = frame.endTime(); + + updateError(QPlatformSurfaceCapture::NoError); + + emit frameGrabbed(frame); + } + }; + + doGrab(); + + m_context->timer.callOnTimeout(&m_context->timer, doGrab); + m_context->timer.start(); +} + +void QFFmpegSurfaceCaptureGrabber::finalizeGrabbingContext() +{ + Q_ASSERT(isGrabbingContextInitialized()); + qCDebug(qLcScreenCaptureGrabber) + << "end screen capture thread; avg grabbing time:" << m_context->profiler.avgTime() + << "ms, grabbings number:" << m_context->profiler.number(); + m_context.reset(); +} + +bool QFFmpegSurfaceCaptureGrabber::isGrabbingContextInitialized() const +{ + return m_context != nullptr; +} + +QT_END_NAMESPACE + +#include "moc_qffmpegsurfacecapturegrabber_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegsurfacecapturegrabber_p.h b/src/plugins/multimedia/ffmpeg/qffmpegsurfacecapturegrabber_p.h new file mode 100644 index 000000000..9a617bd7a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegsurfacecapturegrabber_p.h @@ -0,0 +1,92 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGSURFACECAPTUREGRABBER_P_H +#define QFFMPEGSURFACECAPTUREGRABBER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qvideoframe.h" +#include "private/qplatformsurfacecapture_p.h" + +#include <memory> +#include <optional> + +QT_BEGIN_NAMESPACE + +class QThread; + +static constexpr qreal DefaultScreenCaptureFrameRate = 60.; + +// Mac screens often support 120 frames per sec; it looks, this is not +// needed for the capturing now since it just affects CPI without valuable +// advantages. In the future, the frame rate should be customized by +// user's API. +static constexpr qreal MaxScreenCaptureFrameRate = 60.; +static constexpr qreal MinScreenCaptureFrameRate = 1.; + +class QFFmpegSurfaceCaptureGrabber : public QObject +{ + Q_OBJECT +public: + enum ThreadPolicy { + UseCurrentThread, + CreateGrabbingThread, + }; + + QFFmpegSurfaceCaptureGrabber(ThreadPolicy threadPolicy = CreateGrabbingThread); + + ~QFFmpegSurfaceCaptureGrabber() override; + + void start(); + void stop(); + + template<typename Object, typename Method> + void addFrameCallback(Object &object, Method method) + { + connect(this, &QFFmpegSurfaceCaptureGrabber::frameGrabbed, + &object, method, Qt::DirectConnection); + } + +signals: + void frameGrabbed(const QVideoFrame&); + void errorUpdated(QPlatformSurfaceCapture::Error error, const QString &description); + +protected: + void updateError(QPlatformSurfaceCapture::Error error, const QString &description = {}); + + virtual QVideoFrame grabFrame() = 0; + + void setFrameRate(qreal rate); + + qreal frameRate() const; + + void updateTimerInterval(); + + virtual void initializeGrabbingContext(); + virtual void finalizeGrabbingContext(); + + bool isGrabbingContextInitialized() const; + +private: + struct GrabbingContext; + class GrabbingThread; + + std::unique_ptr<GrabbingContext> m_context; + qreal m_rate = 0; + std::optional<QPlatformSurfaceCapture::Error> m_prevError; + std::unique_ptr<QThread> m_thread; +}; + +QT_END_NAMESPACE + +#endif // QFFMPEGSURFACECAPTUREGRABBER_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegthread.cpp b/src/plugins/multimedia/ffmpeg/qffmpegthread.cpp index c9584c550..fb14ced54 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegthread.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegthread.cpp @@ -1,93 +1,53 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegthread_p.h" -#include <qloggingcategory.h> QT_BEGIN_NAMESPACE using namespace QFFmpeg; -void Thread::kill() +void ConsumerThread::stopAndDelete() { { - QMutexLocker locker(&mutex); - exit.storeRelease(true); - killHelper(); + QMutexLocker locker(&m_loopDataMutex); + m_exit = true; } - wake(); + dataReady(); wait(); delete this; } -void Thread::maybePause() +void ConsumerThread::dataReady() { - while (timeOut > 0 || shouldWait()) { - if (exit.loadAcquire()) - break; - - QElapsedTimer timer; - timer.start(); - if (condition.wait(&mutex, QDeadlineTimer(timeOut, Qt::PreciseTimer))) { - if (timeOut >= 0) { - timeOut -= timer.elapsed(); - if (timeOut < 0) - timeOut = -1; - } - } else { - timeOut = -1; - } - } + m_condition.wakeAll(); } -void Thread::run() +void ConsumerThread::run() { init(); - QMutexLocker locker(&mutex); - while (1) { - maybePause(); - if (exit.loadAcquire()) - break; - loop(); + + while (true) { + + { + QMutexLocker locker(&m_loopDataMutex); + while (!hasData() && !m_exit) + m_condition.wait(&m_loopDataMutex); + + if (m_exit) + break; + } + + processOne(); } + cleanup(); } +QMutexLocker<QMutex> ConsumerThread::lockLoopData() const +{ + return QMutexLocker(&m_loopDataMutex); +} + QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegthread_p.h b/src/plugins/multimedia/ffmpeg/qffmpegthread_p.h index bd5ecbba5..a7c5b0927 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegthread_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegthread_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGTHREAD_P_H #define QFFMPEGTHREAD_P_H @@ -63,38 +27,66 @@ class QAudioSink; namespace QFFmpeg { -class Thread : public QThread +/*! + FFmpeg thread that is used to implement a consumer pattern. + + This thread processes work items until no more data is available. + When no more data is available, it sleeps until it is notified about + more available data. + */ +class ConsumerThread : public QThread { public: - mutable QMutex mutex; - qint64 timeOut = -1; -private: - QWaitCondition condition; + /*! + Stops the thread and deletes this object + */ + void stopAndDelete(); protected: - QAtomicInteger<bool> exit = false; - -public: - // public API is thread-safe - void kill(); - virtual void killHelper() {} - - void wake() { - condition.wakeAll(); - } - -protected: - virtual void init() {} - virtual void cleanup() {} - // loop() should never block, all blocking has to happen in shouldWait() - virtual void loop() = 0; - virtual bool shouldWait() const { return false; } + /*! + Called on this thread when thread starts + */ + virtual void init() = 0; + + /*! + Called on this thread before thread exits + */ + virtual void cleanup() = 0; + + /*! + Process one work item. Called repeatedly until hasData() returns + false, in which case the thread sleeps until the next dataReady() + notification. + + Note: processOne() should never block. + */ + virtual void processOne() = 0; + + /*! + Wake thread from sleep and process data until + hasData() returns false. The method is supposed to be invoked + right after the scope of QMutexLocker that lockLoopData returns. + */ + void dataReady(); + + /*! + Must return true when data is available for processing + */ + virtual bool hasData() const = 0; + + /*! + Locks the loop data mutex. It must be used to protect loop data + like a queue of video frames. + */ + QMutexLocker<QMutex> lockLoopData() const; private: - void maybePause(); + void run() final; - void run() override; + mutable QMutex m_loopDataMutex; + QWaitCondition m_condition; + bool m_exit = false; }; } diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp index 144281896..3bc5a8c8a 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer.cpp @@ -1,45 +1,11 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegvideobuffer_p.h" #include "private/qvideotexturehelper_p.h" +#include "private/qmultimediautils_p.h" #include "qffmpeghwaccel_p.h" +#include "qloggingcategory.h" extern "C" { #include <libavutil/pixdesc.h> @@ -49,163 +15,113 @@ extern "C" { QT_BEGIN_NAMESPACE -QFFmpegVideoBuffer::QFFmpegVideoBuffer(AVFrame *frame) - : QAbstractVideoBuffer(QVideoFrame::NoHandle) - , frame(frame) +static bool isFrameFlipped(const AVFrame& frame) { + for (int i = 0; i < AV_NUM_DATA_POINTERS && frame.data[i]; ++i) { + if (frame.linesize[i] < 0) + return true; + } + + return false; +} + +Q_STATIC_LOGGING_CATEGORY(qLcFFmpegVideoBuffer, "qt.multimedia.ffmpeg.videobuffer"); + +QFFmpegVideoBuffer::QFFmpegVideoBuffer(AVFrameUPtr frame, AVRational pixelAspectRatio) + : QHwVideoBuffer(QVideoFrame::NoHandle), + m_frame(frame.get()), + m_size(qCalculateFrameSize({ frame->width, frame->height }, + { pixelAspectRatio.num, pixelAspectRatio.den })) { if (frame->hw_frames_ctx) { - hwFrame = frame; - m_pixelFormat = toQtPixelFormat(QFFmpeg::HWAccel::format(frame)); + m_hwFrame = std::move(frame); + m_pixelFormat = toQtPixelFormat(QFFmpeg::HWAccel::format(m_hwFrame.get())); return; } - swFrame = frame; - m_pixelFormat = toQtPixelFormat(AVPixelFormat(swFrame->format)); + m_swFrame = std::move(frame); + m_pixelFormat = toQtPixelFormat(AVPixelFormat(m_swFrame->format)); convertSWFrame(); } -QFFmpegVideoBuffer::~QFFmpegVideoBuffer() -{ - delete textures; - if (swFrame) - av_frame_free(&swFrame); - if (hwFrame) - av_frame_free(&hwFrame); -} +QFFmpegVideoBuffer::~QFFmpegVideoBuffer() = default; void QFFmpegVideoBuffer::convertSWFrame() { - Q_ASSERT(swFrame); - bool needsConversion = false; - auto pixelFormat = toQtPixelFormat(AVPixelFormat(swFrame->format), &needsConversion); -// qDebug() << "SW frame format:" << pixelFormat << swFrame->format << needsConversion; + Q_ASSERT(m_swFrame); - if (pixelFormat != m_pixelFormat) { - AVPixelFormat newFormat = toAVPixelFormat(m_pixelFormat); + const auto actualAVPixelFormat = AVPixelFormat(m_swFrame->format); + const auto targetAVPixelFormat = toAVPixelFormat(m_pixelFormat); + + if (actualAVPixelFormat != targetAVPixelFormat || isFrameFlipped(*m_swFrame) + || m_size != QSize(m_swFrame->width, m_swFrame->height)) { + Q_ASSERT(toQtPixelFormat(targetAVPixelFormat) == m_pixelFormat); // convert the format into something we can handle - SwsContext *c = sws_getContext(swFrame->width, swFrame->height, AVPixelFormat(swFrame->format), - swFrame->width, swFrame->height, newFormat, + SwsContext *c = sws_getContext(m_swFrame->width, m_swFrame->height, actualAVPixelFormat, + m_size.width(), m_size.height(), targetAVPixelFormat, SWS_BICUBIC, nullptr, nullptr, nullptr); - AVFrame *newFrame = av_frame_alloc(); - newFrame->width = swFrame->width; - newFrame->height = swFrame->height; - newFrame->format = newFormat; - av_frame_get_buffer(newFrame, 0); + auto newFrame = QFFmpeg::makeAVFrame(); + newFrame->width = m_size.width(); + newFrame->height = m_size.height(); + newFrame->format = targetAVPixelFormat; + av_frame_get_buffer(newFrame.get(), 0); - sws_scale(c, swFrame->data, swFrame->linesize, 0, swFrame->height, newFrame->data, newFrame->linesize); - av_frame_free(&swFrame); - swFrame = newFrame; + sws_scale(c, m_swFrame->data, m_swFrame->linesize, 0, m_swFrame->height, newFrame->data, newFrame->linesize); + if (m_frame == m_swFrame.get()) + m_frame = newFrame.get(); + m_swFrame = std::move(newFrame); sws_freeContext(c); } } void QFFmpegVideoBuffer::setTextureConverter(const QFFmpeg::TextureConverter &converter) { - textureConverter = converter; - textureConverter.init(hwFrame); + m_textureConverter = converter; + m_textureConverter.init(m_hwFrame.get()); m_type = converter.isNull() ? QVideoFrame::NoHandle : QVideoFrame::RhiTextureHandle; } QVideoFrameFormat::ColorSpace QFFmpegVideoBuffer::colorSpace() const { - switch (frame->colorspace) { - default: - case AVCOL_SPC_UNSPECIFIED: - case AVCOL_SPC_RESERVED: - case AVCOL_SPC_FCC: - case AVCOL_SPC_SMPTE240M: - case AVCOL_SPC_YCGCO: - case AVCOL_SPC_SMPTE2085: - case AVCOL_SPC_CHROMA_DERIVED_NCL: - case AVCOL_SPC_CHROMA_DERIVED_CL: - case AVCOL_SPC_ICTCP: // BT.2100 ICtCp - return QVideoFrameFormat::ColorSpace_Undefined; - case AVCOL_SPC_RGB: - return QVideoFrameFormat::ColorSpace_AdobeRgb; - case AVCOL_SPC_BT709: - return QVideoFrameFormat::ColorSpace_BT709; - case AVCOL_SPC_BT470BG: // BT601 - case AVCOL_SPC_SMPTE170M: // Also BT601 - return QVideoFrameFormat::ColorSpace_BT601; - case AVCOL_SPC_BT2020_NCL: // Non constant luminence - case AVCOL_SPC_BT2020_CL: // Constant luminence - return QVideoFrameFormat::ColorSpace_BT2020; - } + return QFFmpeg::fromAvColorSpace(m_frame->colorspace); } QVideoFrameFormat::ColorTransfer QFFmpegVideoBuffer::colorTransfer() const { - switch (frame->color_trc) { - case AVCOL_TRC_BT709: - // The following three cases have transfer characteristics identical to BT709 - case AVCOL_TRC_BT1361_ECG: - case AVCOL_TRC_BT2020_10: - case AVCOL_TRC_BT2020_12: - case AVCOL_TRC_SMPTE240M: // almost identical to bt709 - return QVideoFrameFormat::ColorTransfer_BT709; - case AVCOL_TRC_GAMMA22: - case AVCOL_TRC_SMPTE428 : // No idea, let's hope for the best... - case AVCOL_TRC_IEC61966_2_1: // sRGB, close enough to 2.2... - case AVCOL_TRC_IEC61966_2_4: // not quite, but probably close enough - return QVideoFrameFormat::ColorTransfer_Gamma22; - case AVCOL_TRC_GAMMA28: - return QVideoFrameFormat::ColorTransfer_Gamma28; - case AVCOL_TRC_SMPTE170M: - return QVideoFrameFormat::ColorTransfer_BT601; - case AVCOL_TRC_LINEAR: - return QVideoFrameFormat::ColorTransfer_Linear; - case AVCOL_TRC_SMPTE2084: - return QVideoFrameFormat::ColorTransfer_ST2084; - case AVCOL_TRC_ARIB_STD_B67: - return QVideoFrameFormat::ColorTransfer_STD_B67; - default: - break; - } - return QVideoFrameFormat::ColorTransfer_Unknown; + return QFFmpeg::fromAvColorTransfer(m_frame->color_trc); } QVideoFrameFormat::ColorRange QFFmpegVideoBuffer::colorRange() const { - switch (frame->color_range) { - case AVCOL_RANGE_MPEG: - return QVideoFrameFormat::ColorRange_Video; - case AVCOL_RANGE_JPEG: - return QVideoFrameFormat::ColorRange_Full; - default: - return QVideoFrameFormat::ColorRange_Unknown; - } + return QFFmpeg::fromAvColorRange(m_frame->color_range); } float QFFmpegVideoBuffer::maxNits() { float maxNits = -1; - for (int i = 0; i <frame->nb_side_data; ++i) { - AVFrameSideData *sd = frame->side_data[i]; + for (int i = 0; i < m_frame->nb_side_data; ++i) { + AVFrameSideData *sd = m_frame->side_data[i]; // TODO: Longer term we might want to also support HDR10+ dynamic metadata if (sd->type == AV_FRAME_DATA_MASTERING_DISPLAY_METADATA) { auto *data = reinterpret_cast<AVMasteringDisplayMetadata *>(sd->data); - maxNits = float(data->max_luminance.num)/float(data->max_luminance.den)*10000.; + auto maybeLum = QFFmpeg::mul(10'000., data->max_luminance); + if (maybeLum) + maxNits = float(maybeLum.value()); } } return maxNits; } -QVideoFrame::MapMode QFFmpegVideoBuffer::mapMode() const +QAbstractVideoBuffer::MapData QFFmpegVideoBuffer::map(QtVideo::MapMode mode) { - return m_mode; -} - -QAbstractVideoBuffer::MapData QFFmpegVideoBuffer::map(QVideoFrame::MapMode mode) -{ - if (!swFrame) { - Q_ASSERT(hwFrame && hwFrame->hw_frames_ctx); - swFrame = av_frame_alloc(); + if (!m_swFrame) { + Q_ASSERT(m_hwFrame && m_hwFrame->hw_frames_ctx); + m_swFrame = QFFmpeg::makeAVFrame(); /* retrieve data from GPU to CPU */ - int ret = av_hwframe_transfer_data(swFrame, hwFrame, 0); + int ret = av_hwframe_transfer_data(m_swFrame.get(), m_hwFrame.get(), 0); if (ret < 0) { - qWarning() << "Error transferring the data to system memory\n"; + qWarning() << "Error transferring the data to system memory:" << ret; return {}; } convertSWFrame(); @@ -213,42 +129,62 @@ QAbstractVideoBuffer::MapData QFFmpegVideoBuffer::map(QVideoFrame::MapMode mode) m_mode = mode; -// qDebug() << "MAP:"; MapData mapData; auto *desc = QVideoTextureHelper::textureDescription(pixelFormat()); - mapData.nPlanes = desc->nplanes; - for (int i = 0; i < mapData.nPlanes; ++i) { - mapData.data[i] = swFrame->data[i]; - mapData.bytesPerLine[i] = swFrame->linesize[i]; - mapData.size[i] = mapData.bytesPerLine[i]*desc->heightForPlane(swFrame->height, i); -// qDebug() << " " << i << mapData.data[i] << mapData.size[i]; + mapData.planeCount = desc->nplanes; + for (int i = 0; i < mapData.planeCount; ++i) { + Q_ASSERT(m_swFrame->linesize[i] >= 0); + + mapData.data[i] = m_swFrame->data[i]; + mapData.bytesPerLine[i] = m_swFrame->linesize[i]; + mapData.dataSize[i] = mapData.bytesPerLine[i]*desc->heightForPlane(m_swFrame->height, i); + } + + if ((mode & QtVideo::MapMode::WriteOnly) != QtVideo::MapMode::NotMapped && m_hwFrame) { + m_type = QVideoFrame::NoHandle; + m_hwFrame.reset(); + if (m_textures) { + qCDebug(qLcFFmpegVideoBuffer) + << "Mapping of FFmpeg video buffer with write mode when " + "textures have been created. Visual artifacts might " + "happen if the frame is still in the rendering pipeline"; + m_textures.reset(); + } } + return mapData; } void QFFmpegVideoBuffer::unmap() { - // nothing to do here for SW buffers + // nothing to do here for SW buffers. + // Set NotMapped mode to ensure map/unmap/mapMode consisteny. + m_mode = QtVideo::MapMode::NotMapped; } -void QFFmpegVideoBuffer::mapTextures() +std::unique_ptr<QVideoFrameTextures> QFFmpegVideoBuffer::mapTextures(QRhi *) { - if (textures || !hwFrame) - return; -// qDebug() << ">>>>> mapTextures"; - textures = textureConverter.getTextures(hwFrame); - if (!textures) - qWarning() << " failed to get textures for frame" << textureConverter.isNull(); -} + if (m_textures) + return {}; + if (!m_hwFrame) + return {}; + if (m_textureConverter.isNull()) { + m_textures = nullptr; + return {}; + } -quint64 QFFmpegVideoBuffer::textureHandle(int plane) const -{ - return textures ? textures->textureHandle(plane) : 0; + m_textures.reset(m_textureConverter.getTextures(m_hwFrame.get())); + if (!m_textures) { + static thread_local int lastFormat = 0; + if (std::exchange(lastFormat, m_hwFrame->format) != m_hwFrame->format) // prevent logging spam + qWarning() << " failed to get textures for frame; format:" << m_hwFrame->format; + } + return {}; } -std::unique_ptr<QRhiTexture> QFFmpegVideoBuffer::texture(int plane) const +quint64 QFFmpegVideoBuffer::textureHandle(QRhi *rhi, int plane) const { - return textures ? textures->texture(plane) : std::unique_ptr<QRhiTexture>(); + return m_textures ? m_textures->textureHandle(rhi, plane) : 0; } QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::pixelFormat() const @@ -258,7 +194,7 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::pixelFormat() const QSize QFFmpegVideoBuffer::size() const { - return QSize(frame->width, frame->height); + return m_size; } QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat avPixelFormat, bool *needsConversion) @@ -269,6 +205,9 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat switch (avPixelFormat) { default: break; + case AV_PIX_FMT_NONE: + Q_ASSERT(!"Invalid avPixelFormat!"); + return QVideoFrameFormat::Format_Invalid; case AV_PIX_FMT_ARGB: return QVideoFrameFormat::Format_ARGB8888; case AV_PIX_FMT_0RGB: @@ -309,6 +248,8 @@ QVideoFrameFormat::PixelFormat QFFmpegVideoBuffer::toQtPixelFormat(AVPixelFormat return QVideoFrameFormat::Format_P010; case AV_PIX_FMT_P016: return QVideoFrameFormat::Format_P016; + case AV_PIX_FMT_MEDIACODEC: + return QVideoFrameFormat::Format_SamplerExternalOES; } if (needsConversion) @@ -341,13 +282,13 @@ AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat // We're using the data from the converted QImage here, which is in BGRA. return AV_PIX_FMT_BGRA; case QVideoFrameFormat::Format_ARGB8888: - case QVideoFrameFormat::Format_ARGB8888_Premultiplied: return AV_PIX_FMT_ARGB; + case QVideoFrameFormat::Format_ARGB8888_Premultiplied: case QVideoFrameFormat::Format_XRGB8888: return AV_PIX_FMT_0RGB; case QVideoFrameFormat::Format_BGRA8888: - case QVideoFrameFormat::Format_BGRA8888_Premultiplied: return AV_PIX_FMT_BGRA; + case QVideoFrameFormat::Format_BGRA8888_Premultiplied: case QVideoFrameFormat::Format_BGRX8888: return AV_PIX_FMT_BGR0; case QVideoFrameFormat::Format_ABGR8888: @@ -356,6 +297,8 @@ AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat return AV_PIX_FMT_0BGR; case QVideoFrameFormat::Format_RGBA8888: return AV_PIX_FMT_RGBA; + // to be added in 6.8: + // case QVideoFrameFormat::Format_RGBA8888_Premultiplied: case QVideoFrameFormat::Format_RGBX8888: return AV_PIX_FMT_RGB0; @@ -382,6 +325,9 @@ AVPixelFormat QFFmpegVideoBuffer::toAVPixelFormat(QVideoFrameFormat::PixelFormat return AV_PIX_FMT_P010; case QVideoFrameFormat::Format_P016: return AV_PIX_FMT_P016; + + case QVideoFrameFormat::Format_SamplerExternalOES: + return AV_PIX_FMT_MEDIACODEC; } } diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h index 9f0d0cae6..c61c3f5ff 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegvideobuffer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGVIDEOBUFFER_P_H #define QFFMPEGVIDEOBUFFER_P_H @@ -51,9 +15,7 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <private/qabstractvideobuffer_p.h> -#include <qvideoframe.h> +#include <private/qhwvideobuffer_p.h> #include <QtCore/qvariant.h> #include "qffmpeg_p.h" @@ -61,20 +23,19 @@ QT_BEGIN_NAMESPACE -class QFFmpegVideoBuffer : public QAbstractVideoBuffer +class QFFmpegVideoBuffer : public QHwVideoBuffer { public: + using AVFrameUPtr = QFFmpeg::AVFrameUPtr; - QFFmpegVideoBuffer(AVFrame *frame); - ~QFFmpegVideoBuffer(); + QFFmpegVideoBuffer(AVFrameUPtr frame, AVRational pixelAspectRatio = { 1, 1 }); + ~QFFmpegVideoBuffer() override; - QVideoFrame::MapMode mapMode() const override; - MapData map(QVideoFrame::MapMode mode) override; + MapData map(QtVideo::MapMode mode) override; void unmap() override; - virtual void mapTextures() override; - virtual quint64 textureHandle(int plane) const override; - std::unique_ptr<QRhiTexture> texture(int plane) const override; + virtual std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *) override; + virtual quint64 textureHandle(QRhi *rhi, int plane) const override; QVideoFrameFormat::PixelFormat pixelFormat() const; QSize size() const; @@ -84,7 +45,7 @@ public: void convertSWFrame(); - AVFrame *getHWFrame() const { return hwFrame; } + AVFrame *getHWFrame() const { return m_hwFrame.get(); } void setTextureConverter(const QFFmpeg::TextureConverter &converter); @@ -96,12 +57,13 @@ public: private: QVideoFrameFormat::PixelFormat m_pixelFormat; - AVFrame *frame = nullptr; - AVFrame *hwFrame = nullptr; - AVFrame *swFrame = nullptr; - QFFmpeg::TextureConverter textureConverter; - QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped; - QFFmpeg::TextureSet *textures = nullptr; + AVFrame *m_frame = nullptr; + AVFrameUPtr m_hwFrame; + AVFrameUPtr m_swFrame; + QSize m_size; + QFFmpeg::TextureConverter m_textureConverter; + QtVideo::MapMode m_mode = QtVideo::MapMode::NotMapped; + std::unique_ptr<QFFmpeg::TextureSet> m_textures; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp deleted file mode 100644 index 4b576c441..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder.cpp +++ /dev/null @@ -1,407 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qffmpegvideoframeencoder_p.h" -#include "qffmpegvideobuffer_p.h" -#include "qffmpegmediaformatinfo_p.h" -#include "qffmpegencoderoptions_p.h" -#include "private/qplatformmediarecorder_p.h" -#include "private/qmultimediautils_p.h" -#include <qloggingcategory.h> - -extern "C" { -#include <libavutil/pixdesc.h> -} - -/* Infrastructure for HW acceleration goes into this file. */ - -QT_BEGIN_NAMESPACE - -Q_LOGGING_CATEGORY(qLcVideoFrameEncoder, "qt.multimedia.ffmpeg.videoencoder") - -namespace QFFmpeg { - -VideoFrameEncoder::Data::~Data() -{ - if (converter) - sws_freeContext(converter); - avcodec_free_context(&codecContext); -} - -VideoFrameEncoder::VideoFrameEncoder(const QMediaEncoderSettings &encoderSettings, - const QSize &sourceSize, float frameRate, AVPixelFormat sourceFormat, AVPixelFormat swFormat) - : d(new Data) -{ - d->settings = encoderSettings; - d->frameRate = frameRate; - d->sourceSize = sourceSize; - - if (!d->settings.videoResolution().isValid()) - d->settings.setVideoResolution(d->sourceSize); - - d->sourceFormat = sourceFormat; - d->sourceSWFormat = swFormat; - - auto qVideoCodec = encoderSettings.videoCodec(); - auto codecID = QFFmpegMediaFormatInfo::codecIdForVideoCodec(qVideoCodec); - -#ifndef QT_DISABLE_HW_ENCODING - const auto *accels = HWAccel::preferredDeviceTypes(); - while (*accels != AV_HWDEVICE_TYPE_NONE) { - auto accel = HWAccel(*accels); - ++accels; - - auto matchesSizeConstraints = [&]() -> bool { - auto *constraints = av_hwdevice_get_hwframe_constraints(accel.hwDeviceContextAsBuffer(), nullptr); - if (!constraints) - return true; - // Check size constraints - bool result = (d->sourceSize.width() >= constraints->min_width && d->sourceSize.height() >= constraints->min_height && - d->sourceSize.width() <= constraints->max_width && d->sourceSize.height() <= constraints->max_height); - av_hwframe_constraints_free(&constraints); - return result; - }; - - if (!matchesSizeConstraints()) - continue; - - d->codec = accel.hardwareEncoderForCodecId(codecID); - if (!d->codec) - continue; - d->accel = accel; - break; - } -#endif - - if (d->accel.isNull()) { - d->codec = avcodec_find_encoder(codecID); - if (!d->codec) { - qWarning() << "Could not find encoder for codecId" << codecID; - d = {}; - return; - } - } - auto supportsFormat = [&](AVPixelFormat fmt) { - auto *f = d->codec->pix_fmts; - while (*f != -1) { - if (*f == fmt) - return true; - ++f; - } - return false; - }; - - d->targetFormat = d->sourceFormat; - - if (!supportsFormat(d->sourceFormat)) { - if (supportsFormat(swFormat)) - d->targetFormat = swFormat; - else - // Take first format the encoder supports. Might want to improve upon this - d->targetFormat = *d->codec->pix_fmts; - } - - auto desc = av_pix_fmt_desc_get(d->sourceFormat); - d->sourceFormatIsHWFormat = desc->flags & AV_PIX_FMT_FLAG_HWACCEL; - desc = av_pix_fmt_desc_get(d->targetFormat); - d->targetFormatIsHWFormat = desc->flags & AV_PIX_FMT_FLAG_HWACCEL; - - bool needToScale = d->sourceSize != d->settings.videoResolution(); - bool zeroCopy = d->sourceFormatIsHWFormat && d->sourceFormat == d->targetFormat && !needToScale; - - if (zeroCopy) - // no need to initialize any converters - return; - - if (d->sourceFormatIsHWFormat) { - // if source and target formats don't agree, but the source is a HW format or sizes do't agree, we need to download - if (d->sourceFormat != d->targetFormat || needToScale) - d->downloadFromHW = true; - } else { - d->sourceSWFormat = d->sourceFormat; - } - - if (d->targetFormatIsHWFormat) { - Q_ASSERT(!d->accel.isNull()); - // if source and target formats don't agree, but the target is a HW format, we need to upload - if (d->sourceFormat != d->targetFormat || needToScale) { - d->uploadToHW = true; - - // determine the format used by the encoder. - // We prefer YUV422 based formats such as NV12 or P010. Selection trues to find the best matching - // format for the encoder depending on the bit depth of the source format - auto desc = av_pix_fmt_desc_get(d->sourceSWFormat); - int sourceDepth = desc->comp[0].depth; - - d->targetSWFormat = AV_PIX_FMT_NONE; - - auto *constraints = av_hwdevice_get_hwframe_constraints(d->accel.hwDeviceContextAsBuffer(), nullptr); - auto *f = constraints->valid_sw_formats; - int score = INT_MIN; - while (*f != AV_PIX_FMT_NONE) { - auto calcScore = [&](AVPixelFormat fmt) -> int { - auto *desc = av_pix_fmt_desc_get(fmt); - int s = 0; - if (fmt == d->sourceSWFormat) - // prefer exact matches - s += 10; - if (desc->comp[0].depth == sourceDepth) - s += 100; - else if (desc->comp[0].depth < sourceDepth) - s -= 100; - if (desc->log2_chroma_h == 1) - s += 1; - if (desc->log2_chroma_w == 1) - s += 1; - if (desc->flags & AV_PIX_FMT_FLAG_BE) - s -= 10; - if (desc->flags & AV_PIX_FMT_FLAG_PAL) - // we don't want paletted formats - s -= 10000; - if (desc->flags & AV_PIX_FMT_FLAG_RGB) - // we don't want RGB formats - s -= 1000; - if (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) - // we really don't want HW accelerated formats here - s -= 1000000; - qCDebug(qLcVideoFrameEncoder) << "checking format" << fmt << Qt::hex << desc->flags << desc->comp[0].depth - << desc->log2_chroma_h << desc->log2_chroma_w << "score:" << s; - return s; - }; - - int s = calcScore(*f); - if (s > score) { - d->targetSWFormat = *f; - score = s; - } - ++f; - } - if (d->targetSWFormat == AV_PIX_FMT_NONE) // shouldn't happen - d->targetSWFormat = *constraints->valid_sw_formats; - - qCDebug(qLcVideoFrameEncoder) << "using format" << d->targetSWFormat << "as transfer format."; - - av_hwframe_constraints_free(&constraints); - // need to create a frames context to convert the input data - d->accel.createFramesContext(d->targetSWFormat, sourceSize); - } - } else { - d->targetSWFormat = d->targetFormat; - } - - if (d->sourceSWFormat != d->targetSWFormat || needToScale) { - auto resolution = d->settings.videoResolution(); - qCDebug(qLcVideoFrameEncoder) << "camera and encoder use different formats:" << d->sourceSWFormat << d->targetSWFormat; - d->converter = sws_getContext(d->sourceSize.width(), d->sourceSize.height(), d->sourceSWFormat, - resolution.width(), resolution.height(), d->targetSWFormat, - SWS_FAST_BILINEAR, nullptr, nullptr, nullptr); - } -} - -VideoFrameEncoder::~VideoFrameEncoder() -{ -} - -void QFFmpeg::VideoFrameEncoder::initWithFormatContext(AVFormatContext *formatContext) -{ - d->stream = avformat_new_stream(formatContext, nullptr); - d->stream->id = formatContext->nb_streams - 1; - //qCDebug(qLcVideoFrameEncoder) << "Video stream: index" << d->stream->id; - d->stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; - d->stream->codecpar->codec_id = d->codec->id; - - // Apples HEVC decoders don't like the hev1 tag ffmpeg uses by default, use hvc1 as the more commonly accepted tag - if (d->codec->id == AV_CODEC_ID_HEVC) - d->stream->codecpar->codec_tag = MKTAG('h','v','c','1'); - - // ### Fix hardcoded values - d->stream->codecpar->format = d->targetFormat; - d->stream->codecpar->width = d->settings.videoResolution().width(); - d->stream->codecpar->height = d->settings.videoResolution().height(); - d->stream->codecpar->sample_aspect_ratio = AVRational{1, 1}; - float requestedRate = d->frameRate; - d->stream->time_base = AVRational{ 1, (int)(requestedRate*1000) }; - - float delta = 1e10; - if (d->codec->supported_framerates) { - // codec only supports fixed frame rates - auto *f = d->codec->supported_framerates; - auto *best = f; - qCDebug(qLcVideoFrameEncoder) << "Finding fixed rate:"; - while (f->num != 0) { - float rate = float(f->num)/float(f->den); - float d = qAbs(rate - requestedRate); - qCDebug(qLcVideoFrameEncoder) << " " << f->num << f->den << d; - if (d < delta) { - best = f; - delta = d; - } - ++f; - } - qCDebug(qLcVideoFrameEncoder) << "Fixed frame rate required. Requested:" << requestedRate << "Using:" << best->num << "/" << best->den; - d->stream->time_base = { best->den, best->num }; - requestedRate = float(best->num)/float(best->den); - } - - Q_ASSERT(d->codec); - d->codecContext = avcodec_alloc_context3(d->codec); - if (!d->codecContext) { - qWarning() << "Could not allocate codec context"; - d = {}; - return; - } - - avcodec_parameters_to_context(d->codecContext, d->stream->codecpar); - d->codecContext->time_base = d->stream->time_base; - qCDebug(qLcVideoFrameEncoder) << "requesting time base" << d->codecContext->time_base.num << d->codecContext->time_base.den; - int num, den; - qt_real_to_fraction(requestedRate, &num, &den); - d->codecContext->framerate = { num, den }; - auto deviceContext = d->accel.hwDeviceContextAsBuffer(); - if (deviceContext) - d->codecContext->hw_device_ctx = av_buffer_ref(deviceContext); - auto framesContext = d->accel.hwFramesContextAsBuffer(); - if (framesContext) - d->codecContext->hw_frames_ctx = av_buffer_ref(framesContext); -} - -bool VideoFrameEncoder::open() -{ - AVDictionary *opts = nullptr; - applyVideoEncoderOptions(d->settings, d->codec->name, d->codecContext, &opts); - int res = avcodec_open2(d->codecContext, d->codec, &opts); - if (res < 0) { - avcodec_free_context(&d->codecContext); - qWarning() << "Couldn't open codec for writing" << err2str(res); - return false; - } - qCDebug(qLcVideoFrameEncoder) << "video codec opened" << res << "time base" << d->codecContext->time_base.num << d->codecContext->time_base.den; - d->stream->time_base = d->codecContext->time_base; - return true; -} - -qint64 VideoFrameEncoder::getPts(qint64 us) -{ - Q_ASSERT(d); - qint64 div = 1000000*d->stream->time_base.num; - return (us*d->stream->time_base.den + (div>>1))/div; -} - -int VideoFrameEncoder::sendFrame(AVFrame *frame) -{ - if (!frame) - return avcodec_send_frame(d->codecContext, frame); - auto pts = frame->pts; - - if (d->downloadFromHW) { - auto *f = av_frame_alloc(); - f->format = d->sourceSWFormat; - int err = av_hwframe_transfer_data(f, frame, 0); - if (err < 0) { - qCDebug(qLcVideoFrameEncoder) << "Error transferring frame data to surface." << err2str(err); - return err; - } - av_frame_free(&frame); - frame = f; - } - - if (d->converter) { - auto *f = av_frame_alloc(); - f->format = d->targetSWFormat; - f->width = d->settings.videoResolution().width(); - f->height = d->settings.videoResolution().height(); - av_frame_get_buffer(f, 0); - sws_scale(d->converter, frame->data, frame->linesize, 0, f->height, f->data, f->linesize); - av_frame_free(&frame); - frame = f; - } - - if (d->uploadToHW) { - auto *hwFramesContext = d->accel.hwFramesContextAsBuffer(); - Q_ASSERT(hwFramesContext); - auto *f = av_frame_alloc(); - if (!f) - return AVERROR(ENOMEM); - int err = av_hwframe_get_buffer(hwFramesContext, f, 0); - if (err < 0) { - qCDebug(qLcVideoFrameEncoder) << "Error getting HW buffer" << err2str(err); - return err; - } else { - qCDebug(qLcVideoFrameEncoder) << "got HW buffer"; - } - if (!f->hw_frames_ctx) { - qCDebug(qLcVideoFrameEncoder) << "no hw frames context"; - return AVERROR(ENOMEM); - } - err = av_hwframe_transfer_data(f, frame, 0); - if (err < 0) { - qCDebug(qLcVideoFrameEncoder) << "Error transferring frame data to surface." << err2str(err); - return err; - } - av_frame_free(&frame); - frame = f; - } - - qCDebug(qLcVideoFrameEncoder) << "sending frame" << pts; - frame->pts = pts; - int ret = avcodec_send_frame(d->codecContext, frame); - av_frame_free(&frame); - return ret; -} - -AVPacket *VideoFrameEncoder::retrievePacket() -{ - if (!d || !d->codecContext) - return nullptr; - AVPacket *packet = av_packet_alloc(); - int ret = avcodec_receive_packet(d->codecContext, packet); - if (ret < 0) { - av_packet_free(&packet); - if (ret != AVERROR(EOF) && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) - qCDebug(qLcVideoFrameEncoder) << "Error receiving packet" << ret << err2str(ret); - return nullptr; - } - qCDebug(qLcVideoFrameEncoder) << "got a packet" << packet->pts << timeStamp(packet->pts, d->stream->time_base); - packet->stream_index = d->stream->id; - return packet; -} - -} // namespace QFFmpeg - -QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder_p.h b/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder_p.h deleted file mode 100644 index 9aa707181..000000000 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideoframeencoder_p.h +++ /dev/null @@ -1,112 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef QFFMPEGVIDEOFRAMEENCODER_P_H -#define QFFMPEGVIDEOFRAMEENCODER_P_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include "qffmpeghwaccel_p.h" -#include "qvideoframeformat.h" -#include "private/qplatformmediarecorder_p.h" - -QT_BEGIN_NAMESPACE - -namespace QFFmpeg { - -class VideoFrameEncoder -{ - class Data final - { - public: - ~Data(); - QAtomicInt ref = 0; - QMediaEncoderSettings settings; - float frameRate = 0.; - QSize sourceSize; - - HWAccel accel; - const AVCodec *codec = nullptr; - AVStream *stream = nullptr; - AVCodecContext *codecContext = nullptr; - SwsContext *converter = nullptr; - AVPixelFormat sourceFormat = AV_PIX_FMT_NONE; - AVPixelFormat sourceSWFormat = AV_PIX_FMT_NONE; - AVPixelFormat targetFormat = AV_PIX_FMT_NONE; - AVPixelFormat targetSWFormat = AV_PIX_FMT_NONE; - bool sourceFormatIsHWFormat = false; - bool targetFormatIsHWFormat = false; - bool downloadFromHW = false; - bool uploadToHW = false; - }; - - QExplicitlySharedDataPointer<Data> d; -public: - VideoFrameEncoder() = default; - VideoFrameEncoder(const QMediaEncoderSettings &encoderSettings, const QSize &sourceSize, float frameRate, AVPixelFormat sourceFormat, AVPixelFormat swFormat); - ~VideoFrameEncoder(); - - void initWithFormatContext(AVFormatContext *formatContext); - bool open(); - - bool isNull() const { return !d; } - - AVPixelFormat sourceFormat() const { return d ? d->sourceFormat : AV_PIX_FMT_NONE; } - AVPixelFormat targetFormat() const { return d ? d->targetFormat : AV_PIX_FMT_NONE; } - - qint64 getPts(qint64 ms); - - int sendFrame(AVFrame *frame); - AVPacket *retrievePacket(); -}; - - -} - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideosink.cpp b/src/plugins/multimedia/ffmpeg/qffmpegvideosink.cpp index ffc887495..2f02f09c1 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideosink.cpp +++ b/src/plugins/multimedia/ffmpeg/qffmpegvideosink.cpp @@ -1,43 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <qffmpegvideosink_p.h> #include <qffmpegvideobuffer_p.h> +#include <private/qvideoframe_p.h> QT_BEGIN_NAMESPACE @@ -57,7 +22,7 @@ void QFFmpegVideoSink::setRhi(QRhi *rhi) void QFFmpegVideoSink::setVideoFrame(const QVideoFrame &frame) { - auto *buffer = dynamic_cast<QFFmpegVideoBuffer *>(frame.videoBuffer()); + auto *buffer = dynamic_cast<QFFmpegVideoBuffer *>(QVideoFramePrivate::hwBuffer(frame)); if (buffer) buffer->setTextureConverter(textureConverter); @@ -65,3 +30,5 @@ void QFFmpegVideoSink::setVideoFrame(const QVideoFrame &frame) } QT_END_NAMESPACE + +#include "moc_qffmpegvideosink_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegvideosink_p.h b/src/plugins/multimedia/ffmpeg/qffmpegvideosink_p.h index 8da7aa452..92b537ee3 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegvideosink_p.h +++ b/src/plugins/multimedia/ffmpeg/qffmpegvideosink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QFFMPEGVIDEOSINK_H #define QFFMPEGVIDEOSINK_H @@ -69,9 +33,6 @@ public: void setVideoFrame(const QVideoFrame &frame) override; -Q_SIGNALS: - void rhiChanged(QRhi *rhi); - private: QFFmpeg::TextureConverter textureConverter; QRhi *m_rhi = nullptr; diff --git a/src/plugins/multimedia/ffmpeg/qffmpegwindowcapture_uwp.cpp b/src/plugins/multimedia/ffmpeg/qffmpegwindowcapture_uwp.cpp new file mode 100644 index 000000000..c139b942e --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegwindowcapture_uwp.cpp @@ -0,0 +1,508 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegwindowcapture_uwp_p.h" +#include "qffmpegsurfacecapturegrabber_p.h" +#include "qabstractvideobuffer.h" +#include <private/qvideoframe_p.h> + +#include <unknwn.h> +#include <winrt/base.h> +#include <QtCore/private/qfactorycacheregistration_p.h> +// Workaround for Windows SDK bug. +// See https://github.com/microsoft/Windows.UI.Composition-Win32-Samples/issues/47 +namespace winrt::impl +{ +template <typename Async> +auto wait_for(Async const& async, Windows::Foundation::TimeSpan const& timeout); +} +#include <winrt/Windows.Foundation.Collections.h> +#include <winrt/Windows.Graphics.Capture.h> +#include <winrt/Windows.Graphics.DirectX.h> +#include <winrt/Windows.Graphics.DirectX.Direct3D11.h> +#include <Windows.Graphics.Capture.h> +#include <Windows.Graphics.Capture.Interop.h> +#include <windows.graphics.directx.direct3d11.interop.h> + +#include <D3d11.h> +#include <dwmapi.h> +#include <lowlevelmonitorconfigurationapi.h> +#include <physicalmonitorenumerationapi.h> + +#include "qvideoframe.h" +#include <qwindow.h> +#include <qthread.h> +#include <qloggingcategory.h> +#include <qguiapplication.h> +#include <private/qmultimediautils_p.h> +#include <private/qwindowsmultimediautils_p.h> +#include <private/qcapturablewindow_p.h> +#include <qpa/qplatformscreen_p.h> + +#include <memory> +#include <system_error> + +QT_BEGIN_NAMESPACE + +using namespace winrt::Windows::Graphics::Capture; +using namespace winrt::Windows::Graphics::DirectX; +using namespace winrt::Windows::Graphics::DirectX::Direct3D11; +using namespace Windows::Graphics::DirectX::Direct3D11; +using namespace QWindowsMultimediaUtils; + +using winrt::check_hresult; +using winrt::com_ptr; +using winrt::guid_of; + +namespace { + +Q_LOGGING_CATEGORY(qLcWindowCaptureUwp, "qt.multimedia.ffmpeg.windowcapture.uwp"); + +winrt::Windows::Graphics::SizeInt32 getWindowSize(HWND hwnd) +{ + RECT windowRect{}; + ::GetWindowRect(hwnd, &windowRect); + + return { windowRect.right - windowRect.left, windowRect.bottom - windowRect.top }; +} + +QSize asQSize(winrt::Windows::Graphics::SizeInt32 size) +{ + return { size.Width, size.Height }; +} + +struct MultithreadedApartment +{ + MultithreadedApartment(const MultithreadedApartment &) = delete; + MultithreadedApartment &operator=(const MultithreadedApartment &) = delete; + + MultithreadedApartment() { winrt::init_apartment(); } + ~MultithreadedApartment() { winrt::uninit_apartment(); } +}; + +class QUwpTextureVideoBuffer : public QAbstractVideoBuffer +{ +public: + QUwpTextureVideoBuffer(com_ptr<IDXGISurface> &&surface) : m_surface(surface) { } + + ~QUwpTextureVideoBuffer() override { Q_ASSERT(m_mapMode == QtVideo::MapMode::NotMapped); } + + MapData map(QtVideo::MapMode mode) override + { + if (m_mapMode != QtVideo::MapMode::NotMapped) + return {}; + + if (mode == QtVideo::MapMode::ReadOnly) { + DXGI_MAPPED_RECT rect = {}; + HRESULT hr = m_surface->Map(&rect, DXGI_MAP_READ); + if (SUCCEEDED(hr)) { + DXGI_SURFACE_DESC desc = {}; + hr = m_surface->GetDesc(&desc); + + MapData md = {}; + md.planeCount = 1; + md.bytesPerLine[0] = rect.Pitch; + md.data[0] = rect.pBits; + md.dataSize[0] = rect.Pitch * desc.Height; + + m_mapMode = QtVideo::MapMode::ReadOnly; + + return md; + } else { + qCDebug(qLcWindowCaptureUwp) << "Failed to map DXGI surface" << errorString(hr); + return {}; + } + } + + return {}; + } + + void unmap() override + { + if (m_mapMode == QtVideo::MapMode::NotMapped) + return; + + const HRESULT hr = m_surface->Unmap(); + if (FAILED(hr)) + qCDebug(qLcWindowCaptureUwp) << "Failed to unmap surface" << errorString(hr); + + m_mapMode = QtVideo::MapMode::NotMapped; + } + + QVideoFrameFormat format() const override { return {}; } + +private: + QtVideo::MapMode m_mapMode = QtVideo::MapMode::NotMapped; + com_ptr<IDXGISurface> m_surface; +}; + +struct WindowGrabber +{ + WindowGrabber() = default; + + WindowGrabber(IDXGIAdapter1 *adapter, HWND hwnd) + : m_frameSize{ getWindowSize(hwnd) }, m_captureWindow{ hwnd } + { + check_hresult(D3D11CreateDevice(adapter, D3D_DRIVER_TYPE_UNKNOWN, nullptr, 0, nullptr, 0, + D3D11_SDK_VERSION, m_device.put(), nullptr, nullptr)); + + const auto captureItem = createCaptureItem(hwnd); + + m_framePool = Direct3D11CaptureFramePool::CreateFreeThreaded( + getCaptureDevice(m_device), m_pixelFormat, 1, + captureItem.Size()); + + m_session = m_framePool.CreateCaptureSession(captureItem); + + // If supported, enable cursor capture + if (const auto session2 = m_session.try_as<IGraphicsCaptureSession2>()) + session2.IsCursorCaptureEnabled(true); + + // If supported, disable colored border around captured window to match other platforms + if (const auto session3 = m_session.try_as<IGraphicsCaptureSession3>()) + session3.IsBorderRequired(false); + + m_session.StartCapture(); + } + + ~WindowGrabber() + { + m_framePool.Close(); + m_session.Close(); + } + + com_ptr<IDXGISurface> tryGetFrame() + { + const Direct3D11CaptureFrame frame = m_framePool.TryGetNextFrame(); + if (!frame) { + + // Stop capture and report failure if window was closed. If we don't stop, + // testing shows that either we don't get any frames, or we get blank frames. + // Emitting an error will prevent this inconsistent behavior, and makes the + // Windows implementation behave like the Linux implementation + if (!IsWindow(m_captureWindow)) + throw std::runtime_error("Window was closed"); + + // Blank frames may come spuriously if no new window texture + // is available yet. + return {}; + } + + if (m_frameSize != frame.ContentSize()) { + m_frameSize = frame.ContentSize(); + m_framePool.Recreate(getCaptureDevice(m_device), m_pixelFormat, 1, frame.ContentSize()); + return {}; + } + + return copyTexture(m_device, frame.Surface()); + } + +private: + static GraphicsCaptureItem createCaptureItem(HWND hwnd) + { + const auto factory = winrt::get_activation_factory<GraphicsCaptureItem>(); + const auto interop = factory.as<IGraphicsCaptureItemInterop>(); + + GraphicsCaptureItem item = { nullptr }; + winrt::hresult status = S_OK; + + // Attempt to create capture item with retry, because this occasionally fails, + // particularly in unit tests. When the failure code is E_INVALIDARG, it + // seems to help to sleep for a bit and retry. See QTBUG-116025. + constexpr int maxRetry = 10; + constexpr std::chrono::milliseconds retryDelay{ 100 }; + for (int retryNum = 0; retryNum < maxRetry; ++retryNum) { + + status = interop->CreateForWindow(hwnd, winrt::guid_of<GraphicsCaptureItem>(), + winrt::put_abi(item)); + + if (status != E_INVALIDARG) + break; + + qCWarning(qLcWindowCaptureUwp) + << "Failed to create capture item:" + << QString::fromStdWString(winrt::hresult_error(status).message().c_str()) + << "Retry number" << retryNum; + + if (retryNum + 1 < maxRetry) + QThread::sleep(retryDelay); + } + + // Throw if we fail to create the capture item + check_hresult(status); + + return item; + } + + static IDirect3DDevice getCaptureDevice(const com_ptr<ID3D11Device> &d3dDevice) + { + const auto dxgiDevice = d3dDevice.as<IDXGIDevice>(); + + com_ptr<IInspectable> device; + check_hresult(CreateDirect3D11DeviceFromDXGIDevice(dxgiDevice.get(), device.put())); + + return device.as<IDirect3DDevice>(); + } + + static com_ptr<IDXGISurface> copyTexture(const com_ptr<ID3D11Device> &device, + const IDirect3DSurface &capturedTexture) + { + const auto dxgiInterop{ capturedTexture.as<IDirect3DDxgiInterfaceAccess>() }; + if (!dxgiInterop) + return {}; + + com_ptr<IDXGISurface> dxgiSurface; + check_hresult(dxgiInterop->GetInterface(guid_of<IDXGISurface>(), dxgiSurface.put_void())); + + DXGI_SURFACE_DESC desc = {}; + check_hresult(dxgiSurface->GetDesc(&desc)); + + D3D11_TEXTURE2D_DESC texDesc = {}; + texDesc.Width = desc.Width; + texDesc.Height = desc.Height; + texDesc.Format = DXGI_FORMAT_R8G8B8A8_UNORM; + texDesc.Usage = D3D11_USAGE_STAGING; + texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + texDesc.MiscFlags = 0; + texDesc.BindFlags = 0; + texDesc.ArraySize = 1; + texDesc.MipLevels = 1; + texDesc.SampleDesc = { 1, 0 }; + + com_ptr<ID3D11Texture2D> texture; + check_hresult(device->CreateTexture2D(&texDesc, nullptr, texture.put())); + + com_ptr<ID3D11DeviceContext> ctx; + device->GetImmediateContext(ctx.put()); + ctx->CopyResource(texture.get(), dxgiSurface.as<ID3D11Resource>().get()); + + return texture.as<IDXGISurface>(); + } + + MultithreadedApartment m_comApartment{}; + HWND m_captureWindow{}; + winrt::Windows::Graphics::SizeInt32 m_frameSize{}; + com_ptr<ID3D11Device> m_device; + Direct3D11CaptureFramePool m_framePool{ nullptr }; + GraphicsCaptureSession m_session{ nullptr }; + const DirectXPixelFormat m_pixelFormat = DirectXPixelFormat::R8G8B8A8UIntNormalized; +}; + +} // namespace + +class QFFmpegWindowCaptureUwp::Grabber : public QFFmpegSurfaceCaptureGrabber +{ + Q_OBJECT +public: + Grabber(QFFmpegWindowCaptureUwp &capture, HWND hwnd) + : m_hwnd(hwnd), + m_format(QVideoFrameFormat(asQSize(getWindowSize(hwnd)), + QVideoFrameFormat::Format_RGBX8888)) + { + const HMONITOR monitor = MonitorFromWindow(hwnd, MONITOR_DEFAULTTONULL); + m_adapter = getAdapter(monitor); + + const qreal refreshRate = getMonitorRefreshRateHz(monitor); + + m_format.setStreamFrameRate(refreshRate); + setFrameRate(refreshRate); + + addFrameCallback(capture, &QFFmpegWindowCaptureUwp::newVideoFrame); + connect(this, &Grabber::errorUpdated, &capture, &QFFmpegWindowCaptureUwp::updateError); + } + + ~Grabber() override { stop(); } + + QVideoFrameFormat frameFormat() const { return m_format; } + +protected: + + void initializeGrabbingContext() override + { + if (!m_adapter || !IsWindow(m_hwnd)) + return; // Error already logged + + try { + m_windowGrabber = std::make_unique<WindowGrabber>(m_adapter.get(), m_hwnd); + + QFFmpegSurfaceCaptureGrabber::initializeGrabbingContext(); + } catch (const winrt::hresult_error &err) { + + const QString message = QLatin1String("Unable to capture window: ") + + QString::fromWCharArray(err.message().c_str()); + + updateError(InternalError, message); + } + } + + void finalizeGrabbingContext() override + { + QFFmpegSurfaceCaptureGrabber::finalizeGrabbingContext(); + m_windowGrabber = nullptr; + } + + QVideoFrame grabFrame() override + { + try { + com_ptr<IDXGISurface> texture = m_windowGrabber->tryGetFrame(); + if (!texture) + return {}; // No frame available yet + + const QSize size = getTextureSize(texture); + + m_format.setFrameSize(size); + + return QVideoFramePrivate::createFrame( + std::make_unique<QUwpTextureVideoBuffer>(std::move(texture)), m_format); + + } catch (const winrt::hresult_error &err) { + + const QString message = QLatin1String("Window capture failed: ") + + QString::fromWCharArray(err.message().c_str()); + + updateError(InternalError, message); + } catch (const std::runtime_error& e) { + updateError(CaptureFailed, QString::fromLatin1(e.what())); + } + + return {}; + } + +private: + static com_ptr<IDXGIAdapter1> getAdapter(HMONITOR handle) + { + com_ptr<IDXGIFactory1> factory; + check_hresult(CreateDXGIFactory1(guid_of<IDXGIFactory1>(), factory.put_void())); + + com_ptr<IDXGIAdapter1> adapter; + for (quint32 i = 0; factory->EnumAdapters1(i, adapter.put()) == S_OK; adapter = nullptr, i++) { + com_ptr<IDXGIOutput> output; + for (quint32 j = 0; adapter->EnumOutputs(j, output.put()) == S_OK; output = nullptr, j++) { + DXGI_OUTPUT_DESC desc = {}; + HRESULT hr = output->GetDesc(&desc); + if (hr == S_OK && desc.Monitor == handle) + return adapter; + } + } + return {}; + } + + static QSize getTextureSize(const com_ptr<IDXGISurface> &surf) + { + if (!surf) + return {}; + + DXGI_SURFACE_DESC desc; + check_hresult(surf->GetDesc(&desc)); + + return { static_cast<int>(desc.Width), static_cast<int>(desc.Height) }; + } + + static qreal getMonitorRefreshRateHz(HMONITOR handle) + { + DWORD count = 0; + if (GetNumberOfPhysicalMonitorsFromHMONITOR(handle, &count)) { + std::vector<PHYSICAL_MONITOR> monitors{ count }; + if (GetPhysicalMonitorsFromHMONITOR(handle, count, monitors.data())) { + for (const auto &monitor : std::as_const(monitors)) { + MC_TIMING_REPORT screenTiming = {}; + if (GetTimingReport(monitor.hPhysicalMonitor, &screenTiming)) { + // Empirically we found that GetTimingReport does not return + // the frequency in updates per second as documented, but in + // updates per 100 seconds. + return static_cast<qreal>(screenTiming.dwVerticalFrequencyInHZ) / 100.0; + } + } + } + } + return DefaultScreenCaptureFrameRate; + } + + HWND m_hwnd{}; + com_ptr<IDXGIAdapter1> m_adapter{}; + std::unique_ptr<WindowGrabber> m_windowGrabber; + QVideoFrameFormat m_format; +}; + +QFFmpegWindowCaptureUwp::QFFmpegWindowCaptureUwp() : QPlatformSurfaceCapture(WindowSource{}) +{ + qCDebug(qLcWindowCaptureUwp) << "Creating UWP screen capture"; +} + +QFFmpegWindowCaptureUwp::~QFFmpegWindowCaptureUwp() = default; + +static QString isCapturableWindow(HWND hwnd) +{ + if (!IsWindow(hwnd)) + return "Invalid window handle"; + + if (hwnd == GetShellWindow()) + return "Cannot capture the shell window"; + + wchar_t className[MAX_PATH] = {}; + GetClassName(hwnd, className, MAX_PATH); + if (QString::fromWCharArray(className).length() == 0) + return "Cannot capture windows without a class name"; + + if (!IsWindowVisible(hwnd)) + return "Cannot capture invisible windows"; + + if (GetAncestor(hwnd, GA_ROOT) != hwnd) + return "Can only capture root windows"; + + const LONG_PTR style = GetWindowLongPtr(hwnd, GWL_STYLE); + if (style & WS_DISABLED) + return "Cannot capture disabled windows"; + + const LONG_PTR exStyle = GetWindowLongPtr(hwnd, GWL_EXSTYLE); + if (exStyle & WS_EX_TOOLWINDOW) + return "No tooltips"; + + DWORD cloaked = FALSE; + const HRESULT hr = DwmGetWindowAttribute(hwnd, DWMWA_CLOAKED, &cloaked, sizeof(cloaked)); + if (SUCCEEDED(hr) && cloaked == DWM_CLOAKED_SHELL) + return "Cannot capture cloaked windows"; + + return {}; +} + +bool QFFmpegWindowCaptureUwp::setActiveInternal(bool active) +{ + if (static_cast<bool>(m_grabber) == active) + return false; + + if (m_grabber) { + m_grabber.reset(); + return true; + } + + const auto window = source<WindowSource>(); + const auto handle = QCapturableWindowPrivate::handle(window); + + const auto hwnd = reinterpret_cast<HWND>(handle ? handle->id : 0); + if (const QString error = isCapturableWindow(hwnd); !error.isEmpty()) { + updateError(InternalError, error); + return false; + } + + m_grabber = std::make_unique<Grabber>(*this, hwnd); + m_grabber->start(); + + return true; +} + +bool QFFmpegWindowCaptureUwp::isSupported() +{ + return GraphicsCaptureSession::IsSupported(); +} + +QVideoFrameFormat QFFmpegWindowCaptureUwp::frameFormat() const +{ + if (m_grabber) + return m_grabber->frameFormat(); + return {}; +} + +QT_END_NAMESPACE + +#include "qffmpegwindowcapture_uwp.moc" diff --git a/src/plugins/multimedia/ffmpeg/qffmpegwindowcapture_uwp_p.h b/src/plugins/multimedia/ffmpeg/qffmpegwindowcapture_uwp_p.h new file mode 100644 index 000000000..10f6e62be --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qffmpegwindowcapture_uwp_p.h @@ -0,0 +1,46 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGWINDOWCAPTURE_UWP_P_H +#define QFFMPEGWINDOWCAPTURE_UWP_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/qnamespace.h> +#include "private/qplatformsurfacecapture_p.h" +#include "qvideoframeformat.h" +#include <memory> + +QT_BEGIN_NAMESPACE + +class QFFmpegWindowCaptureUwp : public QPlatformSurfaceCapture +{ +public: + QFFmpegWindowCaptureUwp(); + ~QFFmpegWindowCaptureUwp() override; + + QVideoFrameFormat frameFormat() const override; + + static bool isSupported(); + +private: + class Grabber; + + bool setActiveInternal(bool active) override; + +private: + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // QFFMPEGWINDOWCAPTURE_UWP_P_H diff --git a/src/plugins/multimedia/ffmpeg/qgdiwindowcapture.cpp b/src/plugins/multimedia/ffmpeg/qgdiwindowcapture.cpp new file mode 100644 index 000000000..97742043c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qgdiwindowcapture.cpp @@ -0,0 +1,197 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qgdiwindowcapture_p.h" + +#include "qvideoframe.h" +#include "qffmpegsurfacecapturegrabber_p.h" +#include "private/qcapturablewindow_p.h" +#include "private/qmemoryvideobuffer_p.h" +#include "private/qvideoframe_p.h" + +#include <qt_windows.h> +#include <QtCore/qloggingcategory.h> + +static Q_LOGGING_CATEGORY(qLcGdiWindowCapture, "qt.multimedia.ffmpeg.gdiwindowcapture"); + +QT_BEGIN_NAMESPACE + +class QGdiWindowCapture::Grabber : public QFFmpegSurfaceCaptureGrabber +{ +public: + static std::unique_ptr<Grabber> create(QGdiWindowCapture &capture, HWND hWnd) + { + auto hdcWindow = GetDC(hWnd); + if (!hdcWindow) { + capture.updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot create a window drawing context")); + return nullptr; + } + + auto hdcMem = CreateCompatibleDC(hdcWindow); + + if (!hdcMem) { + capture.updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot create a compatible drawing context")); + return nullptr; + } + + std::unique_ptr<Grabber> result(new Grabber(capture, hWnd, hdcWindow, hdcMem)); + if (!result->update()) + return nullptr; + + result->start(); + return result; + } + + ~Grabber() override + { + stop(); + + if (m_hBitmap) + DeleteObject(m_hBitmap); + + if (m_hdcMem) + DeleteDC(m_hdcMem); + + if (m_hdcWindow) + ReleaseDC(m_hwnd, m_hdcWindow); + } + + QVideoFrameFormat format() const { return m_format; } + +private: + Grabber(QGdiWindowCapture &capture, HWND hWnd, HDC hdcWindow, HDC hdcMem) + : m_hwnd(hWnd), m_hdcWindow(hdcWindow), m_hdcMem(hdcMem) + { + if (auto rate = GetDeviceCaps(hdcWindow, VREFRESH); rate > 0) + setFrameRate(rate); + + addFrameCallback(capture, &QGdiWindowCapture::newVideoFrame); + connect(this, &Grabber::errorUpdated, &capture, &QGdiWindowCapture::updateError); + } + + bool update() + { + RECT windowRect{}; + if (!GetWindowRect(m_hwnd, &windowRect)) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot get window size")); + return false; + } + + const QSize size{ windowRect.right - windowRect.left, windowRect.bottom - windowRect.top }; + + if (m_format.isValid() && size == m_format.frameSize() && m_hBitmap) + return true; + + if (m_hBitmap) + DeleteObject(std::exchange(m_hBitmap, nullptr)); + + if (size.isEmpty()) { + m_format = {}; + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Invalid window size")); + return false; + } + + m_hBitmap = CreateCompatibleBitmap(m_hdcWindow, size.width(), size.height()); + + if (!m_hBitmap) { + m_format = {}; + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot create a compatible bitmap")); + return false; + } + + QVideoFrameFormat format(size, QVideoFrameFormat::Format_BGRX8888); + format.setStreamFrameRate(frameRate()); + m_format = format; + return true; + } + + QVideoFrame grabFrame() override + { + if (!update()) + return {}; + + const auto oldBitmap = SelectObject(m_hdcMem, m_hBitmap); + auto deselect = qScopeGuard([&]() { SelectObject(m_hdcMem, oldBitmap); }); + + const auto size = m_format.frameSize(); + + if (!BitBlt(m_hdcMem, 0, 0, size.width(), size.height(), m_hdcWindow, 0, 0, SRCCOPY)) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot copy image to the compatible DC")); + return {}; + } + + BITMAPINFO info{}; + auto &header = info.bmiHeader; + header.biSize = sizeof(BITMAPINFOHEADER); + header.biWidth = size.width(); + header.biHeight = -size.height(); // negative height to ensure top-down orientation + header.biPlanes = 1; + header.biBitCount = 32; + header.biCompression = BI_RGB; + + const auto bytesPerLine = size.width() * 4; + + QByteArray array(size.height() * bytesPerLine, Qt::Uninitialized); + + const auto copiedHeight = GetDIBits(m_hdcMem, m_hBitmap, 0, size.height(), array.data(), &info, DIB_RGB_COLORS); + if (copiedHeight != size.height()) { + qCWarning(qLcGdiWindowCapture) << copiedHeight << "lines have been copied, expected:" << size.height(); + // In practice, it might fail randomly first time after start. So we don't consider it as an error. + // TODO: investigate reasons and properly handle the error + // updateError(QPlatformSurfaceCapture::CaptureFailed, + // QLatin1String("Cannot get raw image data")); + return {}; + } + + if (header.biWidth != size.width() || header.biHeight != -size.height() + || header.biPlanes != 1 || header.biBitCount != 32 || header.biCompression != BI_RGB) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Output bitmap info is unexpected")); + return {}; + } + + return QVideoFramePrivate::createFrame( + std::make_unique<QMemoryVideoBuffer>(std::move(array), bytesPerLine), m_format); + } + +private: + HWND m_hwnd = {}; + QVideoFrameFormat m_format; + HDC m_hdcWindow = {}; + HDC m_hdcMem = {}; + HBITMAP m_hBitmap = {}; +}; + +QGdiWindowCapture::QGdiWindowCapture() : QPlatformSurfaceCapture(WindowSource{}) { } + +QGdiWindowCapture::~QGdiWindowCapture() = default; + +QVideoFrameFormat QGdiWindowCapture::frameFormat() const +{ + return m_grabber ? m_grabber->format() : QVideoFrameFormat(); +} + +bool QGdiWindowCapture::setActiveInternal(bool active) +{ + if (active == static_cast<bool>(m_grabber)) + return true; + + if (m_grabber) { + m_grabber.reset(); + } else { + auto window = source<WindowSource>(); + auto handle = QCapturableWindowPrivate::handle(window); + + m_grabber = Grabber::create(*this, reinterpret_cast<HWND>(handle ? handle->id : 0)); + } + + return static_cast<bool>(m_grabber) == active; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qgdiwindowcapture_p.h b/src/plugins/multimedia/ffmpeg/qgdiwindowcapture_p.h new file mode 100644 index 000000000..d4498455c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qgdiwindowcapture_p.h @@ -0,0 +1,43 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGDIWINDOWCAPTURE_P_H +#define QGDIWINDOWCAPTURE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformsurfacecapture_p.h" + +#include <memory> + +QT_BEGIN_NAMESPACE + +class QGdiWindowCapture : public QPlatformSurfaceCapture +{ + class Grabber; + +public: + QGdiWindowCapture(); + ~QGdiWindowCapture() override; + + QVideoFrameFormat frameFormat() const override; + +protected: + bool setActiveInternal(bool active) override; + +private: + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // QGDIWINDOWCAPTURE_P_H diff --git a/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture.cpp b/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture.cpp new file mode 100644 index 000000000..4bd1f6a65 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture.cpp @@ -0,0 +1,221 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qvideoframe.h" +#include "qgrabwindowsurfacecapture_p.h" +#include "qscreencapture.h" +#include "qffmpegsurfacecapturegrabber_p.h" + +#include "private/qimagevideobuffer_p.h" +#include "private/qcapturablewindow_p.h" +#include "private/qvideoframe_p.h" + +#include "qscreen.h" +#include "qmutex.h" +#include "qwaitcondition.h" +#include "qpixmap.h" +#include "qguiapplication.h" +#include "qwindow.h" +#include "qpointer.h" + +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace { + +using WindowUPtr = std::unique_ptr<QWindow>; + +} // namespace + +class QGrabWindowSurfaceCapture::Grabber : public QFFmpegSurfaceCaptureGrabber +{ +public: + Grabber(QGrabWindowSurfaceCapture &capture, QScreen *screen) : Grabber(capture, screen, nullptr) + { + Q_ASSERT(screen); + } + + Grabber(QGrabWindowSurfaceCapture &capture, WindowUPtr window) + : Grabber(capture, nullptr, std::move(window)) + { + Q_ASSERT(m_window); + } + + ~Grabber() override { + stop(); + + Q_ASSERT(!m_screenRemovingLocked); + } + + const QVideoFrameFormat &format() + { + QMutexLocker locker(&m_formatMutex); + while (!m_format) + m_waitForFormat.wait(&m_formatMutex); + return *m_format; + } + +private: + Grabber(QGrabWindowSurfaceCapture &capture, QScreen *screen, WindowUPtr window) + : QFFmpegSurfaceCaptureGrabber( + QGuiApplication::platformName() == QLatin1String("eglfs") + ? QFFmpegSurfaceCaptureGrabber::UseCurrentThread + : QFFmpegSurfaceCaptureGrabber::CreateGrabbingThread), + m_capture(capture), + m_screen(screen), + m_window(std::move(window)) + { + connect(qApp, &QGuiApplication::screenRemoved, this, &Grabber::onScreenRemoved); + addFrameCallback(m_capture, &QGrabWindowSurfaceCapture::newVideoFrame); + connect(this, &Grabber::errorUpdated, &m_capture, &QGrabWindowSurfaceCapture::updateError); + } + + void onScreenRemoved(QScreen *screen) + { + /* The hack allows to lock screens removing while QScreen::grabWindow is in progress. + * The current solution works since QGuiApplication::screenRemoved is emitted from + * the destructor of QScreen before destruction members of the object. + * Note, QGuiApplication works with screens in the main thread, and any removing of a screen + * must be synchronized with grabbing thread. + */ + QMutexLocker locker(&m_screenRemovingMutex); + + if (m_screenRemovingLocked) { + qDebug() << "Screen" << screen->name() + << "is removed while screen window grabbing lock is active"; + } + + while (m_screenRemovingLocked) + m_screenRemovingWc.wait(&m_screenRemovingMutex); + } + + void setScreenRemovingLocked(bool locked) + { + Q_ASSERT(locked != m_screenRemovingLocked); + + { + QMutexLocker locker(&m_screenRemovingMutex); + m_screenRemovingLocked = locked; + } + + if (!locked) + m_screenRemovingWc.wakeAll(); + } + + void updateFormat(const QVideoFrameFormat &format) + { + if (m_format && m_format->isValid()) + return; + + { + QMutexLocker locker(&m_formatMutex); + m_format = format; + } + + m_waitForFormat.wakeAll(); + } + + QVideoFrame grabFrame() override + { + setScreenRemovingLocked(true); + auto screenGuard = qScopeGuard(std::bind(&Grabber::setScreenRemovingLocked, this, false)); + + WId wid = m_window ? m_window->winId() : 0; + QScreen *screen = m_window ? m_window->screen() : m_screen ? m_screen.data() : nullptr; + + if (!screen) { + updateError(QPlatformSurfaceCapture::CaptureFailed, "Screen not found"); + return {}; + } + + setFrameRate(screen->refreshRate()); + + QPixmap p = screen->grabWindow(wid); + auto buffer = std::make_unique<QImageVideoBuffer>(p.toImage()); + const auto img = buffer->underlyingImage(); + + QVideoFrameFormat format(img.size(), + QVideoFrameFormat::pixelFormatFromImageFormat(img.format())); + format.setStreamFrameRate(screen->refreshRate()); + updateFormat(format); + + if (!format.isValid()) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + "Failed to grab the screen content"); + return {}; + } + + return QVideoFramePrivate::createFrame(std::move(buffer), std::move(format)); + } + +private: + QGrabWindowSurfaceCapture &m_capture; + QPointer<QScreen> m_screen; + WindowUPtr m_window; + + QMutex m_formatMutex; + QWaitCondition m_waitForFormat; + std::optional<QVideoFrameFormat> m_format; + + QMutex m_screenRemovingMutex; + bool m_screenRemovingLocked = false; + QWaitCondition m_screenRemovingWc; +}; + +QGrabWindowSurfaceCapture::QGrabWindowSurfaceCapture(Source initialSource) + : QPlatformSurfaceCapture(initialSource) +{ +} + +QGrabWindowSurfaceCapture::~QGrabWindowSurfaceCapture() = default; + +QVideoFrameFormat QGrabWindowSurfaceCapture::frameFormat() const +{ + if (m_grabber) + return m_grabber->format(); + else + return {}; +} + +bool QGrabWindowSurfaceCapture::setActiveInternal(bool active) +{ + if (active == static_cast<bool>(m_grabber)) + return true; + + if (m_grabber) + m_grabber.reset(); + else + std::visit([this](auto source) { activate(source); }, source()); + + return static_cast<bool>(m_grabber) == active; +} + +void QGrabWindowSurfaceCapture::activate(ScreenSource screen) +{ + if (!checkScreenWithError(screen)) + return; + + m_grabber = std::make_unique<Grabber>(*this, screen); + m_grabber->start(); +} + +void QGrabWindowSurfaceCapture::activate(WindowSource window) +{ + auto handle = QCapturableWindowPrivate::handle(window); + auto wid = handle ? handle->id : 0; + if (auto wnd = WindowUPtr(QWindow::fromWinId(wid))) { + if (!wnd->screen()) { + updateError(InternalError, + "Window " + QString::number(wid) + " doesn't belong to any screen"); + } else { + m_grabber = std::make_unique<Grabber>(*this, std::move(wnd)); + m_grabber->start(); + } + } else { + updateError(NotFound, + "Window " + QString::number(wid) + "doesn't exist or permissions denied"); + } +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture_p.h b/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture_p.h new file mode 100644 index 000000000..a76ce9507 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qgrabwindowsurfacecapture_p.h @@ -0,0 +1,48 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGRABWINDOWSURFACECAPTURE_P_H +#define QGRABWINDOWSURFACECAPTURE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformsurfacecapture_p.h" + +#include <memory> + +QT_BEGIN_NAMESPACE + +class QGrabWindowSurfaceCapture : public QPlatformSurfaceCapture +{ + class Grabber; + +public: + explicit QGrabWindowSurfaceCapture(Source initialSource); + ~QGrabWindowSurfaceCapture() override; + + QVideoFrameFormat frameFormat() const override; + +protected: + bool setActiveInternal(bool active) override; + +private: + void activate(ScreenSource); + + void activate(WindowSource); + +private: + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // QGRABWINDOWSURFACECAPTURE_P_H diff --git a/src/plugins/multimedia/ffmpeg/qopenglvideobuffer.cpp b/src/plugins/multimedia/ffmpeg/qopenglvideobuffer.cpp new file mode 100644 index 000000000..4ac08fd24 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qopenglvideobuffer.cpp @@ -0,0 +1,96 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qopenglvideobuffer_p.h" + +#include <qoffscreensurface.h> +#include <qthread.h> +#include <private/qimagevideobuffer_p.h> + +#include <QtOpenGL/private/qopenglcompositor_p.h> +#include <QtOpenGL/private/qopenglframebufferobject_p.h> + +QT_BEGIN_NAMESPACE + +static QOpenGLContext *createContext(QOpenGLContext *shareContext) +{ + // Create an OpenGL context for the current thread. The lifetime of the context is tied to the + // lifetime of the current thread. + auto context = std::make_unique<QOpenGLContext>(); + context->setShareContext(shareContext); + + if (!context->create()) { + qWarning() << "Couldn't create an OpenGL context for QOpenGLVideoBuffer"; + return nullptr; + } + + QObject::connect(QThread::currentThread(), &QThread::finished, + context.get(), &QOpenGLContext::deleteLater); + return context.release(); +} + +static bool setCurrentOpenGLContext() +{ + auto compositorContext = QOpenGLCompositor::instance()->context(); + + // A thread-local variable is used to avoid creating a new context if we're called on the same + // thread. The context lifetime is tied to the current thread lifetime (see createContext()). + static thread_local QOpenGLContext *context = nullptr; + static thread_local QOffscreenSurface *surface = nullptr; + + if (!context) { + context = (compositorContext->thread() == QThread::currentThread()) + ? compositorContext + : createContext(compositorContext); + + if (!context) + return false; + + surface = new QOffscreenSurface(nullptr, context); + surface->setFormat(context->format()); + surface->create(); + } + + return context->makeCurrent(surface); +} + +QOpenGLVideoBuffer::QOpenGLVideoBuffer(std::unique_ptr<QOpenGLFramebufferObject> fbo) + : QHwVideoBuffer(QVideoFrame::RhiTextureHandle), m_fbo(std::move(fbo)) +{ + Q_ASSERT(m_fbo); +} + +QOpenGLVideoBuffer::~QOpenGLVideoBuffer() { } + +QAbstractVideoBuffer::MapData QOpenGLVideoBuffer::map(QtVideo::MapMode mode) +{ + return ensureImageBuffer().map(mode); +} + +void QOpenGLVideoBuffer::unmap() +{ + if (m_imageBuffer) + m_imageBuffer->unmap(); +} + +quint64 QOpenGLVideoBuffer::textureHandle(QRhi *, int plane) const +{ + Q_UNUSED(plane); + return m_fbo->texture(); +} + +QImageVideoBuffer &QOpenGLVideoBuffer::ensureImageBuffer() +{ + // Create image buffer if not yet created. + // This is protected by mapMutex in QVideoFrame::map. + if (!m_imageBuffer) { + if (!setCurrentOpenGLContext()) + qWarning() << "Failed to set current OpenGL context"; + + m_imageBuffer = std::make_unique<QImageVideoBuffer>(m_fbo->toImage(false)); + } + + return *m_imageBuffer; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qopenglvideobuffer_p.h b/src/plugins/multimedia/ffmpeg/qopenglvideobuffer_p.h new file mode 100644 index 000000000..6e62625d0 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qopenglvideobuffer_p.h @@ -0,0 +1,44 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QOPENGLVIDEOBUFFER_P_H +#define QOPENGLVIDEOBUFFER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qhwvideobuffer_p.h> + +QT_BEGIN_NAMESPACE + +class QImageVideoBuffer; +class QOpenGLFramebufferObject; + +class QOpenGLVideoBuffer : public QHwVideoBuffer +{ +public: + QOpenGLVideoBuffer(std::unique_ptr<QOpenGLFramebufferObject> fbo); + ~QOpenGLVideoBuffer(); + + MapData map(QtVideo::MapMode mode) override; + void unmap() override; + quint64 textureHandle(QRhi *, int plane) const override; + + QImageVideoBuffer &ensureImageBuffer(); + +private: + std::unique_ptr<QOpenGLFramebufferObject> m_fbo; + std::unique_ptr<QImageVideoBuffer> m_imageBuffer; +}; + +QT_END_NAMESPACE + +#endif // QOPENGLVIDEOBUFFER_P_H diff --git a/src/plugins/multimedia/ffmpeg/qv4l2camera.cpp b/src/plugins/multimedia/ffmpeg/qv4l2camera.cpp index 36ef020db..f9f18296b 100644 --- a/src/plugins/multimedia/ffmpeg/qv4l2camera.cpp +++ b/src/plugins/multimedia/ffmpeg/qv4l2camera.cpp @@ -1,85 +1,24 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qv4l2camera_p.h" +#include "qv4l2filedescriptor_p.h" +#include "qv4l2memorytransfer_p.h" -#include <qdir.h> -#include <qmutex.h> -#include <qendian.h> #include <private/qcameradevice_p.h> -#include <private/qabstractvideobuffer_p.h> -#include <private/qvideotexturehelper_p.h> #include <private/qmultimediautils_p.h> -#include <private/qplatformmediadevices_p.h> - -#include <sys/types.h> -#include <sys/stat.h> -#include <sys/ioctl.h> -#include <unistd.h> -#include <fcntl.h> +#include <private/qmemoryvideobuffer_p.h> +#include <private/qvideoframe_p.h> #include <private/qcore_unix_p.h> -#include <sys/mman.h> -#include <linux/videodev2.h> +#include <qsocketnotifier.h> +#include <qloggingcategory.h> QT_BEGIN_NAMESPACE -QV4L2CameraDevices::QV4L2CameraDevices(QPlatformMediaIntegration *integration) - : QPlatformVideoDevices(integration) -{ - deviceWatcher.addPath(QLatin1String("/dev")); - connect(&deviceWatcher, &QFileSystemWatcher::directoryChanged, this, &QV4L2CameraDevices::checkCameras); - doCheckCameras(); -} - -QList<QCameraDevice> QV4L2CameraDevices::videoDevices() const -{ - return cameras; -} - -void QV4L2CameraDevices::checkCameras() -{ - doCheckCameras(); - videoInputsChanged(); -} +Q_STATIC_LOGGING_CATEGORY(qLcV4L2Camera, "qt.multimedia.ffmpeg.v4l2camera"); -const struct { +static const struct { QVideoFrameFormat::PixelFormat fmt; uint32_t v4l2Format; } formatMap[] = { @@ -105,7 +44,7 @@ const struct { { QVideoFrameFormat::Format_Invalid, 0 }, }; -static QVideoFrameFormat::PixelFormat formatForV4L2Format(uint32_t v4l2Format) +QVideoFrameFormat::PixelFormat formatForV4L2Format(uint32_t v4l2Format) { auto *f = formatMap; while (f->v4l2Format) { @@ -116,7 +55,7 @@ static QVideoFrameFormat::PixelFormat formatForV4L2Format(uint32_t v4l2Format) return QVideoFrameFormat::Format_Invalid; } -static uint32_t v4l2FormatForPixelFormat(QVideoFrameFormat::PixelFormat format) +uint32_t v4l2FormatForPixelFormat(QVideoFrameFormat::PixelFormat format) { auto *f = formatMap; while (f->v4l2Format) { @@ -127,177 +66,6 @@ static uint32_t v4l2FormatForPixelFormat(QVideoFrameFormat::PixelFormat format) return 0; } - -void QV4L2CameraDevices::doCheckCameras() -{ - cameras.clear(); - - QDir dir(QLatin1String("/dev")); - const auto devices = dir.entryList(QDir::System); - - bool first = true; - - for (auto device : devices) { -// qDebug() << "device:" << device; - if (!device.startsWith(QLatin1String("video"))) - continue; - - QByteArray file = QFile::encodeName(dir.filePath(device)); - int fd = open(file.constData(), O_RDONLY); - if (fd < 0) - continue; - - QCameraDevicePrivate *camera = nullptr; - v4l2_fmtdesc formatDesc = {}; - - struct v4l2_capability cap; - if (ioctl(fd, VIDIOC_QUERYCAP, &cap) < 0) - goto fail; - - if (cap.device_caps & V4L2_CAP_META_CAPTURE) - goto fail; - if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE)) - goto fail; - if (!(cap.capabilities & V4L2_CAP_STREAMING)) - goto fail; - - camera = new QCameraDevicePrivate; - camera->id = file; - camera->description = QString::fromUtf8((const char *)cap.card); -// qDebug() << "found camera" << camera->id << camera->description; - - formatDesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - - while (!ioctl(fd, VIDIOC_ENUM_FMT, &formatDesc)) { - auto pixelFmt = formatForV4L2Format(formatDesc.pixelformat); - qDebug() << " " << pixelFmt; - - if (pixelFmt == QVideoFrameFormat::Format_Invalid) { - ++formatDesc.index; - continue; - } - -// qDebug() << "frame sizes:"; - v4l2_frmsizeenum frameSize = {}; - frameSize.pixel_format = formatDesc.pixelformat; - - while (!ioctl(fd, VIDIOC_ENUM_FRAMESIZES, &frameSize)) { - if (frameSize.type != V4L2_FRMSIZE_TYPE_DISCRETE) - continue; - - QSize resolution(frameSize.discrete.width, frameSize.discrete.height); - float min = 1e10; - float max = 0; - - v4l2_frmivalenum frameInterval = {}; - frameInterval.pixel_format = formatDesc.pixelformat; - frameInterval.width = frameSize.discrete.width; - frameInterval.height = frameSize.discrete.height; - - while (!ioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &frameInterval)) { - if (frameInterval.type != V4L2_FRMIVAL_TYPE_DISCRETE) - continue; - ++frameInterval.index; - float rate = float(frameInterval.discrete.denominator)/float(frameInterval.discrete.numerator); - if (rate > max) - max = rate; - if (rate < min) - min = rate; - } - -// qDebug() << " " << resolution << min << max; - ++frameSize.index; - - if (min <= max) { - QCameraFormatPrivate *fmt = new QCameraFormatPrivate; - fmt->pixelFormat = pixelFmt; - fmt->resolution = resolution; - fmt->minFrameRate = min; - fmt->maxFrameRate = max; - camera->videoFormats.append(fmt->create()); - camera->photoResolutions.append(resolution); - } - } - - ++formatDesc.index; - } - - // first camera is default - camera->isDefault = first; - first = false; - - cameras.append(camera->create()); - - close(fd); - continue; - - fail: - if (camera) - delete camera; - close(fd); - } -} - -class QV4L2VideoBuffer : public QAbstractVideoBuffer -{ -public: - QV4L2VideoBuffer(QV4L2CameraBuffers *d, int index) - : QAbstractVideoBuffer(QVideoFrame::NoHandle, nullptr) - , index(index) - , d(d) - {} - ~QV4L2VideoBuffer() - { - d->release(index); - } - - QVideoFrame::MapMode mapMode() const override { return m_mode; } - MapData map(QVideoFrame::MapMode mode) override { - m_mode = mode; - return d->v4l2FileDescriptor >= 0 ? data : MapData{}; - } - void unmap() override { - m_mode = QVideoFrame::NotMapped; - } - - QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped; - MapData data; - int index = 0; - QExplicitlySharedDataPointer<QV4L2CameraBuffers> d; -}; - -QV4L2CameraBuffers::~QV4L2CameraBuffers() -{ - QMutexLocker locker(&mutex); - Q_ASSERT(v4l2FileDescriptor < 0); - unmapBuffers(); -} - - - -void QV4L2CameraBuffers::release(int index) -{ - QMutexLocker locker(&mutex); - if (v4l2FileDescriptor < 0 || index >= mappedBuffers.size()) - return; - - struct v4l2_buffer buf = {}; - - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf.memory = V4L2_MEMORY_MMAP; - buf.index = index; - - if (ioctl(v4l2FileDescriptor, VIDIOC_QBUF, &buf) < 0) - qWarning() << "Couldn't release V4L2 buffer" << errno << strerror(errno) << index; -} - -void QV4L2CameraBuffers::unmapBuffers() -{ - for (const auto &b : qAsConst(mappedBuffers)) - munmap(b.data, b.size); - mappedBuffers.clear(); -} - QV4L2Camera::QV4L2Camera(QCamera *camera) : QPlatformCamera(camera) { @@ -305,7 +73,6 @@ QV4L2Camera::QV4L2Camera(QCamera *camera) QV4L2Camera::~QV4L2Camera() { - setActive(false); stopCapturing(); closeV4L2Fd(); } @@ -326,13 +93,11 @@ void QV4L2Camera::setActive(bool active) resolveCameraFormat({}); m_active = active; - if (m_active) { - setV4L2CameraFormat(); - initMMap(); + if (m_active) startCapturing(); - } else { + else stopCapturing(); - } + emit newVideoFrame({}); emit activeChanged(active); @@ -342,9 +107,8 @@ void QV4L2Camera::setCamera(const QCameraDevice &camera) { if (m_cameraDevice == camera) return; - if (m_active) - stopCapturing(); + stopCapturing(); closeV4L2Fd(); m_cameraDevice = camera; @@ -352,11 +116,8 @@ void QV4L2Camera::setCamera(const QCameraDevice &camera) initV4L2Controls(); - if (m_active) { - setV4L2CameraFormat(); - initMMap(); + if (m_active) startCapturing(); - } } bool QV4L2Camera::setCameraFormat(const QCameraFormat &format) @@ -370,9 +131,8 @@ bool QV4L2Camera::setCameraFormat(const QCameraFormat &format) if (m_active) { stopCapturing(); closeV4L2Fd(); + initV4L2Controls(); - setV4L2CameraFormat(); - initMMap(); startCapturing(); } @@ -398,31 +158,31 @@ void QV4L2Camera::setFocusMode(QCamera::FocusMode mode) return; bool focusDist = supportedFeatures() & QCamera::Feature::FocusDistance; - if (!focusDist && !v4l2RangedFocus) + if (!focusDist && !m_v4l2Info.rangedFocus) return; switch (mode) { default: case QCamera::FocusModeAuto: setV4L2Parameter(V4L2_CID_FOCUS_AUTO, 1); - if (v4l2RangedFocus) + if (m_v4l2Info.rangedFocus) setV4L2Parameter(V4L2_CID_AUTO_FOCUS_RANGE, V4L2_AUTO_FOCUS_RANGE_AUTO); break; case QCamera::FocusModeAutoNear: setV4L2Parameter(V4L2_CID_FOCUS_AUTO, 1); - if (v4l2RangedFocus) + if (m_v4l2Info.rangedFocus) setV4L2Parameter(V4L2_CID_AUTO_FOCUS_RANGE, V4L2_AUTO_FOCUS_RANGE_MACRO); else if (focusDist) - setV4L2Parameter(V4L2_CID_FOCUS_ABSOLUTE, v4l2MinFocus); + setV4L2Parameter(V4L2_CID_FOCUS_ABSOLUTE, m_v4l2Info.minFocus); break; case QCamera::FocusModeAutoFar: setV4L2Parameter(V4L2_CID_FOCUS_AUTO, 1); - if (v4l2RangedFocus) + if (m_v4l2Info.rangedFocus) setV4L2Parameter(V4L2_CID_AUTO_FOCUS_RANGE, V4L2_AUTO_FOCUS_RANGE_INFINITY); break; case QCamera::FocusModeInfinity: setV4L2Parameter(V4L2_CID_FOCUS_AUTO, 0); - setV4L2Parameter(V4L2_CID_FOCUS_ABSOLUTE, v4l2MaxFocus); + setV4L2Parameter(V4L2_CID_FOCUS_ABSOLUTE, m_v4l2Info.maxFocus); break; case QCamera::FocusModeManual: setV4L2Parameter(V4L2_CID_FOCUS_AUTO, 0); @@ -434,17 +194,17 @@ void QV4L2Camera::setFocusMode(QCamera::FocusMode mode) void QV4L2Camera::setFocusDistance(float d) { - int distance = v4l2MinFocus + int((v4l2MaxFocus - v4l2MinFocus)*d); + int distance = m_v4l2Info.minFocus + int((m_v4l2Info.maxFocus - m_v4l2Info.minFocus) * d); setV4L2Parameter(V4L2_CID_FOCUS_ABSOLUTE, distance); focusDistanceChanged(d); } void QV4L2Camera::zoomTo(float factor, float) { - if (v4l2MaxZoom == v4l2MinZoom) + if (m_v4l2Info.maxZoom == m_v4l2Info.minZoom) return; factor = qBound(1., factor, 2.); - int zoom = v4l2MinZoom + (factor - 1.)*(v4l2MaxZoom - v4l2MinZoom); + int zoom = m_v4l2Info.minZoom + (factor - 1.) * (m_v4l2Info.maxZoom - m_v4l2Info.minZoom); setV4L2Parameter(V4L2_CID_ZOOM_ABSOLUTE, zoom); zoomFactorChanged(factor); } @@ -460,7 +220,7 @@ bool QV4L2Camera::isFocusModeSupported(QCamera::FocusMode mode) const void QV4L2Camera::setFlashMode(QCamera::FlashMode mode) { - if (!v4l2FlashSupported || mode == QCamera::FlashOn) + if (!m_v4l2Info.flashSupported || mode == QCamera::FlashOn) return; setV4L2Parameter(V4L2_CID_FLASH_LED_MODE, mode == QCamera::FlashAuto ? V4L2_FLASH_LED_MODE_FLASH : V4L2_FLASH_LED_MODE_NONE); flashModeChanged(mode); @@ -468,7 +228,7 @@ void QV4L2Camera::setFlashMode(QCamera::FlashMode mode) bool QV4L2Camera::isFlashModeSupported(QCamera::FlashMode mode) const { - if (v4l2FlashSupported && mode == QCamera::FlashAuto) + if (m_v4l2Info.flashSupported && mode == QCamera::FlashAuto) return true; return mode == QCamera::FlashOff; } @@ -479,15 +239,12 @@ bool QV4L2Camera::isFlashReady() const ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_AUTO_WHITE_BALANCE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) - return true; - - return false; + return m_v4l2FileDescriptor && m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl); } void QV4L2Camera::setTorchMode(QCamera::TorchMode mode) { - if (!v4l2TorchSupported || mode == QCamera::TorchOn) + if (!m_v4l2Info.torchSupported || mode == QCamera::TorchOn) return; setV4L2Parameter(V4L2_CID_FLASH_LED_MODE, mode == QCamera::TorchOn ? V4L2_FLASH_LED_MODE_TORCH : V4L2_FLASH_LED_MODE_NONE); torchModeChanged(mode); @@ -496,13 +253,13 @@ void QV4L2Camera::setTorchMode(QCamera::TorchMode mode) bool QV4L2Camera::isTorchModeSupported(QCamera::TorchMode mode) const { if (mode == QCamera::TorchOn) - return v4l2TorchSupported; + return m_v4l2Info.torchSupported; return mode == QCamera::TorchOff; } void QV4L2Camera::setExposureMode(QCamera::ExposureMode mode) { - if (v4l2AutoExposureSupported && v4l2ManualExposureSupported) { + if (m_v4l2Info.autoExposureSupported && m_v4l2Info.manualExposureSupported) { if (mode != QCamera::ExposureAuto && mode != QCamera::ExposureManual) return; int value = QCamera::ExposureAuto ? V4L2_EXPOSURE_AUTO : V4L2_EXPOSURE_MANUAL; @@ -516,15 +273,16 @@ bool QV4L2Camera::isExposureModeSupported(QCamera::ExposureMode mode) const { if (mode == QCamera::ExposureAuto) return true; - if (v4l2ManualExposureSupported && v4l2AutoExposureSupported) + if (m_v4l2Info.manualExposureSupported && m_v4l2Info.autoExposureSupported) return mode == QCamera::ExposureManual; return false; } void QV4L2Camera::setExposureCompensation(float compensation) { - if ((v4l2MinExposureAdjustment != 0 || v4l2MaxExposureAdjustment != 0)) { - int value = qBound(v4l2MinExposureAdjustment, (int)(compensation*1000), v4l2MaxExposureAdjustment); + if ((m_v4l2Info.minExposureAdjustment != 0 || m_v4l2Info.maxExposureAdjustment != 0)) { + int value = qBound(m_v4l2Info.minExposureAdjustment, (int)(compensation * 1000), + m_v4l2Info.maxExposureAdjustment); setV4L2Parameter(V4L2_CID_AUTO_EXPOSURE_BIAS, value); exposureCompensationChanged(value/1000.); return; @@ -552,8 +310,9 @@ int QV4L2Camera::isoSensitivity() const void QV4L2Camera::setManualExposureTime(float secs) { - if (v4l2ManualExposureSupported && v4l2AutoExposureSupported) { - int exposure = qBound(v4l2MinExposure, qRound(secs*10000.), v4l2MaxExposure); + if (m_v4l2Info.manualExposureSupported && m_v4l2Info.autoExposureSupported) { + int exposure = + qBound(m_v4l2Info.minExposure, qRound(secs * 10000.), m_v4l2Info.maxExposure); setV4L2Parameter(V4L2_CID_EXPOSURE_ABSOLUTE, exposure); exposureTimeChanged(exposure/10000.); return; @@ -567,7 +326,7 @@ float QV4L2Camera::exposureTime() const bool QV4L2Camera::isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const { - if (v4l2AutoWhiteBalanceSupported && v4l2ColorTemperatureSupported) + if (m_v4l2Info.autoWhiteBalanceSupported && m_v4l2Info.colorTemperatureSupported) return true; return mode == QCamera::WhiteBalanceAuto; @@ -600,127 +359,114 @@ void QV4L2Camera::setColorTemperature(int temperature) void QV4L2Camera::readFrame() { - if (!d) - return; + Q_ASSERT(m_memoryTransfer); - v4l2_buffer buf = {}; - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf.memory = V4L2_MEMORY_MMAP; + auto buffer = m_memoryTransfer->dequeueBuffer(); + if (!buffer) { + qCWarning(qLcV4L2Camera) << "Cannot take buffer"; - if (ioctl(d->v4l2FileDescriptor, VIDIOC_DQBUF, &buf) < 0) { if (errno == ENODEV) { // camera got removed while being active stopCapturing(); closeV4L2Fd(); - return; } - if (errno != EAGAIN) - qWarning() << "error calling VIDIOC_DQBUF" << errno << strerror(errno); + + return; } - Q_ASSERT(buf.index < d->mappedBuffers.size()); - int i = buf.index; - -// auto textureDesc = QVideoTextureHelper::textureDescription(m_format.pixelFormat()); - - QV4L2VideoBuffer *buffer = new QV4L2VideoBuffer(d.get(), i); - buffer->data.nPlanes = 1; - buffer->data.bytesPerLine[0] = bytesPerLine; - buffer->data.data[0] = (uchar *)d->mappedBuffers.at(i).data; - buffer->data.size[0] = d->mappedBuffers.at(i).size; - QVideoFrameFormat fmt(m_cameraFormat.resolution(), m_cameraFormat.pixelFormat()); - fmt.setColorSpace(colorSpace); -// qDebug() << "got a frame" << d->mappedBuffers.at(i).data << d->mappedBuffers.at(i).size << fmt << i; - QVideoFrame frame(buffer, fmt); - - if (firstFrameTime.tv_sec == -1) - firstFrameTime = buf.timestamp; - qint64 secs = buf.timestamp.tv_sec - firstFrameTime.tv_sec; - qint64 usecs = buf.timestamp.tv_usec - firstFrameTime.tv_usec; + auto videoBuffer = std::make_unique<QMemoryVideoBuffer>(buffer->data, m_bytesPerLine); + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(videoBuffer), frameFormat()); + + auto &v4l2Buffer = buffer->v4l2Buffer; + + if (m_firstFrameTime.tv_sec == -1) + m_firstFrameTime = v4l2Buffer.timestamp; + qint64 secs = v4l2Buffer.timestamp.tv_sec - m_firstFrameTime.tv_sec; + qint64 usecs = v4l2Buffer.timestamp.tv_usec - m_firstFrameTime.tv_usec; frame.setStartTime(secs*1000000 + usecs); - frame.setEndTime(frame.startTime() + frameDuration); + frame.setEndTime(frame.startTime() + m_frameDuration); emit newVideoFrame(frame); + + if (!m_memoryTransfer->enqueueBuffer(v4l2Buffer.index)) + qCWarning(qLcV4L2Camera) << "Cannot add buffer"; } void QV4L2Camera::setCameraBusy() { - cameraBusy = true; - error(QCamera::CameraError, tr("Camera is in use.")); + m_cameraBusy = true; + updateError(QCamera::CameraError, QLatin1String("Camera is in use")); } void QV4L2Camera::initV4L2Controls() { - v4l2AutoWhiteBalanceSupported = false; - v4l2ColorTemperatureSupported = false; - v4l2RangedFocus = false; - v4l2FlashSupported = false; - v4l2TorchSupported = false; + m_v4l2Info = {}; QCamera::Features features; - const QByteArray deviceName = m_cameraDevice.id(); Q_ASSERT(!deviceName.isEmpty()); closeV4L2Fd(); - Q_ASSERT(!d); - d = new QV4L2CameraBuffers; - - d->v4l2FileDescriptor = qt_safe_open(deviceName.constData(), O_RDWR); - if (d->v4l2FileDescriptor == -1) { - qWarning() << "Unable to open the camera" << deviceName - << "for read to query the parameter info:" << qt_error_string(errno); + const int descriptor = qt_safe_open(deviceName.constData(), O_RDWR); + if (descriptor == -1) { + qCWarning(qLcV4L2Camera) << "Unable to open the camera" << deviceName + << "for read to query the parameter info:" + << qt_error_string(errno); + updateError(QCamera::CameraError, QLatin1String("Cannot open camera")); return; } - qDebug() << "FD=" << d->v4l2FileDescriptor; + + m_v4l2FileDescriptor = std::make_shared<QV4L2FileDescriptor>(descriptor); + + qCDebug(qLcV4L2Camera) << "FD=" << descriptor; struct v4l2_queryctrl queryControl; ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_AUTO_WHITE_BALANCE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2AutoWhiteBalanceSupported = true; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.autoWhiteBalanceSupported = true; setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, true); } ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_WHITE_BALANCE_TEMPERATURE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinColorTemp = queryControl.minimum; - v4l2MaxColorTemp = queryControl.maximum; - v4l2ColorTemperatureSupported = true; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.minColorTemp = queryControl.minimum; + m_v4l2Info.maxColorTemp = queryControl.maximum; + m_v4l2Info.colorTemperatureSupported = true; features |= QCamera::Feature::ColorTemperature; } ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_EXPOSURE_AUTO; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2AutoExposureSupported = true; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.autoExposureSupported = true; } ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_EXPOSURE_ABSOLUTE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2ManualExposureSupported = true; - v4l2MinExposure = queryControl.minimum; - v4l2MaxExposure = queryControl.maximum; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.manualExposureSupported = true; + m_v4l2Info.minExposure = queryControl.minimum; + m_v4l2Info.maxExposure = queryControl.maximum; features |= QCamera::Feature::ManualExposureTime; } ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_AUTO_EXPOSURE_BIAS; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinExposureAdjustment = queryControl.minimum; - v4l2MaxExposureAdjustment = queryControl.maximum; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.minExposureAdjustment = queryControl.minimum; + m_v4l2Info.maxExposureAdjustment = queryControl.maximum; features |= QCamera::Feature::ExposureCompensation; } ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_ISO_SENSITIVITY_AUTO; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { queryControl.id = V4L2_CID_ISO_SENSITIVITY; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { features |= QCamera::Feature::IsoSensitivity; minIsoChanged(queryControl.minimum); maxIsoChanged(queryControl.minimum); @@ -729,50 +475,48 @@ void QV4L2Camera::initV4L2Controls() ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_FOCUS_ABSOLUTE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinExposureAdjustment = queryControl.minimum; - v4l2MaxExposureAdjustment = queryControl.maximum; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.minExposureAdjustment = queryControl.minimum; + m_v4l2Info.maxExposureAdjustment = queryControl.maximum; features |= QCamera::Feature::FocusDistance; } ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_AUTO_FOCUS_RANGE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2RangedFocus = true; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.rangedFocus = true; } ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_FLASH_LED_MODE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2FlashSupported = queryControl.minimum <= V4L2_FLASH_LED_MODE_FLASH && queryControl.maximum >= V4L2_FLASH_LED_MODE_FLASH; - v4l2TorchSupported = queryControl.minimum <= V4L2_FLASH_LED_MODE_TORCH && queryControl.maximum >= V4L2_FLASH_LED_MODE_TORCH; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.flashSupported = queryControl.minimum <= V4L2_FLASH_LED_MODE_FLASH + && queryControl.maximum >= V4L2_FLASH_LED_MODE_FLASH; + m_v4l2Info.torchSupported = queryControl.minimum <= V4L2_FLASH_LED_MODE_TORCH + && queryControl.maximum >= V4L2_FLASH_LED_MODE_TORCH; } - v4l2MinZoom = 0; - v4l2MaxZoom = 0; ::memset(&queryControl, 0, sizeof(queryControl)); queryControl.id = V4L2_CID_ZOOM_ABSOLUTE; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinZoom = queryControl.minimum; - v4l2MaxZoom = queryControl.maximum; + if (m_v4l2FileDescriptor->call(VIDIOC_QUERYCTRL, &queryControl)) { + m_v4l2Info.minZoom = queryControl.minimum; + m_v4l2Info.maxZoom = queryControl.maximum; } // zoom factors are in arbitrary units, so we simply normalize them to go from 1 to 2 // if they are different minimumZoomFactorChanged(1); - maximumZoomFactorChanged(v4l2MinZoom != v4l2MaxZoom ? 2 : 1); + maximumZoomFactorChanged(m_v4l2Info.minZoom != m_v4l2Info.maxZoom ? 2 : 1); supportedFeaturesChanged(features); } void QV4L2Camera::closeV4L2Fd() { - if (d && d->v4l2FileDescriptor >= 0) { - QMutexLocker locker(&d->mutex); - d->unmapBuffers(); - qt_safe_close(d->v4l2FileDescriptor); - d->v4l2FileDescriptor = -1; - } - d = nullptr; + Q_ASSERT(!m_memoryTransfer); + + m_v4l2Info = {}; + m_cameraBusy = false; + m_v4l2FileDescriptor = nullptr; } int QV4L2Camera::setV4L2ColorTemperature(int temperature) @@ -780,15 +524,17 @@ int QV4L2Camera::setV4L2ColorTemperature(int temperature) struct v4l2_control control; ::memset(&control, 0, sizeof(control)); - if (v4l2AutoWhiteBalanceSupported) { + if (m_v4l2Info.autoWhiteBalanceSupported) { setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, temperature == 0 ? true : false); } else if (temperature == 0) { temperature = 5600; } - if (temperature != 0 && v4l2ColorTemperatureSupported) { - temperature = qBound(v4l2MinColorTemp, temperature, v4l2MaxColorTemp); - if (!setV4L2Parameter(V4L2_CID_WHITE_BALANCE_TEMPERATURE, qBound(v4l2MinColorTemp, temperature, v4l2MaxColorTemp))) + if (temperature != 0 && m_v4l2Info.colorTemperatureSupported) { + temperature = qBound(m_v4l2Info.minColorTemp, temperature, m_v4l2Info.maxColorTemp); + if (!setV4L2Parameter( + V4L2_CID_WHITE_BALANCE_TEMPERATURE, + qBound(m_v4l2Info.minColorTemp, temperature, m_v4l2Info.maxColorTemp))) temperature = 0; } else { temperature = 0; @@ -799,8 +545,8 @@ int QV4L2Camera::setV4L2ColorTemperature(int temperature) bool QV4L2Camera::setV4L2Parameter(quint32 id, qint32 value) { - struct v4l2_control control{id, value}; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_S_CTRL, &control) != 0) { + v4l2_control control{ id, value }; + if (!m_v4l2FileDescriptor->call(VIDIOC_S_CTRL, &control)) { qWarning() << "Unable to set the V4L2 Parameter" << Qt::hex << id << "to" << value << qt_error_string(errno); return false; } @@ -810,7 +556,7 @@ bool QV4L2Camera::setV4L2Parameter(quint32 id, qint32 value) int QV4L2Camera::getV4L2Parameter(quint32 id) const { struct v4l2_control control{id, 0}; - if (::ioctl(d->v4l2FileDescriptor, VIDIOC_G_CTRL, &control) != 0) { + if (!m_v4l2FileDescriptor->call(VIDIOC_G_CTRL, &control)) { qWarning() << "Unable to get the V4L2 Parameter" << Qt::hex << id << qt_error_string(errno); return 0; } @@ -819,8 +565,12 @@ int QV4L2Camera::getV4L2Parameter(quint32 id) const void QV4L2Camera::setV4L2CameraFormat() { + if (m_v4l2Info.formatInitialized || !m_v4l2FileDescriptor) + return; + Q_ASSERT(!m_cameraFormat.isNull()); - qDebug() << "XXXXX" << this << m_cameraDevice.id() << m_cameraFormat.pixelFormat() << m_cameraFormat.resolution(); + qCDebug(qLcV4L2Camera) << "XXXXX" << this << m_cameraDevice.id() << m_cameraFormat.pixelFormat() + << m_cameraFormat.resolution(); v4l2_format fmt = {}; fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; @@ -831,9 +581,9 @@ void QV4L2Camera::setV4L2CameraFormat() fmt.fmt.pix.pixelformat = v4l2FormatForPixelFormat(m_cameraFormat.pixelFormat()); fmt.fmt.pix.field = V4L2_FIELD_ANY; - qDebug() << "setting camera format to" << size; + qCDebug(qLcV4L2Camera) << "setting camera format to" << size << fmt.fmt.pix.pixelformat; - if (ioctl(d->v4l2FileDescriptor, VIDIOC_S_FMT, &fmt) < 0) { + if (!m_v4l2FileDescriptor->call(VIDIOC_S_FMT, &fmt)) { if (errno == EBUSY) { setCameraBusy(); return; @@ -841,25 +591,29 @@ void QV4L2Camera::setV4L2CameraFormat() qWarning() << "Couldn't set video format on v4l2 camera" << strerror(errno); } - bytesPerLine = fmt.fmt.pix.bytesperline; + m_v4l2Info.formatInitialized = true; + m_cameraBusy = false; + + m_bytesPerLine = fmt.fmt.pix.bytesperline; + m_imageSize = std::max(fmt.fmt.pix.sizeimage, m_bytesPerLine * fmt.fmt.pix.height); switch (v4l2_colorspace(fmt.fmt.pix.colorspace)) { default: case V4L2_COLORSPACE_DCI_P3: - colorSpace = QVideoFrameFormat::ColorSpace_Undefined; + m_colorSpace = QVideoFrameFormat::ColorSpace_Undefined; break; case V4L2_COLORSPACE_REC709: - colorSpace = QVideoFrameFormat::ColorSpace_BT709; + m_colorSpace = QVideoFrameFormat::ColorSpace_BT709; break; case V4L2_COLORSPACE_JPEG: - colorSpace = QVideoFrameFormat::ColorSpace_AdobeRgb; + m_colorSpace = QVideoFrameFormat::ColorSpace_AdobeRgb; break; case V4L2_COLORSPACE_SRGB: // ##### is this correct??? - colorSpace = QVideoFrameFormat::ColorSpace_BT601; + m_colorSpace = QVideoFrameFormat::ColorSpace_BT601; break; case V4L2_COLORSPACE_BT2020: - colorSpace = QVideoFrameFormat::ColorSpace_BT2020; + m_colorSpace = QVideoFrameFormat::ColorSpace_BT2020; break; } @@ -867,107 +621,88 @@ void QV4L2Camera::setV4L2CameraFormat() streamParam.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; streamParam.parm.capture.capability = V4L2_CAP_TIMEPERFRAME; - int num, den; - qt_real_to_fraction(1./m_cameraFormat.maxFrameRate(), &num, &den); + auto [num, den] = qRealToFraction(1./m_cameraFormat.maxFrameRate()); streamParam.parm.capture.timeperframe = { (uint)num, (uint)den }; - ioctl(d->v4l2FileDescriptor, VIDIOC_S_PARM, &streamParam); + m_v4l2FileDescriptor->call(VIDIOC_S_PARM, &streamParam); - frameDuration = 1000000*streamParam.parm.capture.timeperframe.numerator - /streamParam.parm.capture.timeperframe.denominator; + m_frameDuration = 1000000 * streamParam.parm.capture.timeperframe.numerator + / streamParam.parm.capture.timeperframe.denominator; } -void QV4L2Camera::initMMap() +void QV4L2Camera::initV4L2MemoryTransfer() { - if (cameraBusy) + if (m_cameraBusy) return; - v4l2_requestbuffers req = {}; - req.count = 4; - req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - req.memory = V4L2_MEMORY_MMAP; + Q_ASSERT(!m_memoryTransfer); - if (ioctl(d->v4l2FileDescriptor, VIDIOC_REQBUFS, &req) < 0) { - if (errno == EBUSY) - setCameraBusy(); - qWarning() << "requesting mmap'ed buffers failed" << strerror(errno); + m_memoryTransfer = makeUserPtrMemoryTransfer(m_v4l2FileDescriptor, m_imageSize); + + if (m_memoryTransfer) return; - } - if (req.count < 2) { - qWarning() << "Can't map 2 or more buffers"; + if (errno == EBUSY) { + setCameraBusy(); return; } - for (uint32_t n = 0; n < req.count; ++n) { - v4l2_buffer buf = {}; - buf.index = n; - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf.memory = V4L2_MEMORY_MMAP; - - if (ioctl(d->v4l2FileDescriptor, VIDIOC_QUERYBUF, &buf) != 0) { - qWarning() << "Can't map buffer" << n; - return; - } - - QV4L2CameraBuffers::MappedBuffer buffer; - buffer.size = buf.length; - buffer.data = mmap(NULL, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, - d->v4l2FileDescriptor, buf.m.offset); + qCDebug(qLcV4L2Camera) << "Cannot init V4L2_MEMORY_USERPTR; trying V4L2_MEMORY_MMAP"; - if (buffer.data == MAP_FAILED) { - qWarning() << "mmap failed" << n << buf.length << buf.m.offset; - return; - } + m_memoryTransfer = makeMMapMemoryTransfer(m_v4l2FileDescriptor); - d->mappedBuffers.append(buffer); + if (!m_memoryTransfer) { + qCWarning(qLcV4L2Camera) << "Cannot init v4l2 memory transfer," << qt_error_string(errno); + updateError(QCamera::CameraError, QLatin1String("Cannot init V4L2 memory transfer")); } - } void QV4L2Camera::stopCapturing() { - if (!d) + if (!m_memoryTransfer || !m_v4l2FileDescriptor) return; - delete notifier; - notifier = nullptr; + m_notifier = nullptr; - enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - - if (ioctl(d->v4l2FileDescriptor, VIDIOC_STREAMOFF, &type) < 0) { + if (!m_v4l2FileDescriptor->stopStream()) { + // TODO: handle the case carefully to avoid possible memory corruption if (errno != ENODEV) qWarning() << "failed to stop capture"; } - cameraBusy = false; + + m_memoryTransfer = nullptr; + m_cameraBusy = false; } void QV4L2Camera::startCapturing() { - if (cameraBusy) + if (!m_v4l2FileDescriptor) return; - // #### better to use the user data method instead of mmap??? - unsigned int i; + setV4L2CameraFormat(); + initV4L2MemoryTransfer(); - for (i = 0; i < d->mappedBuffers.size(); ++i) { - v4l2_buffer buf = {}; - buf.index = i; - buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - buf.memory = V4L2_MEMORY_MMAP; + if (m_cameraBusy || !m_memoryTransfer) + return; - if (ioctl(d->v4l2FileDescriptor, VIDIOC_QBUF, &buf) < 0) { - qWarning() << "failed to setup mapped buffer"; - return; - } + if (!m_v4l2FileDescriptor->startStream()) { + qWarning() << "Couldn't start v4l2 camera stream"; + return; } - enum v4l2_buf_type type = V4L2_BUF_TYPE_VIDEO_CAPTURE; - if (ioctl(d->v4l2FileDescriptor, VIDIOC_STREAMON, &type) < 0) - qWarning() << "failed to start capture"; - notifier = new QSocketNotifier(d->v4l2FileDescriptor, QSocketNotifier::Read); - connect(notifier, &QSocketNotifier::activated, this, &QV4L2Camera::readFrame); + m_notifier = + std::make_unique<QSocketNotifier>(m_v4l2FileDescriptor->get(), QSocketNotifier::Read); + connect(m_notifier.get(), &QSocketNotifier::activated, this, &QV4L2Camera::readFrame); - firstFrameTime = { -1, -1 }; + m_firstFrameTime = { -1, -1 }; +} + +QVideoFrameFormat QV4L2Camera::frameFormat() const +{ + auto result = QPlatformCamera::frameFormat(); + result.setColorSpace(m_colorSpace); + return result; } QT_END_NAMESPACE + +#include "moc_qv4l2camera_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qv4l2camera_p.h b/src/plugins/multimedia/ffmpeg/qv4l2camera_p.h index bc94df8d8..3033f5ff9 100644 --- a/src/plugins/multimedia/ffmpeg/qv4l2camera_p.h +++ b/src/plugins/multimedia/ffmpeg/qv4l2camera_p.h @@ -1,44 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QFFMPEGCAMERA_H -#define QFFMPEGCAMERA_H +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QV4L2CAMERA_H +#define QV4L2CAMERA_H // // W A R N I N G @@ -52,53 +16,42 @@ // #include <private/qplatformcamera_p.h> -#include <private/qplatformvideodevices_p.h> -#include <private/qplatformmediaintegration_p.h> - -#include <qfilesystemwatcher.h> -#include <qsocketnotifier.h> -#include <qmutex.h> +#include <sys/time.h> QT_BEGIN_NAMESPACE -class QV4L2CameraDevices : public QObject, - public QPlatformVideoDevices -{ - Q_OBJECT -public: - QV4L2CameraDevices(QPlatformMediaIntegration *integration); - - QList<QCameraDevice> videoDevices() const override; - -public Q_SLOTS: - void checkCameras(); +class QV4L2FileDescriptor; +class QV4L2MemoryTransfer; +class QSocketNotifier; -private: - void doCheckCameras(); - - QList<QCameraDevice> cameras; - QFileSystemWatcher deviceWatcher; -}; - -struct QV4L2CameraBuffers +struct V4L2CameraInfo { -public: - ~QV4L2CameraBuffers(); - - void release(int index); - void unmapBuffers(); - - QAtomicInt ref; - QMutex mutex; - struct MappedBuffer { - void *data; - qsizetype size; - }; - QList<MappedBuffer> mappedBuffers; - int v4l2FileDescriptor = -1; + bool formatInitialized = false; + + bool autoWhiteBalanceSupported = false; + bool colorTemperatureSupported = false; + bool autoExposureSupported = false; + bool manualExposureSupported = false; + bool flashSupported = false; + bool torchSupported = false; + qint32 minColorTemp = 5600; // Daylight... + qint32 maxColorTemp = 5600; + qint32 minExposure = 0; + qint32 maxExposure = 0; + qint32 minExposureAdjustment = 0; + qint32 maxExposureAdjustment = 0; + qint32 minFocus = 0; + qint32 maxFocus = 0; + qint32 rangedFocus = false; + + int minZoom = 0; + int maxZoom = 0; }; -class Q_MULTIMEDIA_EXPORT QV4L2Camera : public QPlatformCamera +QVideoFrameFormat::PixelFormat formatForV4L2Format(uint32_t v4l2Format); +uint32_t v4l2FormatForPixelFormat(QVideoFrameFormat::PixelFormat format); + +class QV4L2Camera : public QPlatformCamera { Q_OBJECT @@ -139,18 +92,13 @@ public: void setWhiteBalanceMode(QCamera::WhiteBalanceMode /*mode*/) override; void setColorTemperature(int /*temperature*/) override; - void releaseBuffer(int index); + QVideoFrameFormat frameFormat() const override; private Q_SLOTS: void readFrame(); private: void setCameraBusy(); - - bool m_active = false; - - QCameraDevice m_cameraDevice; - void initV4L2Controls(); void closeV4L2Fd(); int setV4L2ColorTemperature(int temperature); @@ -158,39 +106,28 @@ private: int getV4L2Parameter(quint32 id) const; void setV4L2CameraFormat(); - void initMMap(); + void initV4L2MemoryTransfer(); void startCapturing(); void stopCapturing(); - QSocketNotifier *notifier = nullptr; - QExplicitlySharedDataPointer<QV4L2CameraBuffers> d; - - bool v4l2AutoWhiteBalanceSupported = false; - bool v4l2ColorTemperatureSupported = false; - bool v4l2AutoExposureSupported = false; - bool v4l2ManualExposureSupported = false; - qint32 v4l2MinColorTemp = 5600; // Daylight... - qint32 v4l2MaxColorTemp = 5600; - qint32 v4l2MinExposure = 0; - qint32 v4l2MaxExposure = 0; - qint32 v4l2MinExposureAdjustment = 0; - qint32 v4l2MaxExposureAdjustment = 0; - qint32 v4l2MinFocus = 0; - qint32 v4l2MaxFocus = 0; - qint32 v4l2RangedFocus = false; - bool v4l2FlashSupported = false; - bool v4l2TorchSupported = false; - int v4l2MinZoom = 0; - int v4l2MaxZoom = 0; - timeval firstFrameTime = {-1, -1}; - int bytesPerLine = -1; - QVideoFrameFormat::ColorSpace colorSpace = QVideoFrameFormat::ColorSpace_Undefined; - qint64 frameDuration = -1; - bool cameraBusy = false; -}; +private: + bool m_active = false; + QCameraDevice m_cameraDevice; -QT_END_NAMESPACE + std::unique_ptr<QSocketNotifier> m_notifier; + std::unique_ptr<QV4L2MemoryTransfer> m_memoryTransfer; + std::shared_ptr<QV4L2FileDescriptor> m_v4l2FileDescriptor; + V4L2CameraInfo m_v4l2Info; -#endif // QFFMPEGCAMERA_H + timeval m_firstFrameTime = { -1, -1 }; + quint32 m_bytesPerLine = 0; + quint32 m_imageSize = 0; + QVideoFrameFormat::ColorSpace m_colorSpace = QVideoFrameFormat::ColorSpace_Undefined; + qint64 m_frameDuration = -1; + bool m_cameraBusy = false; +}; + +QT_END_NAMESPACE +#endif // QV4L2CAMERA_H diff --git a/src/plugins/multimedia/ffmpeg/qv4l2cameradevices.cpp b/src/plugins/multimedia/ffmpeg/qv4l2cameradevices.cpp new file mode 100644 index 000000000..82a9658c7 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qv4l2cameradevices.cpp @@ -0,0 +1,182 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qv4l2cameradevices_p.h" +#include "qv4l2filedescriptor_p.h" +#include "qv4l2camera_p.h" + +#include <private/qcameradevice_p.h> +#include <private/qcore_unix_p.h> + +#include <qdir.h> +#include <qfile.h> +#include <qdebug.h> +#include <qloggingcategory.h> + +#include <linux/videodev2.h> + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcV4L2CameraDevices, "qt.multimedia.ffmpeg.v4l2cameradevices"); + +static bool areCamerasEqual(QList<QCameraDevice> a, QList<QCameraDevice> b) +{ + auto areCamerasDataEqual = [](const QCameraDevice &a, const QCameraDevice &b) { + Q_ASSERT(QCameraDevicePrivate::handle(a)); + Q_ASSERT(QCameraDevicePrivate::handle(b)); + return *QCameraDevicePrivate::handle(a) == *QCameraDevicePrivate::handle(b); + }; + + return std::equal(a.cbegin(), a.cend(), b.cbegin(), b.cend(), areCamerasDataEqual); +} + +QV4L2CameraDevices::QV4L2CameraDevices(QPlatformMediaIntegration *integration) + : QPlatformVideoDevices(integration) +{ + m_deviceWatcher.addPath(QLatin1String("/dev")); + connect(&m_deviceWatcher, &QFileSystemWatcher::directoryChanged, this, + &QV4L2CameraDevices::checkCameras); + doCheckCameras(); +} + +QList<QCameraDevice> QV4L2CameraDevices::videoDevices() const +{ + return m_cameras; +} + +void QV4L2CameraDevices::checkCameras() +{ + if (doCheckCameras()) + emit videoInputsChanged(); +} + +bool QV4L2CameraDevices::doCheckCameras() +{ + QList<QCameraDevice> newCameras; + + QDir dir(QLatin1String("/dev")); + const auto devices = dir.entryList(QDir::System); + + bool first = true; + + for (auto device : devices) { + // qCDebug(qLcV4L2Camera) << "device:" << device; + if (!device.startsWith(QLatin1String("video"))) + continue; + + QByteArray file = QFile::encodeName(dir.filePath(device)); + const int fd = open(file.constData(), O_RDONLY); + if (fd < 0) + continue; + + auto fileCloseGuard = qScopeGuard([fd]() { close(fd); }); + + v4l2_fmtdesc formatDesc = {}; + + struct v4l2_capability cap; + if (xioctl(fd, VIDIOC_QUERYCAP, &cap) < 0) + continue; + + if (cap.device_caps & V4L2_CAP_META_CAPTURE) + continue; + if (!(cap.capabilities & V4L2_CAP_VIDEO_CAPTURE)) + continue; + if (!(cap.capabilities & V4L2_CAP_STREAMING)) + continue; + + auto camera = std::make_unique<QCameraDevicePrivate>(); + + camera->id = file; + camera->description = QString::fromUtf8((const char *)cap.card); + qCDebug(qLcV4L2CameraDevices) << "found camera" << camera->id << camera->description; + + formatDesc.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + + while (!xioctl(fd, VIDIOC_ENUM_FMT, &formatDesc)) { + auto pixelFmt = formatForV4L2Format(formatDesc.pixelformat); + qCDebug(qLcV4L2CameraDevices) << " " << pixelFmt; + + if (pixelFmt == QVideoFrameFormat::Format_Invalid) { + ++formatDesc.index; + continue; + } + + qCDebug(qLcV4L2CameraDevices) << "frame sizes:"; + v4l2_frmsizeenum frameSize = {}; + frameSize.pixel_format = formatDesc.pixelformat; + + while (!xioctl(fd, VIDIOC_ENUM_FRAMESIZES, &frameSize)) { + QList<QSize> resolutions; + if (frameSize.type == V4L2_FRMSIZE_TYPE_DISCRETE) { + resolutions.append(QSize(frameSize.discrete.width, + frameSize.discrete.height)); + } else { + resolutions.append(QSize(frameSize.stepwise.max_width, + frameSize.stepwise.max_height)); + resolutions.append(QSize(frameSize.stepwise.min_width, + frameSize.stepwise.min_height)); + } + + for (auto resolution : resolutions) { + float min = 1e10; + float max = 0; + auto updateMaxMinFrameRate = [&max, &min](auto discreteFrameRate) { + const float rate = float(discreteFrameRate.denominator) + / float(discreteFrameRate.numerator); + if (rate > max) + max = rate; + if (rate < min) + min = rate; + }; + + v4l2_frmivalenum frameInterval = {}; + frameInterval.pixel_format = formatDesc.pixelformat; + frameInterval.width = resolution.width(); + frameInterval.height = resolution.height(); + + while (!xioctl(fd, VIDIOC_ENUM_FRAMEINTERVALS, &frameInterval)) { + if (frameInterval.type == V4L2_FRMIVAL_TYPE_DISCRETE) { + updateMaxMinFrameRate(frameInterval.discrete); + } else { + updateMaxMinFrameRate(frameInterval.stepwise.max); + updateMaxMinFrameRate(frameInterval.stepwise.min); + } + ++frameInterval.index; + } + + qCDebug(qLcV4L2CameraDevices) << " " << resolution << min << max; + + if (min <= max) { + auto fmt = std::make_unique<QCameraFormatPrivate>(); + fmt->pixelFormat = pixelFmt; + fmt->resolution = resolution; + fmt->minFrameRate = min; + fmt->maxFrameRate = max; + camera->videoFormats.append(fmt.release()->create()); + camera->photoResolutions.append(resolution); + } + } + ++frameSize.index; + } + ++formatDesc.index; + } + + if (camera->videoFormats.empty()) + continue; + + // first camera is default + camera->isDefault = std::exchange(first, false); + + newCameras.append(camera.release()->create()); + } + + if (areCamerasEqual(m_cameras, newCameras)) + return false; + + m_cameras = std::move(newCameras); + return true; +} + +QT_END_NAMESPACE + +#include "moc_qv4l2cameradevices_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/qv4l2cameradevices_p.h b/src/plugins/multimedia/ffmpeg/qv4l2cameradevices_p.h new file mode 100644 index 000000000..ce424d3b6 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qv4l2cameradevices_p.h @@ -0,0 +1,46 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QV4L2CAMERADEVICES_P_H +#define QV4L2CAMERADEVICES_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformvideodevices_p.h> +#include <private/qplatformmediaintegration_p.h> + +#include <qfilesystemwatcher.h> + +QT_BEGIN_NAMESPACE + +class QV4L2CameraDevices : public QPlatformVideoDevices +{ + Q_OBJECT +public: + QV4L2CameraDevices(QPlatformMediaIntegration *integration); + + QList<QCameraDevice> videoDevices() const override; + +public Q_SLOTS: + void checkCameras(); + +private: + bool doCheckCameras(); + +private: + QList<QCameraDevice> m_cameras; + QFileSystemWatcher m_deviceWatcher; +}; + +QT_END_NAMESPACE + +#endif // QV4L2CAMERADEVICES_P_H diff --git a/src/plugins/multimedia/ffmpeg/qv4l2filedescriptor.cpp b/src/plugins/multimedia/ffmpeg/qv4l2filedescriptor.cpp new file mode 100644 index 000000000..7f7b099c7 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qv4l2filedescriptor.cpp @@ -0,0 +1,71 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qv4l2filedescriptor_p.h" + +#include <sys/ioctl.h> +#include <private/qcore_unix_p.h> + +#include <linux/videodev2.h> + +QT_BEGIN_NAMESPACE + +int xioctl(int fd, int request, void *arg) +{ + int res; + + do { + res = ::ioctl(fd, request, arg); + } while (res == -1 && EINTR == errno); + + return res; +} + +QV4L2FileDescriptor::QV4L2FileDescriptor(int descriptor) : m_descriptor(descriptor) +{ + Q_ASSERT(descriptor >= 0); +} + +QV4L2FileDescriptor::~QV4L2FileDescriptor() +{ + qt_safe_close(m_descriptor); +} + +bool QV4L2FileDescriptor::call(int request, void *arg) const +{ + return ::xioctl(m_descriptor, request, arg) >= 0; +} + +bool QV4L2FileDescriptor::requestBuffers(quint32 memoryType, quint32 &buffersCount) const +{ + v4l2_requestbuffers req = {}; + req.count = buffersCount; + req.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + req.memory = memoryType; + + if (!call(VIDIOC_REQBUFS, &req)) + return false; + + buffersCount = req.count; + return true; +} + +bool QV4L2FileDescriptor::startStream() +{ + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + if (!call(VIDIOC_STREAMON, &type)) + return false; + + m_streamStarted = true; + return true; +} + +bool QV4L2FileDescriptor::stopStream() +{ + int type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + auto result = call(VIDIOC_STREAMOFF, &type); + m_streamStarted = false; + return result; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qv4l2filedescriptor_p.h b/src/plugins/multimedia/ffmpeg/qv4l2filedescriptor_p.h new file mode 100644 index 000000000..1058c7a82 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qv4l2filedescriptor_p.h @@ -0,0 +1,50 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QV4L2FILEDESCRIPTOR_P_H +#define QV4L2FILEDESCRIPTOR_P_H + +#include <private/qtmultimediaglobal_p.h> + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +QT_BEGIN_NAMESPACE + +int xioctl(int fd, int request, void *arg); + +class QV4L2FileDescriptor +{ +public: + QV4L2FileDescriptor(int descriptor); + + ~QV4L2FileDescriptor(); + + bool call(int request, void *arg) const; + + int get() const { return m_descriptor; } + + bool requestBuffers(quint32 memoryType, quint32 &buffersCount) const; + + bool startStream(); + + bool stopStream(); + + bool streamStarted() const { return m_streamStarted; } + +private: + int m_descriptor; + bool m_streamStarted = false; +}; + +QT_END_NAMESPACE + +#endif // QV4L2FILEDESCRIPTOR_P_H diff --git a/src/plugins/multimedia/ffmpeg/qv4l2memorytransfer.cpp b/src/plugins/multimedia/ffmpeg/qv4l2memorytransfer.cpp new file mode 100644 index 000000000..a2873235a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qv4l2memorytransfer.cpp @@ -0,0 +1,223 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qv4l2memorytransfer_p.h" +#include "qv4l2filedescriptor_p.h" + +#include <qloggingcategory.h> +#include <qdebug.h> +#include <sys/mman.h> +#include <optional> + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcV4L2MemoryTransfer, "qt.multimedia.ffmpeg.v4l2camera.memorytransfer"); + +namespace { + +v4l2_buffer makeV4l2Buffer(quint32 memoryType, quint32 index = 0) +{ + v4l2_buffer buf = {}; + buf.type = V4L2_BUF_TYPE_VIDEO_CAPTURE; + buf.memory = memoryType; + buf.index = index; + return buf; +} + +class UserPtrMemoryTransfer : public QV4L2MemoryTransfer +{ +public: + static QV4L2MemoryTransferUPtr create(QV4L2FileDescriptorPtr fileDescriptor, quint32 imageSize) + { + quint32 buffersCount = 2; + if (!fileDescriptor->requestBuffers(V4L2_MEMORY_USERPTR, buffersCount)) { + qCWarning(qLcV4L2MemoryTransfer) << "Cannot request V4L2_MEMORY_USERPTR buffers"; + return {}; + } + + std::unique_ptr<UserPtrMemoryTransfer> result( + new UserPtrMemoryTransfer(std::move(fileDescriptor), buffersCount, imageSize)); + + return result->enqueueBuffers() ? std::move(result) : nullptr; + } + + std::optional<Buffer> dequeueBuffer() override + { + auto v4l2Buffer = makeV4l2Buffer(V4L2_MEMORY_USERPTR); + if (!fileDescriptor().call(VIDIOC_DQBUF, &v4l2Buffer)) + return {}; + + Q_ASSERT(v4l2Buffer.index < m_byteArrays.size()); + Q_ASSERT(!m_byteArrays[v4l2Buffer.index].isEmpty()); + + return Buffer{ v4l2Buffer, std::move(m_byteArrays[v4l2Buffer.index]) }; + } + + bool enqueueBuffer(quint32 index) override + { + Q_ASSERT(index < m_byteArrays.size()); + Q_ASSERT(m_byteArrays[index].isEmpty()); + + auto buf = makeV4l2Buffer(V4L2_MEMORY_USERPTR, index); + static_assert(sizeof(decltype(buf.m.userptr)) == sizeof(size_t), "Not compatible sizes"); + + m_byteArrays[index] = QByteArray(static_cast<int>(m_imageSize), Qt::Uninitialized); + + buf.m.userptr = (decltype(buf.m.userptr))m_byteArrays[index].data(); + buf.length = m_byteArrays[index].size(); + + if (!fileDescriptor().call(VIDIOC_QBUF, &buf)) { + qWarning() << "Couldn't add V4L2 buffer" << errno << strerror(errno) << index; + return false; + } + + return true; + } + + quint32 buffersCount() const override { return static_cast<quint32>(m_byteArrays.size()); } + +private: + UserPtrMemoryTransfer(QV4L2FileDescriptorPtr fileDescriptor, quint32 buffersCount, + quint32 imageSize) + : QV4L2MemoryTransfer(std::move(fileDescriptor)), + m_imageSize(imageSize), + m_byteArrays(buffersCount) + { + } + +private: + quint32 m_imageSize; + std::vector<QByteArray> m_byteArrays; +}; + +class MMapMemoryTransfer : public QV4L2MemoryTransfer +{ +public: + struct MemorySpan + { + void *data = nullptr; + size_t size = 0; + bool inQueue = false; + }; + + static QV4L2MemoryTransferUPtr create(QV4L2FileDescriptorPtr fileDescriptor) + { + quint32 buffersCount = 2; + if (!fileDescriptor->requestBuffers(V4L2_MEMORY_MMAP, buffersCount)) { + qCWarning(qLcV4L2MemoryTransfer) << "Cannot request V4L2_MEMORY_MMAP buffers"; + return {}; + } + + std::unique_ptr<MMapMemoryTransfer> result( + new MMapMemoryTransfer(std::move(fileDescriptor))); + + return result->init(buffersCount) ? std::move(result) : nullptr; + } + + bool init(quint32 buffersCount) + { + for (quint32 index = 0; index < buffersCount; ++index) { + auto buf = makeV4l2Buffer(V4L2_MEMORY_MMAP, index); + + if (!fileDescriptor().call(VIDIOC_QUERYBUF, &buf)) { + qWarning() << "Can't map buffer" << index; + return false; + } + + auto mappedData = mmap(nullptr, buf.length, PROT_READ | PROT_WRITE, MAP_SHARED, + fileDescriptor().get(), buf.m.offset); + + if (mappedData == MAP_FAILED) { + qWarning() << "mmap failed" << index << buf.length << buf.m.offset; + return false; + } + + m_spans.push_back(MemorySpan{ mappedData, buf.length, false }); + } + + m_spans.shrink_to_fit(); + + return enqueueBuffers(); + } + + ~MMapMemoryTransfer() override + { + for (const auto &span : m_spans) + munmap(span.data, span.size); + } + + std::optional<Buffer> dequeueBuffer() override + { + auto v4l2Buffer = makeV4l2Buffer(V4L2_MEMORY_MMAP); + if (!fileDescriptor().call(VIDIOC_DQBUF, &v4l2Buffer)) + return {}; + + const auto index = v4l2Buffer.index; + + Q_ASSERT(index < m_spans.size()); + + auto &span = m_spans[index]; + + Q_ASSERT(span.inQueue); + span.inQueue = false; + + return Buffer{ v4l2Buffer, + QByteArray(reinterpret_cast<const char *>(span.data), span.size) }; + } + + bool enqueueBuffer(quint32 index) override + { + Q_ASSERT(index < m_spans.size()); + Q_ASSERT(!m_spans[index].inQueue); + + auto buf = makeV4l2Buffer(V4L2_MEMORY_MMAP, index); + if (!fileDescriptor().call(VIDIOC_QBUF, &buf)) + return false; + + m_spans[index].inQueue = true; + return true; + } + + quint32 buffersCount() const override { return static_cast<quint32>(m_spans.size()); } + +private: + using QV4L2MemoryTransfer::QV4L2MemoryTransfer; + +private: + std::vector<MemorySpan> m_spans; +}; +} // namespace + +QV4L2MemoryTransfer::QV4L2MemoryTransfer(QV4L2FileDescriptorPtr fileDescriptor) + : m_fileDescriptor(std::move(fileDescriptor)) +{ + Q_ASSERT(m_fileDescriptor); + Q_ASSERT(!m_fileDescriptor->streamStarted()); +} + +QV4L2MemoryTransfer::~QV4L2MemoryTransfer() +{ + Q_ASSERT(!m_fileDescriptor->streamStarted()); // to avoid possible corruptions +} + +bool QV4L2MemoryTransfer::enqueueBuffers() +{ + for (quint32 i = 0; i < buffersCount(); ++i) + if (!enqueueBuffer(i)) + return false; + + return true; +} + +QV4L2MemoryTransferUPtr makeUserPtrMemoryTransfer(QV4L2FileDescriptorPtr fileDescriptor, + quint32 imageSize) +{ + return UserPtrMemoryTransfer::create(std::move(fileDescriptor), imageSize); +} + +QV4L2MemoryTransferUPtr makeMMapMemoryTransfer(QV4L2FileDescriptorPtr fileDescriptor) +{ + return MMapMemoryTransfer::create(std::move(fileDescriptor)); +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qv4l2memorytransfer_p.h b/src/plugins/multimedia/ffmpeg/qv4l2memorytransfer_p.h new file mode 100644 index 000000000..6b5e3913f --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qv4l2memorytransfer_p.h @@ -0,0 +1,66 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QV4L2MEMORYTRANSFER_P_H +#define QV4L2MEMORYTRANSFER_P_H + +#include <private/qtmultimediaglobal_p.h> +#include <qbytearray.h> +#include <linux/videodev2.h> + +#include <memory> + +QT_BEGIN_NAMESPACE + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +class QV4L2FileDescriptor; +using QV4L2FileDescriptorPtr = std::shared_ptr<QV4L2FileDescriptor>; + +class QV4L2MemoryTransfer +{ +public: + struct Buffer + { + v4l2_buffer v4l2Buffer = {}; + QByteArray data; + }; + + QV4L2MemoryTransfer(QV4L2FileDescriptorPtr fileDescriptor); + + virtual ~QV4L2MemoryTransfer(); + + virtual std::optional<Buffer> dequeueBuffer() = 0; + + virtual bool enqueueBuffer(quint32 index) = 0; + + virtual quint32 buffersCount() const = 0; + +protected: + bool enqueueBuffers(); + + const QV4L2FileDescriptor &fileDescriptor() const { return *m_fileDescriptor; } + +private: + QV4L2FileDescriptorPtr m_fileDescriptor; +}; + +using QV4L2MemoryTransferUPtr = std::unique_ptr<QV4L2MemoryTransfer>; + +QV4L2MemoryTransferUPtr makeUserPtrMemoryTransfer(QV4L2FileDescriptorPtr fileDescriptor, + quint32 imageSize); + +QV4L2MemoryTransferUPtr makeMMapMemoryTransfer(QV4L2FileDescriptorPtr fileDescriptor); + +QT_END_NAMESPACE + +#endif // QV4L2MEMORYTRANSFER_P_H diff --git a/src/plugins/multimedia/ffmpeg/qwincapturablewindows.cpp b/src/plugins/multimedia/ffmpeg/qwincapturablewindows.cpp new file mode 100644 index 000000000..aac77aec4 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qwincapturablewindows.cpp @@ -0,0 +1,74 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwincapturablewindows_p.h" +#include "private/qcapturablewindow_p.h" + +#include <qt_windows.h> + +QT_BEGIN_NAMESPACE + +static bool isTopLevelWindow(HWND hwnd) +{ + return hwnd && ::GetAncestor(hwnd, GA_ROOT) == hwnd; +} + +static bool canCaptureWindow(HWND hwnd) +{ + Q_ASSERT(hwnd); + + if (!::IsWindowVisible(hwnd)) + return false; + + RECT rect{}; + if (!::GetWindowRect(hwnd, &rect)) + return false; + + if (rect.left >= rect.right || rect.top >= rect.bottom) + return false; + + return true; +} + +static QString windowTitle(HWND hwnd) { + // QTBUG-114890 + // TODO: investigate the case when hwnd is inner and belows to another thread. + // It might causes deadlocks in specific cases. + auto titleLength = ::GetWindowTextLengthW(hwnd); + std::wstring buffer(titleLength + 1, L'\0'); + titleLength = ::GetWindowTextW(hwnd, buffer.data(), titleLength + 1); + buffer.resize(titleLength); + + return QString::fromStdWString(buffer); +} + +QList<QCapturableWindow> QWinCapturableWindows::windows() const +{ + QList<QCapturableWindow> result; + + auto windowHandler = [](HWND hwnd, LPARAM lParam) { + if (!canCaptureWindow(hwnd)) + return TRUE; // Ignore window and continue enumerating + + auto& windows = *reinterpret_cast<QList<QCapturableWindow>*>(lParam); + + auto windowData = std::make_unique<QCapturableWindowPrivate>(); + windowData->id = reinterpret_cast<QCapturableWindowPrivate::Id>(hwnd); + windowData->description = windowTitle(hwnd); + windows.push_back(windowData.release()->create()); + + return TRUE; + }; + + ::EnumWindows(windowHandler, reinterpret_cast<LPARAM>(&result)); + + return result; +} + +bool QWinCapturableWindows::isWindowValid(const QCapturableWindowPrivate &window) const +{ + const auto hwnd = reinterpret_cast<HWND>(window.id); + return isTopLevelWindow(hwnd) && canCaptureWindow(hwnd); +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qwincapturablewindows_p.h b/src/plugins/multimedia/ffmpeg/qwincapturablewindows_p.h new file mode 100644 index 000000000..1e38708ef --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qwincapturablewindows_p.h @@ -0,0 +1,32 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWINCAPTURABLEWINDOWS_P_H +#define QWINCAPTURABLEWINDOWS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformcapturablewindows_p.h" + +QT_BEGIN_NAMESPACE + +class QWinCapturableWindows : public QPlatformCapturableWindows +{ +public: + QList<QCapturableWindow> windows() const override; + + bool isWindowValid(const QCapturableWindowPrivate &window) const override; +}; + +QT_END_NAMESPACE + +#endif // QWINCAPTURABLEWINDOWS_P_H diff --git a/src/plugins/multimedia/ffmpeg/qwindowscamera.cpp b/src/plugins/multimedia/ffmpeg/qwindowscamera.cpp index b6e031633..61a4ebe52 100644 --- a/src/plugins/multimedia/ffmpeg/qwindowscamera.cpp +++ b/src/plugins/multimedia/ffmpeg/qwindowscamera.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowscamera_p.h" #include "qsemaphore.h" @@ -43,53 +7,24 @@ #include <private/qmemoryvideobuffer_p.h> #include <private/qwindowsmfdefs_p.h> +#include <private/qwindowsmultimediautils_p.h> +#include <private/qvideoframe_p.h> +#include <private/qcomobject_p.h> #include <mfapi.h> #include <mfidl.h> -#include <Mferror.h> -#include <Mfreadwrite.h> +#include <mferror.h> +#include <mfreadwrite.h> #include <system_error> QT_BEGIN_NAMESPACE -class CameraReaderCallback : public IMFSourceReaderCallback +using namespace QWindowsMultimediaUtils; + +class CameraReaderCallback : public QComObject<IMFSourceReaderCallback> { public: - CameraReaderCallback() : m_cRef(1) {} - virtual ~CameraReaderCallback() {} - - //from IUnknown - STDMETHODIMP QueryInterface(REFIID riid, LPVOID *ppvObject) override - { - if (!ppvObject) - return E_POINTER; - if (riid == IID_IMFSourceReaderCallback) { - *ppvObject = static_cast<IMFSourceReaderCallback*>(this); - } else if (riid == IID_IUnknown) { - *ppvObject = static_cast<IUnknown*>(static_cast<IMFSourceReaderCallback*>(this)); - } else { - *ppvObject = nullptr; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; - } - - STDMETHODIMP_(ULONG) AddRef() override - { - return InterlockedIncrement(&m_cRef); - } - - STDMETHODIMP_(ULONG) Release() override - { - LONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) { - delete this; - } - return cRef; - } - //from IMFSourceReaderCallback STDMETHODIMP OnReadSample(HRESULT status, DWORD, DWORD, LONGLONG timestamp, IMFSample *sample) override; STDMETHODIMP OnFlush(DWORD) override; @@ -101,22 +36,24 @@ public: m_activeCamera = activeCamera; } private: - LONG m_cRef; + // Destructor is not public. Caller should call Release. + ~CameraReaderCallback() override = default; + ActiveCamera *m_activeCamera = nullptr; QMutex m_mutex; }; -static QWindowsIUPointer<IMFSourceReader> createCameraReader(IMFMediaSource *mediaSource, - const QWindowsIUPointer<CameraReaderCallback> &callback) +static ComPtr<IMFSourceReader> createCameraReader(IMFMediaSource *mediaSource, + const ComPtr<CameraReaderCallback> &callback) { - QWindowsIUPointer<IMFSourceReader> sourceReader; - QWindowsIUPointer<IMFAttributes> readerAttributes; + ComPtr<IMFSourceReader> sourceReader; + ComPtr<IMFAttributes> readerAttributes; - HRESULT hr = MFCreateAttributes(readerAttributes.address(), 1); + HRESULT hr = MFCreateAttributes(readerAttributes.GetAddressOf(), 1); if (SUCCEEDED(hr)) { - hr = readerAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, callback.get()); + hr = readerAttributes->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, callback.Get()); if (SUCCEEDED(hr)) { - hr = MFCreateSourceReaderFromMediaSource(mediaSource, readerAttributes.get(), sourceReader.address()); + hr = MFCreateSourceReaderFromMediaSource(mediaSource, readerAttributes.Get(), sourceReader.GetAddressOf()); if (SUCCEEDED(hr)) return sourceReader; } @@ -126,18 +63,18 @@ static QWindowsIUPointer<IMFSourceReader> createCameraReader(IMFMediaSource *med return sourceReader; } -static QWindowsIUPointer<IMFMediaSource> createCameraSource(const QString &deviceId) +static ComPtr<IMFMediaSource> createCameraSource(const QString &deviceId) { - QWindowsIUPointer<IMFMediaSource> mediaSource; - QWindowsIUPointer<IMFAttributes> sourceAttributes; - HRESULT hr = MFCreateAttributes(sourceAttributes.address(), 2); + ComPtr<IMFMediaSource> mediaSource; + ComPtr<IMFAttributes> sourceAttributes; + HRESULT hr = MFCreateAttributes(sourceAttributes.GetAddressOf(), 2); if (SUCCEEDED(hr)) { hr = sourceAttributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, QMM_MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID); if (SUCCEEDED(hr)) { hr = sourceAttributes->SetString(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, reinterpret_cast<LPCWSTR>(deviceId.utf16())); if (SUCCEEDED(hr)) { - hr = MFCreateDeviceSource(sourceAttributes.get(), mediaSource.address()); + hr = MFCreateDeviceSource(sourceAttributes.Get(), mediaSource.GetAddressOf()); if (SUCCEEDED(hr)) return mediaSource; } @@ -147,53 +84,68 @@ static QWindowsIUPointer<IMFMediaSource> createCameraSource(const QString &devic return mediaSource; } -static int calculateVideoFrameStride(IMFSourceReader *sourceReader, qsizetype formatIndex, int width) +static int calculateVideoFrameStride(IMFMediaType *videoType, int width) { - Q_ASSERT(sourceReader); - QWindowsIUPointer<IMFMediaType> videoType; - HRESULT hr = sourceReader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, - formatIndex, videoType.address()); + Q_ASSERT(videoType); + + GUID subtype = GUID_NULL; + HRESULT hr = videoType->GetGUID(MF_MT_SUBTYPE, &subtype); if (SUCCEEDED(hr)) { - GUID subtype = GUID_NULL; - hr = videoType->GetGUID(MF_MT_SUBTYPE, &subtype); - if (SUCCEEDED(hr)) { - LONG stride = 0; - hr = MFGetStrideForBitmapInfoHeader(subtype.Data1, width, &stride); - if (SUCCEEDED(hr)) - return int(stride); - } + LONG stride = 0; + hr = MFGetStrideForBitmapInfoHeader(subtype.Data1, width, &stride); + if (SUCCEEDED(hr)) + return int(qAbs(stride)); } - qWarning() << "Failed to calculate video stride" << hr; + qWarning() << "Failed to calculate video stride" << errorString(hr); return 0; } -static bool setCameraReaderFormat(IMFSourceReader *sourceReader, qsizetype formatIndex) +static bool setCameraReaderFormat(IMFSourceReader *sourceReader, IMFMediaType *videoType) { Q_ASSERT(sourceReader); + Q_ASSERT(videoType); - QWindowsIUPointer<IMFMediaType> videoType; - HRESULT hr = sourceReader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, - formatIndex, videoType.address()); - if (SUCCEEDED(hr)) { - hr = sourceReader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, - nullptr, videoType.get()); - if (SUCCEEDED(hr)) { - return true; - } else { - qWarning() << "Failed to set video format" << hr << std::system_category().message(hr).c_str(); - } - } else { - qWarning() << "Failed to select video format at index" << formatIndex << hr << std::system_category().message(hr).c_str(); - } + HRESULT hr = sourceReader->SetCurrentMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, nullptr, + videoType); + if (FAILED(hr)) + qWarning() << "Failed to set video format" << errorString(hr); - return false; + return SUCCEEDED(hr); +} + +static ComPtr<IMFMediaType> findVideoType(IMFSourceReader *reader, + const QCameraFormat &format) +{ + for (DWORD i = 0;; ++i) { + ComPtr<IMFMediaType> candidate; + HRESULT hr = reader->GetNativeMediaType(MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, + candidate.GetAddressOf()); + if (FAILED(hr)) + break; + + GUID subtype = GUID_NULL; + if (FAILED(candidate->GetGUID(MF_MT_SUBTYPE, &subtype))) + continue; + + if (format.pixelFormat() != pixelFormatFromMediaSubtype(subtype)) + continue; + + UINT32 width = 0u; + UINT32 height = 0u; + if (FAILED(MFGetAttributeSize(candidate.Get(), MF_MT_FRAME_SIZE, &width, &height))) + continue; + + if (format.resolution() != QSize{ int(width), int(height) }) + continue; + + return candidate; + } + return {}; } class ActiveCamera { public: - ActiveCamera() = delete; - static std::unique_ptr<ActiveCamera> create(QWindowsCamera &wc, const QCameraDevice &device, const QCameraFormat &format) { auto ac = std::unique_ptr<ActiveCamera>(new ActiveCamera(wc)); @@ -201,54 +153,56 @@ public: if (!ac->m_source) return {}; - ac->m_readerCallback = QWindowsIUPointer<CameraReaderCallback>(new CameraReaderCallback); + ac->m_readerCallback = makeComObject<CameraReaderCallback>(); ac->m_readerCallback->setActiveCamera(ac.get()); - ac->m_reader = createCameraReader(ac->m_source.get(), ac->m_readerCallback); + ac->m_reader = createCameraReader(ac->m_source.Get(), ac->m_readerCallback); if (!ac->m_reader) return {}; - qsizetype index = device.videoFormats().indexOf(format); - if (index < 0) - return {}; - - if (!ac->setFormat(format, index)) + if (!ac->setFormat(format)) return {}; return ac; } - bool setFormat(const QCameraFormat &format, int formatIndex) + bool setFormat(const QCameraFormat &format) { - m_reader->Flush(MF_SOURCE_READER_FIRST_VIDEO_STREAM); - m_flushWait.acquire(); - - if (!setCameraReaderFormat(m_reader.get(), formatIndex)) - return false; - - m_frameFormat = { format.resolution(), format.pixelFormat() }; - m_videoFrameStride = calculateVideoFrameStride(m_reader.get(), formatIndex, format.resolution().width()); + flush(); + + auto videoType = findVideoType(m_reader.Get(), format); + if (videoType) { + if (setCameraReaderFormat(m_reader.Get(), videoType.Get())) { + m_frameFormat = { format.resolution(), format.pixelFormat() }; + m_videoFrameStride = + calculateVideoFrameStride(videoType.Get(), format.resolution().width()); + } + } - m_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, nullptr, - nullptr, nullptr, nullptr); + m_reader->ReadSample(MF_SOURCE_READER_FIRST_VIDEO_STREAM, 0, nullptr, nullptr, nullptr, + nullptr); return true; } void onReadSample(HRESULT status, LONGLONG timestamp, IMFSample *sample) { if (FAILED(status)) { - emit m_windowsCamera.error(int(status), std::system_category().message(status).c_str()); + const std::string msg{ std::system_category().message(status) }; + m_windowsCamera.updateError(QCamera::CameraError, QString::fromStdString(msg)); return; } if (sample) { - QWindowsIUPointer<IMFMediaBuffer> mediaBuffer; - if (SUCCEEDED(sample->ConvertToContiguousBuffer(mediaBuffer.address()))) { + ComPtr<IMFMediaBuffer> mediaBuffer; + if (SUCCEEDED(sample->ConvertToContiguousBuffer(mediaBuffer.GetAddressOf()))) { DWORD bufLen = 0; BYTE *buffer = nullptr; if (SUCCEEDED(mediaBuffer->Lock(&buffer, nullptr, &bufLen))) { QByteArray bytes(reinterpret_cast<char*>(buffer), qsizetype(bufLen)); - QVideoFrame frame(new QMemoryVideoBuffer(bytes, m_videoFrameStride), m_frameFormat); + auto buffer = std::make_unique<QMemoryVideoBuffer>(std::move(bytes), + m_videoFrameStride); + QVideoFrame frame = + QVideoFramePrivate::createFrame(std::move(buffer), m_frameFormat); // WMF uses 100-nanosecond units, Qt uses microseconds frame.setStartTime(timestamp / 10); @@ -274,21 +228,27 @@ public: ~ActiveCamera() { - m_reader->Flush(MF_SOURCE_READER_FIRST_VIDEO_STREAM); - m_flushWait.acquire(); + flush(); m_readerCallback->setActiveCamera(nullptr); } private: explicit ActiveCamera(QWindowsCamera &wc) : m_windowsCamera(wc), m_flushWait(0) {}; + void flush() + { + if (SUCCEEDED(m_reader->Flush(MF_SOURCE_READER_FIRST_VIDEO_STREAM))) { + m_flushWait.acquire(); + } + } + QWindowsCamera &m_windowsCamera; QSemaphore m_flushWait; - QWindowsIUPointer<IMFMediaSource> m_source; - QWindowsIUPointer<IMFSourceReader> m_reader; - QWindowsIUPointer<CameraReaderCallback> m_readerCallback; + ComPtr<IMFMediaSource> m_source; + ComPtr<IMFSourceReader> m_reader; + ComPtr<CameraReaderCallback> m_readerCallback; QVideoFrameFormat m_frameFormat; int m_videoFrameStride = 0; @@ -339,8 +299,8 @@ void QWindowsCamera::setActive(bool active) activeChanged(true); } else { - emit activeChanged(false); m_active.reset(); + emit activeChanged(false); } } @@ -357,11 +317,10 @@ void QWindowsCamera::setCamera(const QCameraDevice &camera) bool QWindowsCamera::setCameraFormat(const QCameraFormat &format) { - qsizetype index = m_cameraDevice.videoFormats().indexOf(format); - if (index < 0) + if (format.isNull()) return false; - bool ok = m_active ? m_active->setFormat(format, index) : true; + bool ok = m_active ? m_active->setFormat(format) : true; if (ok) m_cameraFormat = format; diff --git a/src/plugins/multimedia/ffmpeg/qwindowscamera_p.h b/src/plugins/multimedia/ffmpeg/qwindowscamera_p.h index 22a096b81..80c05ff59 100644 --- a/src/plugins/multimedia/ffmpeg/qwindowscamera_p.h +++ b/src/plugins/multimedia/ffmpeg/qwindowscamera_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSCAMERA_H #define QWINDOWSCAMERA_H @@ -52,7 +16,7 @@ // #include <private/qplatformcamera_p.h> -#include <private/qwindowsiupointer_p.h> +#include <private/qcomptr_p.h> QT_BEGIN_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qx11capturablewindows.cpp b/src/plugins/multimedia/ffmpeg/qx11capturablewindows.cpp new file mode 100644 index 000000000..9e57cbc64 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qx11capturablewindows.cpp @@ -0,0 +1,74 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qx11capturablewindows_p.h" +#include "private/qcapturablewindow_p.h" +#include <qdebug.h> + +#include <X11/Xlib.h> + +QT_BEGIN_NAMESPACE + +QX11CapturableWindows::~QX11CapturableWindows() +{ + if (m_display) + XCloseDisplay(m_display); +} + +QList<QCapturableWindow> QX11CapturableWindows::windows() const +{ + auto display = this->display(); + + if (!display) + return {}; + + Atom atom = XInternAtom(display, "_NET_CLIENT_LIST", true); + Atom actualType = 0; + int format = 0; + unsigned long windowsCount = 0; + unsigned long bytesAfter = 0; + unsigned char *data = nullptr; + const int status = XGetWindowProperty(display, XDefaultRootWindow(display), atom, 0L, (~0L), + false, AnyPropertyType, &actualType, &format, + &windowsCount, &bytesAfter, &data); + + if (status < Success || !data) + return {}; + + QList<QCapturableWindow> result; + + auto freeDataGuard = qScopeGuard([data]() { XFree(data); }); + auto windows = reinterpret_cast<XID *>(data); + for (unsigned long i = 0; i < windowsCount; i++) { + auto windowData = std::make_unique<QCapturableWindowPrivate>(); + windowData->id = static_cast<QCapturableWindowPrivate::Id>(windows[i]); + + char *windowTitle = nullptr; + if (XFetchName(display, windows[i], &windowTitle) && windowTitle) { + windowData->description = QString::fromUtf8(windowTitle); + XFree(windowTitle); + } + + if (isWindowValid(*windowData)) + result.push_back(windowData.release()->create()); + } + + return result; +} + +bool QX11CapturableWindows::isWindowValid(const QCapturableWindowPrivate &window) const +{ + auto display = this->display(); + XWindowAttributes windowAttributes = {}; + return display + && XGetWindowAttributes(display, static_cast<Window>(window.id), &windowAttributes) != 0 + && windowAttributes.depth > 0; +} + +Display *QX11CapturableWindows::display() const +{ + std::call_once(m_displayOnceFlag, [this]() { m_display = XOpenDisplay(nullptr); }); + return m_display; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qx11capturablewindows_p.h b/src/plugins/multimedia/ffmpeg/qx11capturablewindows_p.h new file mode 100644 index 000000000..088fe97cb --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qx11capturablewindows_p.h @@ -0,0 +1,45 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QX11CAPTURABLEWINDOWS_P_H +#define QX11CAPTURABLEWINDOWS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformcapturablewindows_p.h" +#include <mutex> + +struct _XDisplay; +typedef struct _XDisplay Display; + +QT_BEGIN_NAMESPACE + +class QX11CapturableWindows : public QPlatformCapturableWindows +{ +public: + ~QX11CapturableWindows() override; + + QList<QCapturableWindow> windows() const override; + + bool isWindowValid(const QCapturableWindowPrivate &window) const override; + +private: + Display *display() const; + +private: + mutable std::once_flag m_displayOnceFlag; + mutable Display *m_display = nullptr; +}; + +QT_END_NAMESPACE + +#endif // QX11CAPTURABLEWINDOWS_P_H diff --git a/src/plugins/multimedia/ffmpeg/qx11surfacecapture.cpp b/src/plugins/multimedia/ffmpeg/qx11surfacecapture.cpp new file mode 100644 index 000000000..e1b236283 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qx11surfacecapture.cpp @@ -0,0 +1,342 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qx11surfacecapture_p.h" +#include "qffmpegsurfacecapturegrabber_p.h" + +#include <qvideoframe.h> +#include <qscreen.h> +#include <qwindow.h> +#include <qdebug.h> +#include <qguiapplication.h> +#include <qloggingcategory.h> + +#include "private/qcapturablewindow_p.h" +#include "private/qmemoryvideobuffer_p.h" +#include "private/qvideoframeconversionhelper_p.h" +#include "private/qvideoframe_p.h" + +#include <X11/Xlib.h> +#include <sys/shm.h> +#include <X11/extensions/XShm.h> +#include <X11/Xutil.h> +#include <X11/extensions/Xrandr.h> + +#include <optional> + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcX11SurfaceCapture, "qt.multimedia.ffmpeg.qx11surfacecapture"); + +namespace { + +void destroyXImage(XImage* image) { + XDestroyImage(image); // macro +} + +template <typename T, typename D> +std::unique_ptr<T, D> makeXUptr(T* ptr, D deleter) { + return std::unique_ptr<T, D>(ptr, deleter); +} + +int screenNumberByName(Display *display, const QString &name) +{ + int size = 0; + auto monitors = makeXUptr( + XRRGetMonitors(display, XDefaultRootWindow(display), true, &size), + &XRRFreeMonitors); + const auto end = monitors.get() + size; + auto found = std::find_if(monitors.get(), end, [&](const XRRMonitorInfo &info) { + auto atomName = makeXUptr(XGetAtomName(display, info.name), &XFree); + return atomName && name == QString::fromUtf8(atomName.get()); + }); + + return found == end ? -1 : std::distance(monitors.get(), found); +} + +QVideoFrameFormat::PixelFormat xImagePixelFormat(const XImage &image) +{ + if (image.bits_per_pixel != 32) return QVideoFrameFormat::Format_Invalid; + + if (image.red_mask == 0xff0000 && + image.green_mask == 0xff00 && + image.blue_mask == 0xff) + return QVideoFrameFormat::Format_BGRX8888; + + if (image.red_mask == 0xff00 && + image.green_mask == 0xff0000 && + image.blue_mask == 0xff000000) + return QVideoFrameFormat::Format_XBGR8888; + + if (image.blue_mask == 0xff0000 && + image.green_mask == 0xff00 && + image.red_mask == 0xff) + return QVideoFrameFormat::Format_RGBX8888; + + if (image.red_mask == 0xff00 && + image.green_mask == 0xff0000 && + image.blue_mask == 0xff000000) + return QVideoFrameFormat::Format_XRGB8888; + + return QVideoFrameFormat::Format_Invalid; +} + +} // namespace + +class QX11SurfaceCapture::Grabber : private QFFmpegSurfaceCaptureGrabber +{ +public: + static std::unique_ptr<Grabber> create(QX11SurfaceCapture &capture, QScreen *screen) + { + std::unique_ptr<Grabber> result(new Grabber(capture)); + return result->init(screen) ? std::move(result) : nullptr; + } + + static std::unique_ptr<Grabber> create(QX11SurfaceCapture &capture, WId wid) + { + std::unique_ptr<Grabber> result(new Grabber(capture)); + return result->init(wid) ? std::move(result) : nullptr; + } + + ~Grabber() override + { + stop(); + + detachShm(); + } + + const QVideoFrameFormat &format() const { return m_format; } + +private: + Grabber(QX11SurfaceCapture &capture) + { + addFrameCallback(capture, &QX11SurfaceCapture::newVideoFrame); + connect(this, &Grabber::errorUpdated, &capture, &QX11SurfaceCapture::updateError); + } + + bool createDisplay() + { + if (!m_display) + m_display.reset(XOpenDisplay(nullptr)); + + if (!m_display) + updateError(QPlatformSurfaceCapture::InternalError, + QLatin1String("Cannot open X11 display")); + + return m_display != nullptr; + } + + bool init(WId wid) + { + if (auto screen = QGuiApplication::primaryScreen()) + setFrameRate(screen->refreshRate()); + + return createDisplay() && initWithXID(static_cast<XID>(wid)); + } + + bool init(QScreen *screen) + { + if (!screen) { + updateError(QPlatformSurfaceCapture::NotFound, QLatin1String("Screen Not Found")); + return false; + } + + if (!createDisplay()) + return false; + + auto screenNumber = screenNumberByName(m_display.get(), screen->name()); + + if (screenNumber < 0) + return false; + + setFrameRate(screen->refreshRate()); + + return initWithXID(RootWindow(m_display.get(), screenNumber)); + } + + bool initWithXID(XID xid) + { + m_xid = xid; + + if (update()) { + start(); + return true; + } + + return false; + } + + void detachShm() + { + if (std::exchange(m_attached, false)) { + XShmDetach(m_display.get(), &m_shmInfo); + shmdt(m_shmInfo.shmaddr); + shmctl(m_shmInfo.shmid, IPC_RMID, 0); + } + } + + void attachShm() + { + Q_ASSERT(!m_attached); + + m_shmInfo.shmid = + shmget(IPC_PRIVATE, m_xImage->bytes_per_line * m_xImage->height, IPC_CREAT | 0777); + + if (m_shmInfo.shmid == -1) + return; + + m_shmInfo.readOnly = false; + m_shmInfo.shmaddr = m_xImage->data = (char *)shmat(m_shmInfo.shmid, 0, 0); + + m_attached = XShmAttach(m_display.get(), &m_shmInfo); + } + + bool update() + { + XWindowAttributes wndattr = {}; + if (XGetWindowAttributes(m_display.get(), m_xid, &wndattr) == 0) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot get window attributes")); + return false; + } + + // TODO: if capture windows, we should adjust offsets and size if + // the window is out of the screen borders + // m_xOffset = ... + // m_yOffset = ... + + // check window params for the root window as well since + // it potentially can be changed (e.g. on VM with resizing) + if (!m_xImage || wndattr.width != m_xImage->width || wndattr.height != m_xImage->height + || wndattr.depth != m_xImage->depth || wndattr.visual->visualid != m_visualID) { + + qCDebug(qLcX11SurfaceCapture) << "recreate ximage: " << wndattr.width << wndattr.height + << wndattr.depth << wndattr.visual->visualid; + + detachShm(); + m_xImage.reset(); + + m_visualID = wndattr.visual->visualid; + m_xImage.reset(XShmCreateImage(m_display.get(), wndattr.visual, wndattr.depth, ZPixmap, + nullptr, &m_shmInfo, wndattr.width, wndattr.height)); + + if (!m_xImage) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot create image")); + return false; + } + + const auto pixelFormat = xImagePixelFormat(*m_xImage); + + // TODO: probably, add a converter instead + if (pixelFormat == QVideoFrameFormat::Format_Invalid) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Not handled pixel format, bpp=") + + QString::number(m_xImage->bits_per_pixel)); + return false; + } + + attachShm(); + + if (!m_attached) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String("Cannot attach shared memory")); + return false; + } + + QVideoFrameFormat format(QSize(m_xImage->width, m_xImage->height), pixelFormat); + format.setStreamFrameRate(frameRate()); + m_format = format; + } + + return m_attached; + } + +protected: + QVideoFrame grabFrame() override + { + if (!update()) + return {}; + + if (!XShmGetImage(m_display.get(), m_xid, m_xImage.get(), m_xOffset, m_yOffset, + AllPlanes)) { + updateError(QPlatformSurfaceCapture::CaptureFailed, + QLatin1String( + "Cannot get ximage; the window may be out of the screen borders")); + return {}; + } + + QByteArray data(m_xImage->bytes_per_line * m_xImage->height, Qt::Uninitialized); + + const auto pixelSrc = reinterpret_cast<const uint32_t *>(m_xImage->data); + const auto pixelDst = reinterpret_cast<uint32_t *>(data.data()); + const auto pixelCount = data.size() / 4; + const auto xImageAlphaVaries = false; // In known cases it doesn't vary - it's 0xff or 0xff + + qCopyPixelsWithAlphaMask(pixelDst, pixelSrc, pixelCount, m_format.pixelFormat(), + xImageAlphaVaries); + + auto buffer = std::make_unique<QMemoryVideoBuffer>(data, m_xImage->bytes_per_line); + return QVideoFramePrivate::createFrame(std::move(buffer), m_format); + } + +private: + std::optional<QPlatformSurfaceCapture::Error> m_prevGrabberError; + XID m_xid = None; + int m_xOffset = 0; + int m_yOffset = 0; + std::unique_ptr<Display, decltype(&XCloseDisplay)> m_display{ nullptr, &XCloseDisplay }; + std::unique_ptr<XImage, decltype(&destroyXImage)> m_xImage{ nullptr, &destroyXImage }; + XShmSegmentInfo m_shmInfo; + bool m_attached = false; + VisualID m_visualID = None; + QVideoFrameFormat m_format; +}; + +QX11SurfaceCapture::QX11SurfaceCapture(Source initialSource) + : QPlatformSurfaceCapture(initialSource) +{ + // For debug + // XSetErrorHandler([](Display *, XErrorEvent * e) { + // qDebug() << "error handler" << e->error_code; + // return 0; + // }); +} + +QX11SurfaceCapture::~QX11SurfaceCapture() = default; + +QVideoFrameFormat QX11SurfaceCapture::frameFormat() const +{ + return m_grabber ? m_grabber->format() : QVideoFrameFormat{}; +} + +bool QX11SurfaceCapture::setActiveInternal(bool active) +{ + qCDebug(qLcX11SurfaceCapture) << "set active" << active; + + if (m_grabber) + m_grabber.reset(); + else + std::visit([this](auto source) { activate(source); }, source()); + + return static_cast<bool>(m_grabber) == active; +} + +void QX11SurfaceCapture::activate(ScreenSource screen) +{ + if (checkScreenWithError(screen)) + m_grabber = Grabber::create(*this, screen); +} + +void QX11SurfaceCapture::activate(WindowSource window) +{ + auto handle = QCapturableWindowPrivate::handle(window); + m_grabber = Grabber::create(*this, handle ? handle->id : 0); +} + +bool QX11SurfaceCapture::isSupported() +{ + return qgetenv("XDG_SESSION_TYPE").compare(QLatin1String("x11"), Qt::CaseInsensitive) == 0; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/qx11surfacecapture_p.h b/src/plugins/multimedia/ffmpeg/qx11surfacecapture_p.h new file mode 100644 index 000000000..7f794fd8b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/qx11surfacecapture_p.h @@ -0,0 +1,48 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef X11SURFACECAPTURE_P_H +#define X11SURFACECAPTURE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "private/qplatformsurfacecapture_p.h" + +QT_BEGIN_NAMESPACE + +class QX11SurfaceCapture : public QPlatformSurfaceCapture +{ + class Grabber; + +public: + explicit QX11SurfaceCapture(Source initialSource); + ~QX11SurfaceCapture() override; + + QVideoFrameFormat frameFormat() const override; + + static bool isSupported(); + +protected: + bool setActiveInternal(bool active) override; + +private: + void activate(ScreenSource); + + void activate(WindowSource); + +private: + std::unique_ptr<Grabber> m_grabber; +}; + +QT_END_NAMESPACE + +#endif // X11SURFACECAPTURE_P_H diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder.cpp new file mode 100644 index 000000000..d8eaae58b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder.cpp @@ -0,0 +1,343 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include "qffmpegaudioencoder_p.h" +#include "qffmpegrecordingengineutils_p.h" +#include "qffmpegaudioencoderutils_p.h" +#include "qffmpegaudioinput_p.h" +#include "qffmpegencoderoptions_p.h" +#include "qffmpegmuxer_p.h" +#include "qffmpegrecordingengine_p.h" +#include "qffmpegmediaformatinfo_p.h" +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +Q_STATIC_LOGGING_CATEGORY(qLcFFmpegAudioEncoder, "qt.multimedia.ffmpeg.audioencoder"); + +AudioEncoder::AudioEncoder(RecordingEngine &recordingEngine, const QAudioFormat &sourceFormat, + const QMediaEncoderSettings &settings) + : EncoderThread(recordingEngine), m_format(sourceFormat), m_settings(settings) +{ + setObjectName(QLatin1String("AudioEncoder")); + qCDebug(qLcFFmpegAudioEncoder) << "AudioEncoder" << settings.audioCodec(); + + auto codecID = QFFmpegMediaFormatInfo::codecIdForAudioCodec(settings.audioCodec()); + Q_ASSERT(avformat_query_codec(recordingEngine.avFormatContext()->oformat, codecID, + FF_COMPLIANCE_NORMAL)); + + const AVAudioFormat requestedAudioFormat(m_format); + + m_avCodec = QFFmpeg::findAVEncoder(codecID, {}, requestedAudioFormat.sampleFormat); + + if (!m_avCodec) + m_avCodec = QFFmpeg::findAVEncoder(codecID); + + qCDebug(qLcFFmpegAudioEncoder) << "found audio codec" << m_avCodec->name; + + Q_ASSERT(m_avCodec); + + m_stream = avformat_new_stream(recordingEngine.avFormatContext(), nullptr); + m_stream->id = recordingEngine.avFormatContext()->nb_streams - 1; + m_stream->codecpar->codec_type = AVMEDIA_TYPE_AUDIO; + m_stream->codecpar->codec_id = codecID; +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + m_stream->codecpar->channel_layout = + adjustChannelLayout(m_avCodec->channel_layouts, requestedAudioFormat.channelLayoutMask); + m_stream->codecpar->channels = qPopulationCount(m_stream->codecpar->channel_layout); +#else + m_stream->codecpar->ch_layout = + adjustChannelLayout(m_avCodec->ch_layouts, requestedAudioFormat.channelLayout); +#endif + const auto sampleRate = + adjustSampleRate(m_avCodec->supported_samplerates, requestedAudioFormat.sampleRate); + + m_stream->codecpar->sample_rate = sampleRate; + m_stream->codecpar->frame_size = 1024; + m_stream->codecpar->format = + adjustSampleFormat(m_avCodec->sample_fmts, requestedAudioFormat.sampleFormat); + + m_stream->time_base = AVRational{ 1, sampleRate }; + + qCDebug(qLcFFmpegAudioEncoder) << "set stream time_base" << m_stream->time_base.num << "/" + << m_stream->time_base.den; +} + +void AudioEncoder::open() +{ + m_codecContext.reset(avcodec_alloc_context3(m_avCodec)); + + if (m_stream->time_base.num != 1 || m_stream->time_base.den != m_format.sampleRate()) { + qCDebug(qLcFFmpegAudioEncoder) << "Most likely, av_format_write_header changed time base from" + << 1 << "/" << m_format.sampleRate() << "to" + << m_stream->time_base; + } + + m_codecContext->time_base = m_stream->time_base; + + avcodec_parameters_to_context(m_codecContext.get(), m_stream->codecpar); + + AVDictionaryHolder opts; + applyAudioEncoderOptions(m_settings, m_avCodec->name, m_codecContext.get(), opts); + applyExperimentalCodecOptions(m_avCodec, opts); + + const int res = avcodec_open2(m_codecContext.get(), m_avCodec, opts); + + qCDebug(qLcFFmpegAudioEncoder) << "audio codec opened" << res; + qCDebug(qLcFFmpegAudioEncoder) << "audio codec params: fmt=" << m_codecContext->sample_fmt + << "rate=" << m_codecContext->sample_rate; + + updateResampler(); +} + +void AudioEncoder::addBuffer(const QAudioBuffer &buffer) +{ + if (!buffer.isValid()) { + setEndOfSourceStream(); + return; + } + + { + const std::chrono::microseconds bufferDuration(buffer.duration()); + auto guard = lockLoopData(); + + resetEndOfSourceStream(); + + if (m_paused) + return; + + // TODO: apply logic with canPushFrame + + m_audioBufferQueue.push(buffer); + m_queueDuration += bufferDuration; + } + + dataReady(); +} + +QAudioBuffer AudioEncoder::takeBuffer() +{ + auto locker = lockLoopData(); + QAudioBuffer result = dequeueIfPossible(m_audioBufferQueue); + m_queueDuration -= std::chrono::microseconds(result.duration()); + return result; +} + +void AudioEncoder::init() +{ + open(); + + // TODO: try to address this dependency here. + if (auto input = qobject_cast<QFFmpegAudioInput *>(source())) + input->setFrameSize(m_codecContext->frame_size); + + qCDebug(qLcFFmpegAudioEncoder) << "AudioEncoder::init started audio device thread."; +} + +void AudioEncoder::cleanup() +{ + while (!m_audioBufferQueue.empty()) + processOne(); + + if (m_avFrameSamplesOffset) { + // the size of the last frame can be less than m_codecContext->frame_size + + retrievePackets(); + sendPendingFrameToAVCodec(); + } + + while (avcodec_send_frame(m_codecContext.get(), nullptr) == AVERROR(EAGAIN)) + retrievePackets(); + retrievePackets(); +} + +bool AudioEncoder::hasData() const +{ + return !m_audioBufferQueue.empty(); +} + +void AudioEncoder::retrievePackets() +{ + while (1) { + AVPacketUPtr packet(av_packet_alloc()); + int ret = avcodec_receive_packet(m_codecContext.get(), packet.get()); + if (ret < 0) { + if (ret != AVERROR(EOF)) + break; + if (ret != AVERROR(EAGAIN)) { + char errStr[1024]; + av_strerror(ret, errStr, 1024); + qCDebug(qLcFFmpegAudioEncoder) << "receive packet" << ret << errStr; + } + break; + } + + // qCDebug(qLcFFmpegEncoder) << "writing audio packet" << packet->size << packet->pts << + // packet->dts; + packet->stream_index = m_stream->id; + m_recordingEngine.getMuxer()->addPacket(std::move(packet)); + } +} + +void AudioEncoder::processOne() +{ + QAudioBuffer buffer = takeBuffer(); + Q_ASSERT(buffer.isValid()); + + // qCDebug(qLcFFmpegEncoder) << "new audio buffer" << buffer.byteCount() << buffer.format() + // << buffer.frameCount() << codec->frame_size; + + if (buffer.format() != m_format) { + m_format = buffer.format(); + updateResampler(); + } + + int samplesOffset = 0; + const int bufferSamplesCount = static_cast<int>(buffer.frameCount()); + + while (samplesOffset < bufferSamplesCount) + handleAudioData(buffer.constData<uint8_t>(), samplesOffset, bufferSamplesCount); + + Q_ASSERT(samplesOffset == bufferSamplesCount); +} + +bool AudioEncoder::checkIfCanPushFrame() const +{ + if (isRunning()) + return m_audioBufferQueue.size() <= 1 || m_queueDuration < m_maxQueueDuration; + if (!isFinished()) + return m_audioBufferQueue.empty(); + + return false; +} + +void AudioEncoder::updateResampler() +{ + m_resampler.reset(); + + const AVAudioFormat requestedAudioFormat(m_format); + const AVAudioFormat codecAudioFormat(m_codecContext.get()); + + if (requestedAudioFormat != codecAudioFormat) + m_resampler = createResampleContext(requestedAudioFormat, codecAudioFormat); + + qCDebug(qLcFFmpegAudioEncoder) + << "Resampler updated. Input format:" << m_format << "Resampler:" << m_resampler.get(); +} + +void AudioEncoder::ensurePendingFrame(int availableSamplesCount) +{ + Q_ASSERT(availableSamplesCount >= 0); + + if (m_avFrame) + return; + + m_avFrame = makeAVFrame(); + + m_avFrame->format = m_codecContext->sample_fmt; +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + m_avFrame->channel_layout = m_codecContext->channel_layout; + m_avFrame->channels = m_codecContext->channels; +#else + m_avFrame->ch_layout = m_codecContext->ch_layout; +#endif + m_avFrame->sample_rate = m_codecContext->sample_rate; + + const bool isFixedFrameSize = !(m_avCodec->capabilities & AV_CODEC_CAP_VARIABLE_FRAME_SIZE) + && m_codecContext->frame_size; + m_avFrame->nb_samples = isFixedFrameSize ? m_codecContext->frame_size : availableSamplesCount; + if (m_avFrame->nb_samples) + av_frame_get_buffer(m_avFrame.get(), 0); + + const auto &timeBase = m_stream->time_base; + const auto pts = timeBase.den && timeBase.num + ? timeBase.den * m_samplesWritten / (m_codecContext->sample_rate * timeBase.num) + : m_samplesWritten; + setAVFrameTime(*m_avFrame, pts, timeBase); +} + +void AudioEncoder::writeDataToPendingFrame(const uchar *data, int &samplesOffset, int samplesCount) +{ + Q_ASSERT(m_avFrame); + Q_ASSERT(m_avFrameSamplesOffset <= m_avFrame->nb_samples); + + const int bytesPerSample = av_get_bytes_per_sample(m_codecContext->sample_fmt); + const bool isPlanar = av_sample_fmt_is_planar(m_codecContext->sample_fmt); + +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + const int channelsCount = m_codecContext->channels; +#else + const int channelsCount = m_codecContext->ch_layout.nb_channels; +#endif + + const int audioDataOffset = isPlanar ? bytesPerSample * m_avFrameSamplesOffset + : bytesPerSample * m_avFrameSamplesOffset * channelsCount; + + const int planesCount = isPlanar ? channelsCount : 1; + m_avFramePlanesData.resize(planesCount); + for (int plane = 0; plane < planesCount; ++plane) + m_avFramePlanesData[plane] = m_avFrame->extended_data[plane] + audioDataOffset; + + const int samplesToRead = + std::min(m_avFrame->nb_samples - m_avFrameSamplesOffset, samplesCount - samplesOffset); + + data += m_format.bytesForFrames(samplesOffset); + + if (m_resampler) { + m_avFrameSamplesOffset += swr_convert(m_resampler.get(), m_avFramePlanesData.data(), + samplesToRead, &data, samplesToRead); + } else { + Q_ASSERT(planesCount == 1); + m_avFrameSamplesOffset += samplesToRead; + memcpy(m_avFramePlanesData[0], data, m_format.bytesForFrames(samplesToRead)); + } + + samplesOffset += samplesToRead; +} + +void AudioEncoder::sendPendingFrameToAVCodec() +{ + Q_ASSERT(m_avFrame); + Q_ASSERT(m_avFrameSamplesOffset <= m_avFrame->nb_samples); + + m_avFrame->nb_samples = m_avFrameSamplesOffset; + + m_samplesWritten += m_avFrameSamplesOffset; + + const qint64 time = m_format.durationForFrames(m_samplesWritten); + m_recordingEngine.newTimeStamp(time / 1000); + + // qCDebug(qLcFFmpegEncoder) << "sending audio frame" << buffer.byteCount() << frame->pts << + // ((double)buffer.frameCount()/frame->sample_rate); + + int ret = avcodec_send_frame(m_codecContext.get(), m_avFrame.get()); + if (ret < 0) { + char errStr[AV_ERROR_MAX_STRING_SIZE]; + av_strerror(ret, errStr, AV_ERROR_MAX_STRING_SIZE); + qCDebug(qLcFFmpegAudioEncoder) << "error sending frame" << ret << errStr; + } + + m_avFrame = nullptr; + m_avFrameSamplesOffset = 0; + std::fill(m_avFramePlanesData.begin(), m_avFramePlanesData.end(), nullptr); +} + +void AudioEncoder::handleAudioData(const uchar *data, int &samplesOffset, int samplesCount) +{ + ensurePendingFrame(samplesCount - samplesOffset); + + writeDataToPendingFrame(data, samplesOffset, samplesCount); + + // The frame is not ready yet + if (m_avFrameSamplesOffset < m_avFrame->nb_samples) + return; + + retrievePackets(); + + sendPendingFrameToAVCodec(); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder_p.h new file mode 100644 index 000000000..4408ff54f --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoder_p.h @@ -0,0 +1,77 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGAUDIOENCODER_P_H +#define QFFMPEGAUDIOENCODER_P_H + +#include "qffmpeg_p.h" +#include "qffmpegencoderthread_p.h" +#include "private/qplatformmediarecorder_p.h" +#include <qaudiobuffer.h> +#include <queue> +#include <chrono> + +QT_BEGIN_NAMESPACE + +class QMediaEncoderSettings; + +namespace QFFmpeg { + +class AudioEncoder : public EncoderThread +{ +public: + AudioEncoder(RecordingEngine &recordingEngine, const QAudioFormat &sourceFormat, + const QMediaEncoderSettings &settings); + + void addBuffer(const QAudioBuffer &buffer); + +protected: + bool checkIfCanPushFrame() const override; + +private: + void open(); + + QAudioBuffer takeBuffer(); + void retrievePackets(); + void updateResampler(); + + void init() override; + void cleanup() override; + bool hasData() const override; + void processOne() override; + + void handleAudioData(const uchar *data, int &samplesOffset, int samplesCount); + + void ensurePendingFrame(int availableSamplesCount); + + void writeDataToPendingFrame(const uchar *data, int &samplesOffset, int samplesCount); + + void sendPendingFrameToAVCodec(); + +private: + std::queue<QAudioBuffer> m_audioBufferQueue; + + // Arbitrarily chosen to limit audio queue duration + const std::chrono::microseconds m_maxQueueDuration = std::chrono::seconds(5); + + std::chrono::microseconds m_queueDuration{ 0 }; + + AVStream *m_stream = nullptr; + AVCodecContextUPtr m_codecContext; + QAudioFormat m_format; + + SwrContextUPtr m_resampler; + qint64 m_samplesWritten = 0; + const AVCodec *m_avCodec = nullptr; + QMediaEncoderSettings m_settings; + + AVFrameUPtr m_avFrame; + int m_avFrameSamplesOffset = 0; + std::vector<uint8_t *> m_avFramePlanesData; +}; + + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoderutils.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoderutils.cpp new file mode 100644 index 000000000..4d4dc69d2 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoderutils.cpp @@ -0,0 +1,97 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegaudioencoderutils_p.h" +#include "qalgorithms.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +AVSampleFormat adjustSampleFormat(const AVSampleFormat *supportedFormats, AVSampleFormat requested) +{ + auto calcScore = [requested](AVSampleFormat format) { + return format == requested ? BestAVScore + : format == av_get_planar_sample_fmt(requested) ? BestAVScore - 1 + : 0; + }; + + const auto result = findBestAVValue(supportedFormats, calcScore).first; + return result == AV_SAMPLE_FMT_NONE ? requested : result; +} + +int adjustSampleRate(const int *supportedRates, int requested) +{ + auto calcScore = [requested](int rate) { + return requested == rate ? BestAVScore + : requested <= rate ? rate - requested + : requested - rate - 1000000; + }; + + const auto result = findBestAVValue(supportedRates, calcScore).first; + return result == 0 ? requested : result; +} + +static AVScore calculateScoreByChannelsCount(int supportedChannelsNumber, + int requestedChannelsNumber) +{ + if (supportedChannelsNumber >= requestedChannelsNumber) + return requestedChannelsNumber - supportedChannelsNumber; + + return supportedChannelsNumber - requestedChannelsNumber - 10000; +} + +static AVScore calculateScoreByChannelsMask(int supportedChannelsNumber, uint64_t supportedMask, + int requestedChannelsNumber, uint64_t requestedMask) +{ + if ((supportedMask & requestedMask) == requestedMask) + return BestAVScore - qPopulationCount(supportedMask & ~requestedMask); + + return calculateScoreByChannelsCount(supportedChannelsNumber, requestedChannelsNumber); +} + +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT + +uint64_t adjustChannelLayout(const uint64_t *supportedMasks, uint64_t requested) +{ + auto calcScore = [requested](uint64_t mask) { + return calculateScoreByChannelsMask(qPopulationCount(mask), mask, + qPopulationCount(requested), requested); + }; + + const auto result = findBestAVValue(supportedMasks, calcScore).first; + return result == 0 ? requested : result; +} + +#else + +AVChannelLayout adjustChannelLayout(const AVChannelLayout *supportedLayouts, + const AVChannelLayout &requested) +{ + auto calcScore = [&requested](const AVChannelLayout &layout) { + if (layout == requested) + return BestAVScore; + + // The only realistic case for now: + // layout.order == requested.order == AV_CHANNEL_ORDER_NATIVE + // Let's consider other orders to make safe code + + if (layout.order == AV_CHANNEL_ORDER_CUSTOM || requested.order == AV_CHANNEL_ORDER_CUSTOM) + return calculateScoreByChannelsCount(layout.nb_channels, requested.nb_channels) - 1000; + + const auto offset = layout.order == requested.order ? 1 : 100; + + return calculateScoreByChannelsMask(layout.nb_channels, layout.u.mask, + requested.nb_channels, requested.u.mask) + - offset; + }; + + const auto result = findBestAVValue(supportedLayouts, calcScore); + return result.second == NotSuitableAVScore ? requested : result.first; +} + +#endif + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoderutils_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoderutils_p.h new file mode 100644 index 000000000..8a7c184ec --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegaudioencoderutils_p.h @@ -0,0 +1,28 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGAUDIOENCODERUTILS_P_H +#define QFFMPEGAUDIOENCODERUTILS_P_H + +#include "qffmpeg_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +AVSampleFormat adjustSampleFormat(const AVSampleFormat *supportedFormats, AVSampleFormat requested); + +int adjustSampleRate(const int *supportedRates, int requested); + +#if QT_FFMPEG_OLD_CHANNEL_LAYOUT +uint64_t adjustChannelLayout(const uint64_t *supportedLayouts, uint64_t requested); +#else +AVChannelLayout adjustChannelLayout(const AVChannelLayout *supportedLayouts, + const AVChannelLayout &requested); +#endif + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGAUDIOENCODERUTILS_P_H diff --git a/src/plugins/multimedia/ffmpeg/qffmpegencoderoptions.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderoptions.cpp index fad90b3c1..bd6a8e09b 100644 --- a/src/plugins/multimedia/ffmpeg/qffmpegencoderoptions.cpp +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderoptions.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qffmpegencoderoptions_p.h" #if QT_CONFIG(vaapi) @@ -93,6 +57,20 @@ static int bitrateForSettings(const QMediaEncoderSettings &settings, bool hdr = return bitrate; } +static void apply_openh264(const QMediaEncoderSettings &settings, AVCodecContext *codec, + AVDictionary **opts) +{ + if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding + || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) { + codec->bit_rate = settings.videoBitRate(); + av_dict_set(opts, "rc_mode", "bitrate", 0); + } else { + av_dict_set(opts, "rc_mode", "quality", 0); + static const int q[] = { 51, 48, 38, 25, 5 }; + codec->qmax = codec->qmin = q[settings.quality()]; + } +} + static void apply_x264(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts) { if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) { @@ -132,7 +110,7 @@ static void apply_libvpx(const QMediaEncoderSettings &settings, AVCodecContext * } #ifdef Q_OS_DARWIN -static void apply_videotoolbox(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **) +static void apply_videotoolbox(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts) { if (settings.encodingMode() == QMediaRecorder::ConstantBitRateEncoding || settings.encodingMode() == QMediaRecorder::AverageBitRateEncoding) { codec->bit_rate = settings.videoBitRate(); @@ -152,6 +130,13 @@ static void apply_videotoolbox(const QMediaEncoderSettings &settings, AVCodecCon codec->bit_rate = bitrateForSettings(settings); #endif } + + // Videotooldox hw acceleration fails of some hardwares, + // allow_sw makes sw encoding available if hw encoding failed. + // Under the hood, ffmpeg sets + // kVTVideoEncoderSpecification_EnableHardwareAcceleratedVideoEncoder instead of + // kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder + av_dict_set(opts, "allow_sw", "1", 0); } #endif @@ -206,14 +191,34 @@ static void apply_vaapi(const QMediaEncoderSettings &settings, AVCodecContext *c break; } - if (quality) { - qDebug() << "using quality" << settings.quality() << quality[settings.quality()]; + if (quality) codec->global_quality = quality[settings.quality()]; - } } } #endif +static void apply_nvenc(const QMediaEncoderSettings &settings, AVCodecContext *codec, + AVDictionary **opts) +{ + switch (settings.encodingMode()) { + case QMediaRecorder::EncodingMode::AverageBitRateEncoding: + av_dict_set(opts, "vbr", "1", 0); + codec->bit_rate = settings.videoBitRate(); + break; + case QMediaRecorder::EncodingMode::ConstantBitRateEncoding: + av_dict_set(opts, "cbr", "1", 0); + codec->bit_rate = settings.videoBitRate(); + codec->rc_max_rate = codec->rc_min_rate = codec->bit_rate; + break; + case QMediaRecorder::EncodingMode::ConstantQualityEncoding: { + static const char *q[] = { "51", "48", "35", "15", "1" }; + av_dict_set(opts, "cq", q[settings.quality()], 0); + } break; + default: + break; + } +} + #ifdef Q_OS_WINDOWS static void apply_mf(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts) { @@ -230,6 +235,49 @@ static void apply_mf(const QMediaEncoderSettings &settings, AVCodecContext *code } #endif +#ifdef Q_OS_ANDROID +static void apply_mediacodec(const QMediaEncoderSettings &settings, AVCodecContext *codec, + AVDictionary **opts) +{ + codec->bit_rate = settings.videoBitRate(); + + const int quality[] = { 25, 50, 75, 90, 100 }; + codec->global_quality = quality[settings.quality()]; + + switch (settings.encodingMode()) { + case QMediaRecorder::EncodingMode::AverageBitRateEncoding: + av_dict_set(opts, "bitrate_mode", "vbr", 1); + break; + case QMediaRecorder::EncodingMode::ConstantBitRateEncoding: + av_dict_set(opts, "bitrate_mode", "cbr", 1); + break; + case QMediaRecorder::EncodingMode::ConstantQualityEncoding: + // av_dict_set(opts, "bitrate_mode", "cq", 1); + av_dict_set(opts, "bitrate_mode", "cbr", 1); + break; + default: + break; + } + + switch (settings.videoCodec()) { + case QMediaFormat::VideoCodec::H264: { + const char *levels[] = { "2.2", "3.2", "4.2", "5.2", "6.2" }; + av_dict_set(opts, "level", levels[settings.quality()], 1); + codec->profile = FF_PROFILE_H264_HIGH; + break; + } + case QMediaFormat::VideoCodec::H265: { + const char *levels[] = { "h2.1", "h3.1", "h4.1", "h5.1", "h6.1" }; + av_dict_set(opts, "level", levels[settings.quality()], 1); + codec->profile = FF_PROFILE_HEVC_MAIN; + break; + } + default: + break; + } +} +#endif + namespace QFFmpeg { using ApplyOptions = void (*)(const QMediaEncoderSettings &settings, AVCodecContext *codec, AVDictionary **opts); @@ -237,31 +285,37 @@ using ApplyOptions = void (*)(const QMediaEncoderSettings &settings, AVCodecCont const struct { const char *name; ApplyOptions apply; -} videoCodecOptionTable[] = { - { "libx264", apply_x264 }, - { "libx265xx", apply_x265 }, - { "libvpx", apply_libvpx }, - { "libvpx_vp9", apply_libvpx }, +} videoCodecOptionTable[] = { { "libx264", apply_x264 }, + { "libx265xx", apply_x265 }, + { "libvpx", apply_libvpx }, + { "libvpx_vp9", apply_libvpx }, + { "libopenh264", apply_openh264 }, + { "h264_nvenc", apply_nvenc }, + { "hevc_nvenc", apply_nvenc }, + { "av1_nvenc", apply_nvenc }, #ifdef Q_OS_DARWIN - { "h264_videotoolbox", apply_videotoolbox }, - { "hevc_videotoolbox", apply_videotoolbox }, - { "prores_videotoolbox", apply_videotoolbox }, - { "vp9_videotoolbox", apply_videotoolbox }, + { "h264_videotoolbox", apply_videotoolbox }, + { "hevc_videotoolbox", apply_videotoolbox }, + { "prores_videotoolbox", apply_videotoolbox }, + { "vp9_videotoolbox", apply_videotoolbox }, #endif #if QT_CONFIG(vaapi) - { "mpeg2_vaapi", apply_vaapi }, - { "mjpeg_vaapi", apply_vaapi }, - { "h264_vaapi", apply_vaapi }, - { "hevc_vaapi", apply_vaapi }, - { "vp8_vaapi", apply_vaapi }, - { "vp9_vaapi", apply_vaapi }, + { "mpeg2_vaapi", apply_vaapi }, + { "mjpeg_vaapi", apply_vaapi }, + { "h264_vaapi", apply_vaapi }, + { "hevc_vaapi", apply_vaapi }, + { "vp8_vaapi", apply_vaapi }, + { "vp9_vaapi", apply_vaapi }, #endif #ifdef Q_OS_WINDOWS - { "hevc_mf", apply_mf }, - { "h264_mf", apply_mf }, + { "hevc_mf", apply_mf }, + { "h264_mf", apply_mf }, #endif - { nullptr, nullptr } -}; +#ifdef Q_OS_ANDROID + { "hevc_mediacodec", apply_mediacodec }, + { "h264_mediacodec", apply_mediacodec }, +#endif + { nullptr, nullptr } }; const struct { const char *name; diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderoptions_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderoptions_p.h new file mode 100644 index 000000000..005ad7652 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderoptions_p.h @@ -0,0 +1,32 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGENCODEROPTIONS_P_H +#define QFFMPEGENCODEROPTIONS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeghwaccel_p.h" +#include "qvideoframeformat.h" +#include "private/qplatformmediarecorder_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +void applyVideoEncoderOptions(const QMediaEncoderSettings &settings, const QByteArray &codecName, AVCodecContext *codec, AVDictionary **opts); +void applyAudioEncoderOptions(const QMediaEncoderSettings &settings, const QByteArray &codecName, AVCodecContext *codec, AVDictionary **opts); + +} + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderthread.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderthread.cpp new file mode 100644 index 000000000..61fe954c8 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderthread.cpp @@ -0,0 +1,40 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include "qffmpegencoderthread_p.h" +#include "qmetaobject.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +EncoderThread::EncoderThread(RecordingEngine &recordingEngine) : m_recordingEngine(recordingEngine) +{ +} + +void EncoderThread::setPaused(bool paused) +{ + auto guard = lockLoopData(); + m_paused = paused; +} + +void EncoderThread::setAutoStop(bool autoStop) +{ + auto guard = lockLoopData(); + m_autoStop = autoStop; +} + +void EncoderThread::setEndOfSourceStream() +{ + { + auto guard = lockLoopData(); + m_endOfSourceStream = true; + } + + emit endOfSourceStream(); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#include "moc_qffmpegencoderthread_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderthread_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderthread_p.h new file mode 100644 index 000000000..f1f6b610a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencoderthread_p.h @@ -0,0 +1,72 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGENCODERTHREAD_P_H +#define QFFMPEGENCODERTHREAD_P_H + +#include "qffmpegthread_p.h" +#include "qpointer.h" + +#include "private/qmediainputencoderinterface_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class RecordingEngine; + +class EncoderThread : public ConsumerThread, public QMediaInputEncoderInterface +{ + Q_OBJECT +public: + EncoderThread(RecordingEngine &recordingEngine); + + void setPaused(bool paused); + + void setAutoStop(bool autoStop); + + void setSource(QObject *source) { m_source = source; } + + QObject *source() const { return m_source; } + + bool canPushFrame() const override { return m_canPushFrame.load(std::memory_order_relaxed); } + + void setEndOfSourceStream(); + + bool isEndOfSourceStream() const { return m_endOfSourceStream; } + +protected: + void updateCanPushFrame(); + + virtual bool checkIfCanPushFrame() const = 0; + + void resetEndOfSourceStream() { m_endOfSourceStream = false; } + + auto lockLoopData() + { + return QScopeGuard([this, locker = ConsumerThread::lockLoopData()]() mutable { + const bool autoStopActivated = m_endOfSourceStream && m_autoStop; + const bool canPush = !autoStopActivated && !m_paused && checkIfCanPushFrame(); + locker.unlock(); + if (m_canPushFrame.exchange(canPush, std::memory_order_relaxed) != canPush) + emit canPushFrameChanged(); + }); + } + +Q_SIGNALS: + void canPushFrameChanged(); + void endOfSourceStream(); + +protected: + bool m_paused = false; + bool m_endOfSourceStream = false; + bool m_autoStop = false; + std::atomic_bool m_canPushFrame = false; + RecordingEngine &m_recordingEngine; + QPointer<QObject> m_source; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencodinginitializer.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencodinginitializer.cpp new file mode 100644 index 000000000..4f8c21bd5 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencodinginitializer.cpp @@ -0,0 +1,165 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegencodinginitializer_p.h" +#include "qffmpegrecordingengineutils_p.h" +#include "qffmpegrecordingengine_p.h" +#include "qffmpegaudioinput_p.h" +#include "qvideoframe.h" + +#include "private/qplatformvideoframeinput_p.h" +#include "private/qplatformaudiobufferinput_p.h" +#include "private/qplatformaudiobufferinput_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +EncodingInitializer::EncodingInitializer(RecordingEngine &engine) : m_recordingEngine(engine) { } + +EncodingInitializer::~EncodingInitializer() +{ + for (QObject *source : m_pendingSources) + setEncoderInterface(source, nullptr); +} + +void EncodingInitializer::start(const std::vector<QPlatformAudioBufferInputBase *> &audioSources, + const std::vector<QPlatformVideoSource *> &videoSources) +{ + for (auto source : audioSources) { + if (auto audioInput = qobject_cast<QFFmpegAudioInput *>(source)) + m_recordingEngine.addAudioInput(audioInput); + else if (auto audioBufferInput = qobject_cast<QPlatformAudioBufferInput *>(source)) + addAudioBufferInput(audioBufferInput); + else + Q_ASSERT(!"Undefined source type"); + } + + for (auto source : videoSources) + addVideoSource(source); + + tryStartRecordingEngine(); +} + +void EncodingInitializer::addAudioBufferInput(QPlatformAudioBufferInput *input) +{ + Q_ASSERT(input); + + if (input->audioFormat().isValid()) + m_recordingEngine.addAudioBufferInput(input, {}); + else + addPendingAudioBufferInput(input); +} + +void EncodingInitializer::addPendingAudioBufferInput(QPlatformAudioBufferInput *input) +{ + addPendingSource(input); + + connect(input, &QPlatformAudioBufferInput::destroyed, this, [this, input]() { + erasePendingSource(input, QStringLiteral("Audio source deleted"), true); + }); + + connect(input, &QPlatformAudioBufferInput::newAudioBuffer, this, + [this, input](const QAudioBuffer &buffer) { + if (buffer.isValid()) + erasePendingSource( + input, [&]() { m_recordingEngine.addAudioBufferInput(input, buffer); }); + else + erasePendingSource(input, + QStringLiteral("Audio source has sent the end frame")); + }); +} + +void EncodingInitializer::addVideoSource(QPlatformVideoSource *source) +{ + Q_ASSERT(source); + Q_ASSERT(source->isActive()); + + if (source->frameFormat().isValid()) + m_recordingEngine.addVideoSource(source, {}); + else if (source->hasError()) + emitStreamInitializationError(QStringLiteral("Video source error: ") + + source->errorString()); + else + addPendingVideoSource(source); +} + +void EncodingInitializer::addPendingVideoSource(QPlatformVideoSource *source) +{ + addPendingSource(source); + + connect(source, &QPlatformVideoSource::errorChanged, this, [this, source]() { + if (source->hasError()) + erasePendingSource(source, + QStringLiteral("Videio source error: ") + source->errorString()); + }); + + connect(source, &QPlatformVideoSource::destroyed, this, [this, source]() { + erasePendingSource(source, QStringLiteral("Source deleted"), true); + }); + + connect(source, &QPlatformVideoSource::activeChanged, this, [this, source]() { + if (!source->isActive()) + erasePendingSource(source, QStringLiteral("Video source deactivated")); + }); + + connect(source, &QPlatformVideoSource::newVideoFrame, this, + [this, source](const QVideoFrame &frame) { + if (frame.isValid()) + erasePendingSource(source, + [&]() { m_recordingEngine.addVideoSource(source, frame); }); + else + erasePendingSource(source, + QStringLiteral("Video source has sent the end frame")); + }); +} + +void EncodingInitializer::tryStartRecordingEngine() +{ + if (m_pendingSources.empty()) + m_recordingEngine.start(); +} + +void EncodingInitializer::emitStreamInitializationError(QString error) +{ + emit m_recordingEngine.streamInitializationError( + QMediaRecorder::ResourceError, + QStringLiteral("Video steam initialization error. ") + error); +} + +void EncodingInitializer::addPendingSource(QObject *source) +{ + Q_ASSERT(m_pendingSources.count(source) == 0); + + setEncoderInterface(source, this); + m_pendingSources.emplace(source); +} + +template <typename F> +void EncodingInitializer::erasePendingSource(QObject *source, F &&functionOrError, bool destroyed) +{ + const auto erasedCount = m_pendingSources.erase(source); + if (erasedCount == 0) + return; // got a queued event, just ignore it. + + if (!destroyed) { + setEncoderInterface(source, nullptr); + disconnect(source, nullptr, this, nullptr); + } + + if constexpr (std::is_invocable_v<F>) + functionOrError(); + else + emitStreamInitializationError(functionOrError); + + tryStartRecordingEngine(); +} + +bool EncodingInitializer::canPushFrame() const +{ + return true; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencodinginitializer_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencodinginitializer_p.h new file mode 100644 index 000000000..e3bcb3428 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegencodinginitializer_p.h @@ -0,0 +1,77 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QENCODINGINITIALIZER_P_H +#define QENCODINGINITIALIZER_P_H + +#include "qobject.h" +#include "private/qmediainputencoderinterface_p.h" +#include <unordered_set> +#include <vector> + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +QT_BEGIN_NAMESPACE + +class QFFmpegAudioInput; +class QPlatformVideoSource; +class QPlatformAudioBufferInput; +class QPlatformAudioBufferInputBase; +class QMediaInputEncoderInterface; + +namespace QFFmpeg { + +class RecordingEngine; + +// Initializes RecordingEngine with audio and video sources, potentially lazily +// upon first frame arrival if video frame format is not pre-determined. +class EncodingInitializer : public QObject, private QMediaInputEncoderInterface +{ +public: + EncodingInitializer(RecordingEngine &engine); + + ~EncodingInitializer() override; + + void start(const std::vector<QPlatformAudioBufferInputBase *> &audioSources, + const std::vector<QPlatformVideoSource *> &videoSources); + +private: + void addAudioBufferInput(QPlatformAudioBufferInput *input); + + void addPendingAudioBufferInput(QPlatformAudioBufferInput *input); + + void addVideoSource(QPlatformVideoSource *source); + + void addPendingVideoSource(QPlatformVideoSource *source); + + void addPendingSource(QObject *source); + + void tryStartRecordingEngine(); + +private: + void emitStreamInitializationError(QString error); + + template <typename F> + void erasePendingSource(QObject *source, F &&functionOrError, bool destroyed = false); + + bool canPushFrame() const override; + +private: + RecordingEngine &m_recordingEngine; + std::unordered_set<QObject *> m_pendingSources; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QENCODINGINITIALIZER_P_H diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegmuxer.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegmuxer.cpp new file mode 100644 index 000000000..dbb96d00c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegmuxer.cpp @@ -0,0 +1,64 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include "qffmpegmuxer_p.h" +#include "qffmpegrecordingengine_p.h" +#include "qffmpegrecordingengineutils_p.h" +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +Q_STATIC_LOGGING_CATEGORY(qLcFFmpegMuxer, "qt.multimedia.ffmpeg.muxer"); + +Muxer::Muxer(RecordingEngine *encoder) : m_encoder(encoder) +{ + setObjectName(QLatin1String("Muxer")); +} + +void Muxer::addPacket(AVPacketUPtr packet) +{ + { + QMutexLocker locker = lockLoopData(); + m_packetQueue.push(std::move(packet)); + } + + // qCDebug(qLcFFmpegEncoder) << "Muxer::addPacket" << packet->pts << packet->stream_index; + dataReady(); +} + +AVPacketUPtr Muxer::takePacket() +{ + QMutexLocker locker = lockLoopData(); + return dequeueIfPossible(m_packetQueue); +} + +void Muxer::init() +{ + qCDebug(qLcFFmpegMuxer) << "Muxer::init started thread."; +} + +void Muxer::cleanup() +{ + while (!m_packetQueue.empty()) + processOne(); +} + +bool QFFmpeg::Muxer::hasData() const +{ + return !m_packetQueue.empty(); +} + +void Muxer::processOne() +{ + auto packet = takePacket(); + // qCDebug(qLcFFmpegEncoder) << "writing packet to file" << packet->pts << packet->duration << + // packet->stream_index; + + // the function takes ownership for the packet + av_interleaved_write_frame(m_encoder->avFormatContext(), packet.release()); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegmuxer_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegmuxer_p.h new file mode 100644 index 000000000..4f8f4d27a --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegmuxer_p.h @@ -0,0 +1,41 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGMUXER_P_H +#define QFFMPEGMUXER_P_H + +#include "qffmpegthread_p.h" +#include "qffmpeg_p.h" +#include <queue> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +class RecordingEngine; + +class Muxer : public ConsumerThread +{ +public: + Muxer(RecordingEngine *encoder); + + void addPacket(AVPacketUPtr packet); + +private: + AVPacketUPtr takePacket(); + + void init() override; + void cleanup() override; + bool hasData() const override; + void processOne() override; + +private: + std::queue<AVPacketUPtr> m_packetQueue; + + RecordingEngine *m_encoder; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp new file mode 100644 index 000000000..469cd1c48 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine.cpp @@ -0,0 +1,278 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include "qffmpegrecordingengine_p.h" +#include "qffmpegencodinginitializer_p.h" +#include "qffmpegaudioencoder_p.h" +#include "qffmpegaudioinput_p.h" +#include "qffmpegrecordingengineutils_p.h" + +#include "private/qmultimediautils_p.h" +#include "private/qplatformaudiobufferinput_p.h" +#include "private/qplatformvideosource_p.h" +#include "private/qplatformvideoframeinput_p.h" + +#include "qdebug.h" +#include "qffmpegvideoencoder_p.h" +#include "qffmpegmediametadata_p.h" +#include "qffmpegmuxer_p.h" +#include "qloggingcategory.h" + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcFFmpegEncoder, "qt.multimedia.ffmpeg.encoder"); + +namespace QFFmpeg +{ + +RecordingEngine::RecordingEngine(const QMediaEncoderSettings &settings, + std::unique_ptr<EncodingFormatContext> context) + : m_settings(settings), m_formatContext(std::move(context)), m_muxer(new Muxer(this)) +{ + Q_ASSERT(m_formatContext); + Q_ASSERT(m_formatContext->isAVIOOpen()); +} + +RecordingEngine::~RecordingEngine() +{ +} + +void RecordingEngine::addAudioInput(QFFmpegAudioInput *input) +{ + Q_ASSERT(input); + + if (input->device.isNull()) { + emit streamInitializationError(QMediaRecorder::ResourceError, + QLatin1StringView("Audio device is null")); + return; + } + + const QAudioFormat format = input->device.preferredFormat(); + + if (!format.isValid()) { + emit streamInitializationError( + QMediaRecorder::FormatError, + QLatin1StringView("Audio device has invalid preferred format")); + return; + } + + AudioEncoder *audioEncoder = createAudioEncoder(format); + connectEncoderToSource(audioEncoder, input); + + input->setRunning(true); +} + +void RecordingEngine::addAudioBufferInput(QPlatformAudioBufferInput *input, + const QAudioBuffer &firstBuffer) +{ + Q_ASSERT(input); + const QAudioFormat format = firstBuffer.isValid() ? firstBuffer.format() : input->audioFormat(); + + AudioEncoder *audioEncoder = createAudioEncoder(format); + + // set the buffer before connecting to avoid potential races + if (firstBuffer.isValid()) + audioEncoder->addBuffer(firstBuffer); + + connectEncoderToSource(audioEncoder, input); +} + +AudioEncoder *RecordingEngine::createAudioEncoder(const QAudioFormat &format) +{ + Q_ASSERT(format.isValid()); + + auto audioEncoder = new AudioEncoder(*this, format, m_settings); + m_audioEncoders.push_back(audioEncoder); + connect(audioEncoder, &EncoderThread::endOfSourceStream, this, + &RecordingEngine::handleSourceEndOfStream); + if (m_autoStop) + audioEncoder->setAutoStop(true); + + return audioEncoder; +} + +void RecordingEngine::addVideoSource(QPlatformVideoSource *source, const QVideoFrame &firstFrame) +{ + QVideoFrameFormat frameFormat = + firstFrame.isValid() ? firstFrame.surfaceFormat() : source->frameFormat(); + + Q_ASSERT(frameFormat.isValid()); + + if (firstFrame.isValid() && frameFormat.streamFrameRate() <= 0.f) { + const qint64 startTime = firstFrame.startTime(); + const qint64 endTime = firstFrame.endTime(); + if (startTime != -1 && endTime > startTime) + frameFormat.setStreamFrameRate(static_cast<qreal>(VideoFrameTimeBase) + / (endTime - startTime)); + } + + std::optional<AVPixelFormat> hwPixelFormat = source->ffmpegHWPixelFormat() + ? AVPixelFormat(*source->ffmpegHWPixelFormat()) + : std::optional<AVPixelFormat>{}; + + qCDebug(qLcFFmpegEncoder) << "adding video source" << source->metaObject()->className() << ":" + << "pixelFormat=" << frameFormat.pixelFormat() + << "frameSize=" << frameFormat.frameSize() + << "frameRate=" << frameFormat.streamFrameRate() + << "ffmpegHWPixelFormat=" << (hwPixelFormat ? *hwPixelFormat : AV_PIX_FMT_NONE); + + auto veUPtr = std::make_unique<VideoEncoder>(*this, m_settings, frameFormat, hwPixelFormat); + if (!veUPtr->isValid()) { + emit streamInitializationError(QMediaRecorder::FormatError, + QLatin1StringView("Cannot initialize encoder")); + return; + } + + auto videoEncoder = veUPtr.release(); + m_videoEncoders.append(videoEncoder); + if (m_autoStop) + videoEncoder->setAutoStop(true); + + connect(videoEncoder, &EncoderThread::endOfSourceStream, this, + &RecordingEngine::handleSourceEndOfStream); + + // set the frame before connecting to avoid potential races + if (firstFrame.isValid()) + videoEncoder->addFrame(firstFrame); + + connectEncoderToSource(videoEncoder, source); +} + +void RecordingEngine::start() +{ + Q_ASSERT(m_initializer); + m_initializer.reset(); + + if (m_audioEncoders.empty() && m_videoEncoders.empty()) { + emit sessionError(QMediaRecorder::ResourceError, + QLatin1StringView("No valid stream found for encoding")); + return; + } + + qCDebug(qLcFFmpegEncoder) << "RecordingEngine::start!"; + + avFormatContext()->metadata = QFFmpegMetaData::toAVMetaData(m_metaData); + + Q_ASSERT(!m_isHeaderWritten); + + int res = avformat_write_header(avFormatContext(), nullptr); + if (res < 0) { + qWarning() << "could not write header, error:" << res << err2str(res); + emit sessionError(QMediaRecorder::ResourceError, + QLatin1StringView("Cannot start writing the stream")); + return; + } + + m_isHeaderWritten = true; + + qCDebug(qLcFFmpegEncoder) << "stream header is successfully written"; + + m_muxer->start(); + + forEachEncoder([](QThread *thread) { thread->start(); }); +} + +void RecordingEngine::initialize(const std::vector<QPlatformAudioBufferInputBase *> &audioSources, + const std::vector<QPlatformVideoSource *> &videoSources) +{ + qCDebug(qLcFFmpegEncoder) << ">>>>>>>>>>>>>>> initialize"; + + m_initializer = std::make_unique<EncodingInitializer>(*this); + m_initializer->start(audioSources, videoSources); +} + +RecordingEngine::EncodingFinalizer::EncodingFinalizer(RecordingEngine &recordingEngine) + : m_recordingEngine(recordingEngine) +{ + connect(this, &QThread::finished, this, &QObject::deleteLater); +} + +void RecordingEngine::EncodingFinalizer::run() +{ + m_recordingEngine.forEachEncoder(&EncoderThread::stopAndDelete); + m_recordingEngine.m_muxer->stopAndDelete(); + + if (m_recordingEngine.m_isHeaderWritten) { + const int res = av_write_trailer(m_recordingEngine.avFormatContext()); + if (res < 0) { + const auto errorDescription = err2str(res); + qCWarning(qLcFFmpegEncoder) << "could not write trailer" << res << errorDescription; + emit m_recordingEngine.sessionError(QMediaRecorder::FormatError, + QLatin1String("Cannot write trailer: ") + + errorDescription); + } + } + // else ffmpeg might crash + + // close AVIO before emitting finalizationDone. + m_recordingEngine.m_formatContext->closeAVIO(); + + qCDebug(qLcFFmpegEncoder) << " done finalizing."; + emit m_recordingEngine.finalizationDone(); + auto recordingEnginePtr = &m_recordingEngine; + delete recordingEnginePtr; +} + +void RecordingEngine::finalize() +{ + qCDebug(qLcFFmpegEncoder) << ">>>>>>>>>>>>>>> finalize"; + + m_initializer.reset(); + + forEachEncoder(&disconnectEncoderFromSource); + + auto *finalizer = new EncodingFinalizer(*this); + finalizer->start(); +} + +void RecordingEngine::setPaused(bool paused) +{ + forEachEncoder(&EncoderThread::setPaused, paused); +} + +void RecordingEngine::setAutoStop(bool autoStop) +{ + m_autoStop = autoStop; + forEachEncoder(&EncoderThread::setAutoStop, autoStop); + handleSourceEndOfStream(); +} + +void RecordingEngine::setMetaData(const QMediaMetaData &metaData) +{ + m_metaData = metaData; +} + +void RecordingEngine::newTimeStamp(qint64 time) +{ + QMutexLocker locker(&m_timeMutex); + if (time > m_timeRecorded) { + m_timeRecorded = time; + emit durationChanged(time); + } +} + +bool RecordingEngine::isEndOfSourceStreams() const +{ + auto isAtEnd = [](EncoderThread *encoder) { return encoder->isEndOfSourceStream(); }; + return std::all_of(m_videoEncoders.cbegin(), m_videoEncoders.cend(), isAtEnd) + && std::all_of(m_audioEncoders.cbegin(), m_audioEncoders.cend(), isAtEnd); +} + +void RecordingEngine::handleSourceEndOfStream() +{ + if (m_autoStop && isEndOfSourceStreams()) + emit autoStopped(); +} + +template <typename F, typename... Args> +void RecordingEngine::forEachEncoder(F &&f, Args &&...args) +{ + for (AudioEncoder *audioEncoder : m_audioEncoders) + std::invoke(f, audioEncoder, args...); + for (VideoEncoder *videoEncoder : m_videoEncoders) + std::invoke(f, videoEncoder, args...); +} +} + +QT_END_NAMESPACE + +#include "moc_qffmpegrecordingengine_p.cpp" diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine_p.h new file mode 100644 index 000000000..ce3aaa6bb --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengine_p.h @@ -0,0 +1,121 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGENCODER_P_H +#define QFFMPEGENCODER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpegthread_p.h" +#include "qffmpegencodingformatcontext_p.h" + +#include <private/qplatformmediarecorder_p.h> +#include <qmediarecorder.h> + +QT_BEGIN_NAMESPACE + +class QFFmpegAudioInput; +class QPlatformAudioBufferInput; +class QPlatformAudioBufferInputBase; +class QVideoFrame; +class QAudioBuffer; +class QPlatformVideoSource; + +namespace QFFmpeg +{ + +class RecordingEngine; +class Muxer; +class AudioEncoder; +class VideoEncoder; +class VideoFrameEncoder; +class EncodingInitializer; + +class RecordingEngine : public QObject +{ + Q_OBJECT +public: + RecordingEngine(const QMediaEncoderSettings &settings, std::unique_ptr<EncodingFormatContext> context); + ~RecordingEngine(); + + void initialize(const std::vector<QPlatformAudioBufferInputBase *> &audioSources, + const std::vector<QPlatformVideoSource *> &videoSources); + void finalize(); + + void setPaused(bool p); + + void setAutoStop(bool autoStop); + + bool autoStop() const { return m_autoStop; } + + void setMetaData(const QMediaMetaData &metaData); + AVFormatContext *avFormatContext() { return m_formatContext->avFormatContext(); } + Muxer *getMuxer() { return m_muxer; } + + bool isEndOfSourceStreams() const; + +public Q_SLOTS: + void newTimeStamp(qint64 time); + +Q_SIGNALS: + void durationChanged(qint64 duration); + void sessionError(QMediaRecorder::Error code, const QString &description); + void streamInitializationError(QMediaRecorder::Error code, const QString &description); + void finalizationDone(); + void autoStopped(); + +private: + class EncodingFinalizer : public QThread + { + public: + EncodingFinalizer(RecordingEngine &recordingEngine); + + void run() override; + + private: + RecordingEngine &m_recordingEngine; + }; + + friend class EncodingInitializer; + void addAudioInput(QFFmpegAudioInput *input); + void addAudioBufferInput(QPlatformAudioBufferInput *input, const QAudioBuffer &firstBuffer); + AudioEncoder *createAudioEncoder(const QAudioFormat &format); + + void addVideoSource(QPlatformVideoSource *source, const QVideoFrame &firstFrame); + void handleSourceEndOfStream(); + + void start(); + + template <typename F, typename... Args> + void forEachEncoder(F &&f, Args &&...args); + +private: + QMediaEncoderSettings m_settings; + QMediaMetaData m_metaData; + std::unique_ptr<EncodingFormatContext> m_formatContext; + Muxer *m_muxer = nullptr; + + QList<AudioEncoder *> m_audioEncoders; + QList<VideoEncoder *> m_videoEncoders; + std::unique_ptr<EncodingInitializer> m_initializer; + + QMutex m_timeMutex; + qint64 m_timeRecorded = 0; + + bool m_isHeaderWritten = false; + bool m_autoStop = false; +}; + +} + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengineutils.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengineutils.cpp new file mode 100644 index 000000000..6c2ba8b15 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengineutils.cpp @@ -0,0 +1,63 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "recordingengine/qffmpegrecordingengineutils_p.h" +#include "recordingengine/qffmpegencoderthread_p.h" +#include "private/qplatformaudiobufferinput_p.h" +#include "private/qplatformvideoframeinput_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +template <typename F> +void doWithMediaFrameInput(QObject *source, F &&f) +{ + if (auto videoFrameInput = qobject_cast<QPlatformVideoFrameInput *>(source)) + f(videoFrameInput); + else if (auto audioBufferInput = qobject_cast<QPlatformAudioBufferInput *>(source)) + f(audioBufferInput); +} + +void setEncoderInterface(QObject *source, QMediaInputEncoderInterface *interface) +{ + doWithMediaFrameInput(source, [&](auto source) { + using Source = std::remove_pointer_t<decltype(source)>; + + source->setEncoderInterface(interface); + if (interface) + // Postpone emit 'encoderUpdated' as the encoding pipeline may be not + // completely ready at the moment. The case is calling QMediaRecorder::stop + // upon handling 'readyToSendFrame' + QMetaObject::invokeMethod(source, &Source::encoderUpdated, Qt::QueuedConnection); + else + emit source->encoderUpdated(); + }); +} + +void setEncoderUpdateConnection(QObject *source, EncoderThread *encoder) +{ + doWithMediaFrameInput(source, [&](auto source) { + using Source = std::remove_pointer_t<decltype(source)>; + QObject::connect(encoder, &EncoderThread::canPushFrameChanged, source, + &Source::encoderUpdated); + }); +} + +void disconnectEncoderFromSource(EncoderThread *encoder) +{ + QObject *source = encoder->source(); + if (!source) + return; + + // We should address the dependency AudioEncoder from QFFmpegAudioInput to + // set null source here. + // encoder->setSource(nullptr); + + QObject::disconnect(source, nullptr, encoder, nullptr); + setEncoderInterface(source, nullptr); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengineutils_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengineutils_p.h new file mode 100644 index 000000000..a60f81696 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegrecordingengineutils_p.h @@ -0,0 +1,81 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QFFMPEGRECORDINGENGINEUTILS_P_H +#define QFFMPEGRECORDINGENGINEUTILS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qobject.h" +#include <queue> + +QT_BEGIN_NAMESPACE + +class QMediaInputEncoderInterface; +class QPlatformVideoSource; + +namespace QFFmpeg { + +constexpr qint64 VideoFrameTimeBase = 1000000; // us in sec + +class EncoderThread; + +template <typename T> +T dequeueIfPossible(std::queue<T> &queue) +{ + if (queue.empty()) + return T{}; + + auto result = std::move(queue.front()); + queue.pop(); + return result; +} + +void setEncoderInterface(QObject *source, QMediaInputEncoderInterface *interface); + +void setEncoderUpdateConnection(QObject *source, EncoderThread *encoder); + +template <typename Encoder, typename Source> +void connectEncoderToSource(Encoder *encoder, Source *source) +{ + Q_ASSERT(!encoder->source()); + encoder->setSource(source); + + if constexpr (std::is_same_v<Source, QPlatformVideoSource>) { + QObject::connect(source, &Source::newVideoFrame, encoder, &Encoder::addFrame, + Qt::DirectConnection); + + QObject::connect(source, &Source::activeChanged, encoder, [=]() { + if (!source->isActive()) + encoder->setEndOfSourceStream(); + }); + } else { + QObject::connect(source, &Source::newAudioBuffer, encoder, &Encoder::addBuffer, + Qt::DirectConnection); + } + + // TODO: + // QObject::connect(source, &Source::disconnectedFromSession, encoder, [=]() { + // encoder->setSourceEndOfStream(); + // }); + + setEncoderUpdateConnection(source, encoder); + setEncoderInterface(source, encoder); +} + +void disconnectEncoderFromSource(EncoderThread *encoder); + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGRECORDINGENGINEUTILS_P_H diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp new file mode 100644 index 000000000..27706580b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder.cpp @@ -0,0 +1,259 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include "qffmpegvideoencoder_p.h" +#include "qffmpegmuxer_p.h" +#include "qffmpegvideobuffer_p.h" +#include "qffmpegrecordingengine_p.h" +#include "qffmpegvideoframeencoder_p.h" +#include "qffmpegrecordingengineutils_p.h" +#include "private/qvideoframe_p.h" +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +Q_STATIC_LOGGING_CATEGORY(qLcFFmpegVideoEncoder, "qt.multimedia.ffmpeg.videoencoder"); + +VideoEncoder::VideoEncoder(RecordingEngine &recordingEngine, const QMediaEncoderSettings &settings, + const QVideoFrameFormat &format, std::optional<AVPixelFormat> hwFormat) + : EncoderThread(recordingEngine) +{ + setObjectName(QLatin1String("VideoEncoder")); + + const AVPixelFormat swFormat = QFFmpegVideoBuffer::toAVPixelFormat(format.pixelFormat()); + qreal frameRate = format.streamFrameRate(); + if (frameRate <= 0.) { + qWarning() << "Invalid frameRate" << frameRate << "; Using the default instead"; + + // set some default frame rate since ffmpeg has UB if it's 0. + frameRate = 30.; + } + + VideoFrameEncoder::SourceParams sourceParams; + sourceParams.size = format.frameSize(); + sourceParams.format = hwFormat && *hwFormat != AV_PIX_FMT_NONE ? *hwFormat : swFormat; + sourceParams.swFormat = swFormat; + sourceParams.rotation = format.rotation(); + sourceParams.xMirrored = format.isMirrored(); + sourceParams.yMirrored = format.scanLineDirection() == QVideoFrameFormat::BottomToTop; + sourceParams.frameRate = frameRate; + sourceParams.colorTransfer = QFFmpeg::toAvColorTransfer(format.colorTransfer()); + sourceParams.colorSpace = QFFmpeg::toAvColorSpace(format.colorSpace()); + sourceParams.colorRange = QFFmpeg::toAvColorRange(format.colorRange()); + + m_frameEncoder = + VideoFrameEncoder::create(settings, sourceParams, recordingEngine.avFormatContext()); +} + +VideoEncoder::~VideoEncoder() = default; + +bool VideoEncoder::isValid() const +{ + return m_frameEncoder != nullptr; +} + +void VideoEncoder::addFrame(const QVideoFrame &frame) +{ + if (!frame.isValid()) { + setEndOfSourceStream(); + return; + } + + { + auto guard = lockLoopData(); + + resetEndOfSourceStream(); + + if (m_paused) { + m_shouldAdjustTimeBaseForNextFrame = true; + return; + } + + // Drop frames if encoder can not keep up with the video source data rate; + // canPushFrame might be used instead + const bool queueFull = m_videoFrameQueue.size() >= m_maxQueueSize; + + if (queueFull) { + qCDebug(qLcFFmpegVideoEncoder) << "RecordingEngine frame queue full. Frame lost."; + return; + } + + m_videoFrameQueue.push({ frame, m_shouldAdjustTimeBaseForNextFrame }); + m_shouldAdjustTimeBaseForNextFrame = false; + } + + dataReady(); +} + +VideoEncoder::FrameInfo VideoEncoder::takeFrame() +{ + auto guard = lockLoopData(); + return dequeueIfPossible(m_videoFrameQueue); +} + +void VideoEncoder::retrievePackets() +{ + if (!m_frameEncoder) + return; + while (auto packet = m_frameEncoder->retrievePacket()) + m_recordingEngine.getMuxer()->addPacket(std::move(packet)); +} + +void VideoEncoder::init() +{ + Q_ASSERT(isValid()); + + qCDebug(qLcFFmpegVideoEncoder) << "VideoEncoder::init started video device thread."; + bool ok = m_frameEncoder->open(); + if (!ok) + emit m_recordingEngine.sessionError(QMediaRecorder::ResourceError, + "Could not initialize encoder"); +} + +void VideoEncoder::cleanup() +{ + while (!m_videoFrameQueue.empty()) + processOne(); + if (m_frameEncoder) { + while (m_frameEncoder->sendFrame(nullptr) == AVERROR(EAGAIN)) + retrievePackets(); + retrievePackets(); + } +} + +bool VideoEncoder::hasData() const +{ + return !m_videoFrameQueue.empty(); +} + +struct QVideoFrameHolder +{ + QVideoFrame f; + QImage i; +}; + +static void freeQVideoFrame(void *opaque, uint8_t *) +{ + delete reinterpret_cast<QVideoFrameHolder *>(opaque); +} + +void VideoEncoder::processOne() +{ + retrievePackets(); + + FrameInfo frameInfo = takeFrame(); + QVideoFrame &frame = frameInfo.frame; + Q_ASSERT(frame.isValid()); + + if (!isValid()) + return; + + // qCDebug(qLcFFmpegEncoder) << "new video buffer" << frame.startTime(); + + AVFrameUPtr avFrame; + + auto *videoBuffer = dynamic_cast<QFFmpegVideoBuffer *>(QVideoFramePrivate::hwBuffer(frame)); + if (videoBuffer) { + // ffmpeg video buffer, let's use the native AVFrame stored in there + auto *hwFrame = videoBuffer->getHWFrame(); + if (hwFrame && hwFrame->format == m_frameEncoder->sourceFormat()) + avFrame.reset(av_frame_clone(hwFrame)); + } + + if (!avFrame) { + frame.map(QtVideo::MapMode::ReadOnly); + auto size = frame.size(); + avFrame = makeAVFrame(); + avFrame->format = m_frameEncoder->sourceFormat(); + avFrame->width = size.width(); + avFrame->height = size.height(); + + for (int i = 0; i < 4; ++i) { + avFrame->data[i] = const_cast<uint8_t *>(frame.bits(i)); + avFrame->linesize[i] = frame.bytesPerLine(i); + } + + // TODO: investigate if we need to set color params to AVFrame. + // Setting only codec carameters might be sufficient. + // What happens if frame color params are set and not equal codec prms? + // + // QVideoFrameFormat format = frame.surfaceFormat(); + // avFrame->color_trc = QFFmpeg::toAvColorTransfer(format.colorTransfer()); + // avFrame->colorspace = QFFmpeg::toAvColorSpace(format.colorSpace()); + // avFrame->color_range = QFFmpeg::toAvColorRange(format.colorRange()); + + QImage img; + if (frame.pixelFormat() == QVideoFrameFormat::Format_Jpeg) { + // the QImage is cached inside the video frame, so we can take the pointer to the image + // data here + img = frame.toImage(); + avFrame->data[0] = (uint8_t *)img.bits(); + avFrame->linesize[0] = img.bytesPerLine(); + } + + Q_ASSERT(avFrame->data[0]); + // ensure the video frame and it's data is alive as long as it's being used in the encoder + avFrame->opaque_ref = av_buffer_create(nullptr, 0, freeQVideoFrame, + new QVideoFrameHolder{ frame, img }, 0); + } + + const auto [startTime, endTime] = frameTimeStamps(frame); + + if (frameInfo.shouldAdjustTimeBase) { + m_baseTime += startTime - m_lastFrameTime; + qCDebug(qLcFFmpegVideoEncoder) + << ">>>> adjusting base time to" << m_baseTime << startTime << m_lastFrameTime; + } + + const qint64 time = startTime - m_baseTime; + m_lastFrameTime = endTime; + + setAVFrameTime(*avFrame, m_frameEncoder->getPts(time), m_frameEncoder->getTimeBase()); + + m_recordingEngine.newTimeStamp(time / 1000); + + qCDebug(qLcFFmpegVideoEncoder) + << ">>> sending frame" << avFrame->pts << time << m_lastFrameTime; + int ret = m_frameEncoder->sendFrame(std::move(avFrame)); + if (ret < 0) { + qCDebug(qLcFFmpegVideoEncoder) << "error sending frame" << ret << err2str(ret); + emit m_recordingEngine.sessionError(QMediaRecorder::ResourceError, err2str(ret)); + } +} + +bool VideoEncoder::checkIfCanPushFrame() const +{ + if (isRunning()) + return m_videoFrameQueue.size() < m_maxQueueSize; + if (!isFinished()) + return m_videoFrameQueue.empty(); + + return false; +} + +std::pair<qint64, qint64> VideoEncoder::frameTimeStamps(const QVideoFrame &frame) const +{ + qint64 startTime = frame.startTime(); + qint64 endTime = frame.endTime(); + + if (startTime == -1) { + startTime = m_lastFrameTime; + endTime = -1; + } + + if (endTime == -1) { + qreal frameRate = frame.streamFrameRate(); + if (frameRate <= 0.) + frameRate = m_frameEncoder->settings().videoFrameRate(); + + Q_ASSERT(frameRate > 0.f); + endTime = startTime + static_cast<qint64>(std::round(VideoFrameTimeBase / frameRate)); + } + + return { startTime, endTime }; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder_p.h new file mode 100644 index 000000000..ff6a74fc8 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoder_p.h @@ -0,0 +1,64 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGVIDEOENCODER_P_H +#define QFFMPEGVIDEOENCODER_P_H + +#include "qffmpegencoderthread_p.h" +#include "qffmpeg_p.h" +#include <qvideoframe.h> +#include <queue> + +QT_BEGIN_NAMESPACE + +class QVideoFrameFormat; +class QMediaEncoderSettings; + +namespace QFFmpeg { +class VideoFrameEncoder; + +class VideoEncoder : public EncoderThread +{ +public: + VideoEncoder(RecordingEngine &recordingEngine, const QMediaEncoderSettings &settings, + const QVideoFrameFormat &format, std::optional<AVPixelFormat> hwFormat); + ~VideoEncoder() override; + + bool isValid() const; + + void addFrame(const QVideoFrame &frame); + +protected: + bool checkIfCanPushFrame() const override; + +private: + struct FrameInfo + { + QVideoFrame frame; + bool shouldAdjustTimeBase = false; + }; + + FrameInfo takeFrame(); + void retrievePackets(); + + void init() override; + void cleanup() override; + bool hasData() const override; + void processOne() override; + + std::pair<qint64, qint64> frameTimeStamps(const QVideoFrame &frame) const; + +private: + std::queue<FrameInfo> m_videoFrameQueue; + const size_t m_maxQueueSize = 10; // Arbitrarily chosen to limit memory usage (332 MB @ 4K) + + std::unique_ptr<VideoFrameEncoder> m_frameEncoder; + qint64 m_baseTime = 0; + bool m_shouldAdjustTimeBaseForNextFrame = true; + qint64 m_lastFrameTime = 0; +}; + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoderutils.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoderutils.cpp new file mode 100644 index 000000000..69073688b --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoderutils.cpp @@ -0,0 +1,214 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegvideoencoderutils_p.h" +#include "private/qmultimediautils_p.h" + +extern "C" { +#include <libavutil/pixdesc.h> +} + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +static AVScore calculateTargetSwFormatScore(const AVPixFmtDescriptor *sourceSwFormatDesc, + AVPixelFormat fmt) +{ + // determine the format used by the encoder. + // We prefer YUV422 based formats such as NV12 or P010. Selection trues to find the best + // matching format for the encoder depending on the bit depth of the source format + + const auto *desc = av_pix_fmt_desc_get(fmt); + if (!desc) + return NotSuitableAVScore; + + const int sourceDepth = sourceSwFormatDesc ? sourceSwFormatDesc->comp[0].depth : 0; + + if (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) + // we really don't want HW accelerated formats here + return NotSuitableAVScore; + + auto score = DefaultAVScore; + + if (desc == sourceSwFormatDesc) + // prefer exact matches + score += 10; + if (desc->comp[0].depth == sourceDepth) + score += 100; + else if (desc->comp[0].depth < sourceDepth) + score -= 100 + (sourceDepth - desc->comp[0].depth); + if (desc->log2_chroma_h == 1) + score += 1; + if (desc->log2_chroma_w == 1) + score += 1; + if (desc->flags & AV_PIX_FMT_FLAG_BE) + score -= 10; + if (desc->flags & AV_PIX_FMT_FLAG_PAL) + // we don't want paletted formats + score -= 10000; + if (desc->flags & AV_PIX_FMT_FLAG_RGB) + // we don't want RGB formats + score -= 1000; + + // qCDebug(qLcVideoFrameEncoder) + // << "checking format" << fmt << Qt::hex << desc->flags << desc->comp[0].depth + // << desc->log2_chroma_h << desc->log2_chroma_w << "score:" << score; + + return score; +} + +static auto targetSwFormatScoreCalculator(AVPixelFormat sourceFormat) +{ + const auto sourceSwFormatDesc = av_pix_fmt_desc_get(sourceFormat); + return [=](AVPixelFormat fmt) { return calculateTargetSwFormatScore(sourceSwFormatDesc, fmt); }; +} + +static bool isHwFormatAcceptedByCodec(AVPixelFormat pixFormat) +{ + switch (pixFormat) { + case AV_PIX_FMT_MEDIACODEC: + // Mediacodec doesn't accept AV_PIX_FMT_MEDIACODEC (QTBUG-116836) + return false; + default: + return true; + } +} + +AVPixelFormat findTargetSWFormat(AVPixelFormat sourceSWFormat, const AVCodec *codec, + const HWAccel &accel) +{ + auto scoreCalculator = targetSwFormatScoreCalculator(sourceSWFormat); + + const auto constraints = accel.constraints(); + if (constraints && constraints->valid_sw_formats) + return findBestAVValue(constraints->valid_sw_formats, scoreCalculator).first; + + // Some codecs, e.g. mediacodec, don't expose constraints, let's find the format in + // codec->pix_fmts + if (codec->pix_fmts) + return findBestAVValue(codec->pix_fmts, scoreCalculator).first; + + return AV_PIX_FMT_NONE; +} + +AVPixelFormat findTargetFormat(AVPixelFormat sourceFormat, AVPixelFormat sourceSWFormat, + const AVCodec *codec, const HWAccel *accel) +{ + Q_UNUSED(sourceFormat); + + if (accel) { + const auto hwFormat = accel->hwFormat(); + + // TODO: handle codec->capabilities & AV_CODEC_CAP_HARDWARE here + if (!isHwFormatAcceptedByCodec(hwFormat)) + return findTargetSWFormat(sourceSWFormat, codec, *accel); + + const auto constraints = accel->constraints(); + if (constraints && hasAVFormat(constraints->valid_hw_formats, hwFormat)) + return hwFormat; + + // Some codecs, don't expose constraints, + // let's find the format in codec->pix_fmts and hw_config + if (isAVFormatSupported(codec, hwFormat)) + return hwFormat; + } + + if (!codec->pix_fmts) { + qWarning() << "Codec pix formats are undefined, it's likely to behave incorrectly"; + + return sourceSWFormat; + } + + auto swScoreCalculator = targetSwFormatScoreCalculator(sourceSWFormat); + return findBestAVValue(codec->pix_fmts, swScoreCalculator).first; +} + +std::pair<const AVCodec *, std::unique_ptr<HWAccel>> findHwEncoder(AVCodecID codecID, + const QSize &resolution) +{ + auto matchesSizeConstraints = [&resolution](const HWAccel &accel) { + const auto constraints = accel.constraints(); + if (!constraints) + return true; + + return resolution.width() >= constraints->min_width + && resolution.height() >= constraints->min_height + && resolution.width() <= constraints->max_width + && resolution.height() <= constraints->max_height; + }; + + // 1st - attempt to find hw accelerated encoder + auto result = HWAccel::findEncoderWithHwAccel(codecID, matchesSizeConstraints); + Q_ASSERT(!!result.first == !!result.second); + + return result; +} + +const AVCodec *findSwEncoder(AVCodecID codecID, AVPixelFormat sourceSWFormat) +{ + auto formatScoreCalculator = targetSwFormatScoreCalculator(sourceSWFormat); + + return findAVEncoder(codecID, [&formatScoreCalculator](const AVCodec *codec) { + if (!codec->pix_fmts) + // codecs without pix_fmts are suspicious + return MinAVScore; + + return findBestAVValue(codec->pix_fmts, formatScoreCalculator).second; + }); +} + +AVRational adjustFrameRate(const AVRational *supportedRates, qreal requestedRate) +{ + auto calcScore = [requestedRate](const AVRational &rate) { + // relative comparison + return qMin(requestedRate * rate.den, qreal(rate.num)) + / qMax(requestedRate * rate.den, qreal(rate.num)); + }; + + const auto result = findBestAVValue(supportedRates, calcScore).first; + if (result.num && result.den) + return result; + + const auto [num, den] = qRealToFraction(requestedRate); + return { num, den }; +} + +AVRational adjustFrameTimeBase(const AVRational *supportedRates, AVRational frameRate) +{ + // TODO: user-specified frame rate might be required. + if (supportedRates) { + auto hasFrameRate = [&]() { + for (auto rate = supportedRates; rate->num && rate->den; ++rate) + if (rate->den == frameRate.den && rate->num == frameRate.num) + return true; + + return false; + }; + + Q_ASSERT(hasFrameRate()); + + return { frameRate.den, frameRate.num }; + } + + constexpr int TimeScaleFactor = 1000; // Allows not to follow fixed rate + return { frameRate.den, frameRate.num * TimeScaleFactor }; +} + +QSize adjustVideoResolution(const AVCodec *codec, QSize requestedResolution) +{ +#ifdef Q_OS_WINDOWS + // TODO: investigate, there might be more encoders not supporting odd resolution + if (strcmp(codec->name, "h264_mf") == 0) { + auto makeEven = [](int size) { return size & ~1; }; + return QSize(makeEven(requestedResolution.width()), makeEven(requestedResolution.height())); + } +#else + Q_UNUSED(codec); +#endif + return requestedResolution; +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoderutils_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoderutils_p.h new file mode 100644 index 000000000..3a16a7de3 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoencoderutils_p.h @@ -0,0 +1,64 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGVIDEOENCODERUTILS_P_H +#define QFFMPEGVIDEOENCODERUTILS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeg_p.h" +#include "qffmpeghwaccel_p.h" + +QT_BEGIN_NAMESPACE + +namespace QFFmpeg { + +AVPixelFormat findTargetSWFormat(AVPixelFormat sourceSWFormat, const AVCodec *codec, + const HWAccel &accel); + +AVPixelFormat findTargetFormat(AVPixelFormat sourceFormat, AVPixelFormat sourceSWFormat, + const AVCodec *codec, const HWAccel *accel); + +std::pair<const AVCodec *, std::unique_ptr<HWAccel>> findHwEncoder(AVCodecID codecID, + const QSize &sourceSize); + +const AVCodec *findSwEncoder(AVCodecID codecID, AVPixelFormat sourceSWFormat); + +/** + * @brief adjustFrameRate get a rational frame rate be requested qreal rate. + * If the codec supports fixed frame rate (non-null supportedRates), + * the function selects the most suitable one, + * otherwise just makes AVRational from qreal. + */ +AVRational adjustFrameRate(const AVRational *supportedRates, qreal requestedRate); + +/** + * @brief adjustFrameTimeBase gets adjusted timebase by a list of supported frame rates + * and an already adjusted frame rate. + * + * Timebase is the fundamental unit of time (in seconds) in terms + * of which frame timestamps are represented. + * For fixed-fps content (non-null supportedRates), + * timebase should be 1/framerate. + * + * For more information, see AVStream::time_base and AVCodecContext::time_base. + * + * The adjusted time base is supposed to be set to stream and codec context. + */ +AVRational adjustFrameTimeBase(const AVRational *supportedRates, AVRational frameRate); + +QSize adjustVideoResolution(const AVCodec *codec, QSize requestedResolution); + +} // namespace QFFmpeg + +QT_END_NAMESPACE + +#endif // QFFMPEGVIDEOENCODERUTILS_P_H diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder.cpp b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder.cpp new file mode 100644 index 000000000..ce2a1af28 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder.cpp @@ -0,0 +1,547 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qffmpegvideoframeencoder_p.h" +#include "qffmpegmediaformatinfo_p.h" +#include "qffmpegencoderoptions_p.h" +#include "qffmpegvideoencoderutils_p.h" +#include <qloggingcategory.h> +#include <QtMultimedia/private/qmaybe_p.h> + +extern "C" { +#include "libavutil/display.h" +} + +QT_BEGIN_NAMESPACE + +Q_STATIC_LOGGING_CATEGORY(qLcVideoFrameEncoder, "qt.multimedia.ffmpeg.videoencoder"); + +namespace QFFmpeg { + +std::unique_ptr<VideoFrameEncoder> +VideoFrameEncoder::create(const QMediaEncoderSettings &encoderSettings, + const SourceParams &sourceParams, AVFormatContext *formatContext) +{ + Q_ASSERT(isSwPixelFormat(sourceParams.swFormat)); + Q_ASSERT(isHwPixelFormat(sourceParams.format) || sourceParams.swFormat == sourceParams.format); + + std::unique_ptr<VideoFrameEncoder> result(new VideoFrameEncoder); + + result->m_settings = encoderSettings; + result->m_sourceSize = sourceParams.size; + result->m_sourceFormat = sourceParams.format; + + // Temporary: check isSwPixelFormat because of android issue (QTBUG-116836) + result->m_sourceSWFormat = + isSwPixelFormat(sourceParams.format) ? sourceParams.format : sourceParams.swFormat; + + if (!result->m_settings.videoResolution().isValid()) + result->m_settings.setVideoResolution(sourceParams.size); + + if (result->m_settings.videoFrameRate() <= 0.) + result->m_settings.setVideoFrameRate(sourceParams.frameRate); + + if (!result->initCodec() || !result->initTargetFormats() + || !result->initCodecContext(sourceParams, formatContext)) { + return nullptr; + } + + // TODO: make VideoFrameEncoder::private and do openning here + // if (!open()) { + // m_error = QMediaRecorder::FormatError; + // m_errorStr = QLatin1StringView("Cannot open codec"); + // return; + // } + + result->updateConversions(); + + return result; +} + +bool VideoFrameEncoder::initCodec() +{ + const auto qVideoCodec = m_settings.videoCodec(); + const auto codecID = QFFmpegMediaFormatInfo::codecIdForVideoCodec(qVideoCodec); + const auto resolution = m_settings.videoResolution(); + + std::tie(m_codec, m_accel) = findHwEncoder(codecID, resolution); + + if (!m_codec) + m_codec = findSwEncoder(codecID, m_sourceSWFormat); + + if (!m_codec) { + qWarning() << "Could not find encoder for codecId" << codecID; + return false; + } + + qCDebug(qLcVideoFrameEncoder) << "found encoder" << m_codec->name << "for id" << m_codec->id; + +#ifdef Q_OS_WINDOWS + // TODO: investigate, there might be more encoders not supporting odd resolution + if (strcmp(m_codec->name, "h264_mf") == 0) { + auto makeEven = [](int size) { return size & ~1; }; + const QSize fixedResolution(makeEven(resolution.width()), makeEven(resolution.height())); + if (fixedResolution != resolution) { + qCDebug(qLcVideoFrameEncoder) << "Fix odd video resolution for codec" << m_codec->name + << ":" << resolution << "->" << fixedResolution; + m_settings.setVideoResolution(fixedResolution); + } + } +#endif + + auto fixedResolution = adjustVideoResolution(m_codec, m_settings.videoResolution()); + if (resolution != fixedResolution) { + qCDebug(qLcVideoFrameEncoder) << "Fix odd video resolution for codec" << m_codec->name + << ":" << resolution << "->" << fixedResolution; + + m_settings.setVideoResolution(fixedResolution); + } + + if (m_codec->supported_framerates && qLcVideoFrameEncoder().isEnabled(QtDebugMsg)) + for (auto rate = m_codec->supported_framerates; rate->num && rate->den; ++rate) + qCDebug(qLcVideoFrameEncoder) << "supported frame rate:" << *rate; + + m_codecFrameRate = adjustFrameRate(m_codec->supported_framerates, m_settings.videoFrameRate()); + qCDebug(qLcVideoFrameEncoder) << "Adjusted frame rate:" << m_codecFrameRate; + + return true; +} + +bool VideoFrameEncoder::initTargetFormats() +{ + m_targetFormat = findTargetFormat(m_sourceFormat, m_sourceSWFormat, m_codec, m_accel.get()); + + if (m_targetFormat == AV_PIX_FMT_NONE) { + qWarning() << "Could not find target format for codecId" << m_codec->id; + return false; + } + + if (isHwPixelFormat(m_targetFormat)) { + Q_ASSERT(m_accel); + + m_targetSWFormat = findTargetSWFormat(m_sourceSWFormat, m_codec, *m_accel); + + if (m_targetSWFormat == AV_PIX_FMT_NONE) { + qWarning() << "Cannot find software target format. sourceSWFormat:" << m_sourceSWFormat + << "targetFormat:" << m_targetFormat; + return false; + } + + m_accel->createFramesContext(m_targetSWFormat, m_settings.videoResolution()); + if (!m_accel->hwFramesContextAsBuffer()) + return false; + } else { + m_targetSWFormat = m_targetFormat; + } + + return true; +} + +VideoFrameEncoder::~VideoFrameEncoder() = default; + +bool VideoFrameEncoder::initCodecContext(const SourceParams &sourceParams, + AVFormatContext *formatContext) +{ + m_stream = avformat_new_stream(formatContext, nullptr); + m_stream->id = formatContext->nb_streams - 1; + //qCDebug(qLcVideoFrameEncoder) << "Video stream: index" << d->stream->id; + m_stream->codecpar->codec_type = AVMEDIA_TYPE_VIDEO; + m_stream->codecpar->codec_id = m_codec->id; + + // Apples HEVC decoders don't like the hev1 tag ffmpeg uses by default, use hvc1 as the more commonly accepted tag + if (m_codec->id == AV_CODEC_ID_HEVC) + m_stream->codecpar->codec_tag = MKTAG('h', 'v', 'c', '1'); + + const auto resolution = m_settings.videoResolution(); + + // ### Fix hardcoded values + m_stream->codecpar->format = m_targetFormat; + m_stream->codecpar->width = resolution.width(); + m_stream->codecpar->height = resolution.height(); + m_stream->codecpar->sample_aspect_ratio = AVRational{ 1, 1 }; + m_stream->codecpar->color_trc = sourceParams.colorTransfer; + m_stream->codecpar->color_space = sourceParams.colorSpace; + m_stream->codecpar->color_range = sourceParams.colorRange; + + if (sourceParams.rotation != QtVideo::Rotation::None || sourceParams.xMirrored + || sourceParams.yMirrored) { + constexpr auto displayMatrixSize = sizeof(int32_t) * 9; + AVPacketSideData sideData = { reinterpret_cast<uint8_t *>(av_malloc(displayMatrixSize)), + displayMatrixSize, AV_PKT_DATA_DISPLAYMATRIX }; + int32_t *matrix = reinterpret_cast<int32_t *>(sideData.data); + av_display_rotation_set(matrix, static_cast<double>(sourceParams.rotation)); + av_display_matrix_flip(matrix, sourceParams.xMirrored, sourceParams.yMirrored); + + addStreamSideData(m_stream, sideData); + } + + Q_ASSERT(m_codec); + + m_stream->time_base = adjustFrameTimeBase(m_codec->supported_framerates, m_codecFrameRate); + m_codecContext.reset(avcodec_alloc_context3(m_codec)); + if (!m_codecContext) { + qWarning() << "Could not allocate codec context"; + return false; + } + + avcodec_parameters_to_context(m_codecContext.get(), m_stream->codecpar); + m_codecContext->time_base = m_stream->time_base; + qCDebug(qLcVideoFrameEncoder) << "codecContext time base" << m_codecContext->time_base.num + << m_codecContext->time_base.den; + + m_codecContext->framerate = m_codecFrameRate; + m_codecContext->pix_fmt = m_targetFormat; + m_codecContext->width = resolution.width(); + m_codecContext->height = resolution.height(); + + if (m_accel) { + auto deviceContext = m_accel->hwDeviceContextAsBuffer(); + Q_ASSERT(deviceContext); + m_codecContext->hw_device_ctx = av_buffer_ref(deviceContext); + + if (auto framesContext = m_accel->hwFramesContextAsBuffer()) + m_codecContext->hw_frames_ctx = av_buffer_ref(framesContext); + } + + return true; +} + +bool VideoFrameEncoder::open() +{ + if (!m_codecContext) + return false; + + AVDictionaryHolder opts; + applyVideoEncoderOptions(m_settings, m_codec->name, m_codecContext.get(), opts); + applyExperimentalCodecOptions(m_codec, opts); + + int res = avcodec_open2(m_codecContext.get(), m_codec, opts); + if (res < 0) { + m_codecContext.reset(); + qWarning() << "Couldn't open codec for writing" << err2str(res); + return false; + } + qCDebug(qLcVideoFrameEncoder) << "video codec opened" << res << "time base" + << m_codecContext->time_base; + return true; +} + +qint64 VideoFrameEncoder::getPts(qint64 us) const +{ + qint64 div = 1'000'000 * m_stream->time_base.num; + return div != 0 ? (us * m_stream->time_base.den + div / 2) / div : 0; +} + +const AVRational &VideoFrameEncoder::getTimeBase() const +{ + return m_stream->time_base; +} + +namespace { +struct FrameConverter +{ + FrameConverter(AVFrameUPtr inputFrame) : m_inputFrame{ std::move(inputFrame) } { } + + int downloadFromHw() + { + AVFrameUPtr cpuFrame = makeAVFrame(); + + int err = av_hwframe_transfer_data(cpuFrame.get(), currentFrame(), 0); + if (err < 0) { + qCDebug(qLcVideoFrameEncoder) + << "Error transferring frame data to surface." << err2str(err); + return err; + } + + setFrame(std::move(cpuFrame)); + return 0; + } + + void convert(SwsContext *converter, AVPixelFormat format, const QSize &size) + { + AVFrameUPtr scaledFrame = makeAVFrame(); + + scaledFrame->format = format; + scaledFrame->width = size.width(); + scaledFrame->height = size.height(); + + av_frame_get_buffer(scaledFrame.get(), 0); + const auto scaledHeight = + sws_scale(converter, currentFrame()->data, currentFrame()->linesize, 0, currentFrame()->height, + scaledFrame->data, scaledFrame->linesize); + + if (scaledHeight != scaledFrame->height) + qCWarning(qLcVideoFrameEncoder) + << "Scaled height" << scaledHeight << "!=" << scaledFrame->height; + + setFrame(std::move(scaledFrame)); + } + + int uploadToHw(HWAccel *accel) + { + auto *hwFramesContext = accel->hwFramesContextAsBuffer(); + Q_ASSERT(hwFramesContext); + AVFrameUPtr hwFrame = makeAVFrame(); + if (!hwFrame) + return AVERROR(ENOMEM); + + int err = av_hwframe_get_buffer(hwFramesContext, hwFrame.get(), 0); + if (err < 0) { + qCDebug(qLcVideoFrameEncoder) << "Error getting HW buffer" << err2str(err); + return err; + } else { + qCDebug(qLcVideoFrameEncoder) << "got HW buffer"; + } + if (!hwFrame->hw_frames_ctx) { + qCDebug(qLcVideoFrameEncoder) << "no hw frames context"; + return AVERROR(ENOMEM); + } + err = av_hwframe_transfer_data(hwFrame.get(), currentFrame(), 0); + if (err < 0) { + qCDebug(qLcVideoFrameEncoder) + << "Error transferring frame data to surface." << err2str(err); + return err; + } + + setFrame(std::move(hwFrame)); + + return 0; + } + + QMaybe<AVFrameUPtr, int> takeResultFrame() + { + // Ensure that object is reset to empty state + AVFrameUPtr converted = std::move(m_convertedFrame); + AVFrameUPtr input = std::move(m_inputFrame); + + if (!converted) + return input; + + // Copy metadata except size and format from input frame + const int status = av_frame_copy_props(converted.get(), input.get()); + if (status != 0) + return status; + + return converted; + } + +private: + void setFrame(AVFrameUPtr frame) { m_convertedFrame = std::move(frame); } + + AVFrame *currentFrame() const + { + if (m_convertedFrame) + return m_convertedFrame.get(); + return m_inputFrame.get(); + } + + AVFrameUPtr m_inputFrame; + AVFrameUPtr m_convertedFrame; +}; +} + +int VideoFrameEncoder::sendFrame(AVFrameUPtr inputFrame) +{ + if (!m_codecContext) { + qWarning() << "codec context is not initialized!"; + return AVERROR(EINVAL); + } + + if (!inputFrame) + return avcodec_send_frame(m_codecContext.get(), nullptr); // Flush + + if (!updateSourceFormatAndSize(inputFrame.get())) + return AVERROR(EINVAL); + + FrameConverter converter{ std::move(inputFrame) }; + + if (m_downloadFromHW) { + const int status = converter.downloadFromHw(); + if (status != 0) + return status; + } + + if (m_converter) + converter.convert(m_converter.get(), m_targetSWFormat, m_settings.videoResolution()); + + if (m_uploadToHW) { + const int status = converter.uploadToHw(m_accel.get()); + if (status != 0) + return status; + } + + const QMaybe<AVFrameUPtr, int> resultFrame = converter.takeResultFrame(); + if (!resultFrame) + return resultFrame.error(); + + AVRational timeBase{}; + int64_t pts{}; + getAVFrameTime(*resultFrame.value(), pts, timeBase); + qCDebug(qLcVideoFrameEncoder) << "sending frame" << pts << "*" << timeBase; + + return avcodec_send_frame(m_codecContext.get(), resultFrame.value().get()); +} + +qint64 VideoFrameEncoder::estimateDuration(const AVPacket &packet, bool isFirstPacket) +{ + qint64 duration = 0; // In stream units, multiply by time_base to get seconds + + if (isFirstPacket) { + // First packet - Estimate duration from frame rate. Duration must + // be set for single-frame videos, otherwise they won't open in + // media player. + const AVRational frameDuration = av_inv_q(m_codecContext->framerate); + duration = av_rescale_q(1, frameDuration, m_stream->time_base); + } else { + // Duration is calculated from actual packet times. TODO: Handle discontinuities + duration = packet.pts - m_lastPacketTime; + } + + return duration; +} + +AVPacketUPtr VideoFrameEncoder::retrievePacket() +{ + if (!m_codecContext) + return nullptr; + + auto getPacket = [&]() { + AVPacketUPtr packet(av_packet_alloc()); + const int ret = avcodec_receive_packet(m_codecContext.get(), packet.get()); + if (ret < 0) { + if (ret != AVERROR(EOF) && ret != AVERROR(EAGAIN) && ret != AVERROR_EOF) + qCDebug(qLcVideoFrameEncoder) << "Error receiving packet" << ret << err2str(ret); + return AVPacketUPtr{}; + } + auto ts = timeStampMs(packet->pts, m_stream->time_base); + + qCDebug(qLcVideoFrameEncoder) + << "got a packet" << packet->pts << packet->dts << (ts ? *ts : 0); + + packet->stream_index = m_stream->id; + + if (packet->duration == 0) { + const bool firstFrame = m_lastPacketTime == AV_NOPTS_VALUE; + packet->duration = estimateDuration(*packet, firstFrame); + } + + m_lastPacketTime = packet->pts; + + return packet; + }; + + auto fixPacketDts = [&](AVPacket &packet) { + // Workaround for some ffmpeg codecs bugs (e.g. nvenc) + // Ideally, packet->pts < packet->dts is not expected + + if (packet.dts == AV_NOPTS_VALUE) + return true; + + packet.dts -= m_packetDtsOffset; + + if (packet.pts != AV_NOPTS_VALUE && packet.pts < packet.dts) { + m_packetDtsOffset += packet.dts - packet.pts; + packet.dts = packet.pts; + + if (m_prevPacketDts != AV_NOPTS_VALUE && packet.dts < m_prevPacketDts) { + qCWarning(qLcVideoFrameEncoder) + << "Skip packet; failed to fix dts:" << packet.dts << m_prevPacketDts; + return false; + } + } + + m_prevPacketDts = packet.dts; + + return true; + }; + + while (auto packet = getPacket()) { + if (fixPacketDts(*packet)) + return packet; + } + + return nullptr; +} + +bool VideoFrameEncoder::updateSourceFormatAndSize(const AVFrame *frame) +{ + Q_ASSERT(frame); + + const QSize frameSize(frame->width, frame->height); + const AVPixelFormat frameFormat = static_cast<AVPixelFormat>(frame->format); + + if (frameSize == m_sourceSize && frameFormat == m_sourceFormat) + return true; + + auto applySourceFormatAndSize = [&](AVPixelFormat swFormat) { + m_sourceSize = frameSize; + m_sourceFormat = frameFormat; + m_sourceSWFormat = swFormat; + updateConversions(); + return true; + }; + + if (frameFormat == m_sourceFormat) + return applySourceFormatAndSize(m_sourceSWFormat); + + if (frameFormat == AV_PIX_FMT_NONE) { + qWarning() << "Got a frame with invalid pixel format"; + return false; + } + + if (isSwPixelFormat(frameFormat)) + return applySourceFormatAndSize(frameFormat); + + auto framesCtx = reinterpret_cast<const AVHWFramesContext *>(frame->hw_frames_ctx->data); + if (!framesCtx || framesCtx->sw_format == AV_PIX_FMT_NONE) { + qWarning() << "Cannot update conversions as hw frame has invalid framesCtx" << framesCtx; + return false; + } + + return applySourceFormatAndSize(framesCtx->sw_format); +} + +void VideoFrameEncoder::updateConversions() +{ + const bool needToScale = m_sourceSize != m_settings.videoResolution(); + const bool zeroCopy = m_sourceFormat == m_targetFormat && !needToScale; + + m_converter.reset(); + + if (zeroCopy) { + m_downloadFromHW = false; + m_uploadToHW = false; + + qCDebug(qLcVideoFrameEncoder) << "zero copy encoding, format" << m_targetFormat; + // no need to initialize any converters + return; + } + + m_downloadFromHW = m_sourceFormat != m_sourceSWFormat; + m_uploadToHW = m_targetFormat != m_targetSWFormat; + + if (m_sourceSWFormat != m_targetSWFormat || needToScale) { + const auto targetSize = m_settings.videoResolution(); + qCDebug(qLcVideoFrameEncoder) + << "video source and encoder use different formats:" << m_sourceSWFormat + << m_targetSWFormat << "or sizes:" << m_sourceSize << targetSize; + + m_converter.reset(sws_getContext(m_sourceSize.width(), m_sourceSize.height(), + m_sourceSWFormat, targetSize.width(), targetSize.height(), + m_targetSWFormat, SWS_FAST_BILINEAR, nullptr, nullptr, + nullptr)); + } + + qCDebug(qLcVideoFrameEncoder) << "VideoFrameEncoder conversions initialized:" + << "sourceFormat:" << m_sourceFormat + << (isHwPixelFormat(m_sourceFormat) ? "(hw)" : "(sw)") + << "targetFormat:" << m_targetFormat + << (isHwPixelFormat(m_targetFormat) ? "(hw)" : "(sw)") + << "sourceSWFormat:" << m_sourceSWFormat + << "targetSWFormat:" << m_targetSWFormat + << "converter:" << m_converter.get(); +} + +} // namespace QFFmpeg + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder_p.h b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder_p.h new file mode 100644 index 000000000..731789926 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/recordingengine/qffmpegvideoframeencoder_p.h @@ -0,0 +1,104 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QFFMPEGVIDEOFRAMEENCODER_P_H +#define QFFMPEGVIDEOFRAMEENCODER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qffmpeghwaccel_p.h" +#include "private/qplatformmediarecorder_p.h" + +QT_BEGIN_NAMESPACE + +class QMediaEncoderSettings; + +namespace QFFmpeg { + +class VideoFrameEncoder +{ +public: + struct SourceParams + { + QSize size; + AVPixelFormat format = AV_PIX_FMT_NONE; + AVPixelFormat swFormat = AV_PIX_FMT_NONE; + QtVideo::Rotation rotation = QtVideo::Rotation::None; + bool xMirrored = false; + bool yMirrored = false; + qreal frameRate = 0.; + AVColorTransferCharacteristic colorTransfer = AVCOL_TRC_UNSPECIFIED; + AVColorSpace colorSpace = AVCOL_SPC_UNSPECIFIED; + AVColorRange colorRange = AVCOL_RANGE_UNSPECIFIED; + }; + static std::unique_ptr<VideoFrameEncoder> create(const QMediaEncoderSettings &encoderSettings, + const SourceParams &sourceParams, + AVFormatContext *formatContext); + + ~VideoFrameEncoder(); + + bool open(); + + AVPixelFormat sourceFormat() const { return m_sourceFormat; } + AVPixelFormat targetFormat() const { return m_targetFormat; } + + qint64 getPts(qint64 ms) const; + + const AVRational &getTimeBase() const; + + int sendFrame(AVFrameUPtr inputFrame); + AVPacketUPtr retrievePacket(); + + const QMediaEncoderSettings &settings() { return m_settings; } + +private: + VideoFrameEncoder() = default; + + bool updateSourceFormatAndSize(const AVFrame *frame); + + void updateConversions(); + + bool initCodec(); + + bool initTargetFormats(); + + bool initCodecContext(const SourceParams &sourceParams, AVFormatContext *formatContext); + + qint64 estimateDuration(const AVPacket &packet, bool isFirstPacket); + +private: + QMediaEncoderSettings m_settings; + QSize m_sourceSize; + + std::unique_ptr<HWAccel> m_accel; + const AVCodec *m_codec = nullptr; + AVStream *m_stream = nullptr; + qint64 m_lastPacketTime = AV_NOPTS_VALUE; + AVCodecContextUPtr m_codecContext; + std::unique_ptr<SwsContext, decltype(&sws_freeContext)> m_converter = { nullptr, + &sws_freeContext }; + AVPixelFormat m_sourceFormat = AV_PIX_FMT_NONE; + AVPixelFormat m_sourceSWFormat = AV_PIX_FMT_NONE; + AVPixelFormat m_targetFormat = AV_PIX_FMT_NONE; + AVPixelFormat m_targetSWFormat = AV_PIX_FMT_NONE; + bool m_downloadFromHW = false; + bool m_uploadToHW = false; + + AVRational m_codecFrameRate = { 0, 1 }; + + int64_t m_prevPacketDts = AV_NOPTS_VALUE; + int64_t m_packetDtsOffset = 0; +}; +} + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/ffmpeg/symbolstubs/openssl3.ver b/src/plugins/multimedia/ffmpeg/symbolstubs/openssl3.ver new file mode 100644 index 000000000..88235a94c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/symbolstubs/openssl3.ver @@ -0,0 +1,7 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +OPENSSL_3.0.0 { + global: + *; +}; diff --git a/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-crypto.cpp b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-crypto.cpp new file mode 100644 index 000000000..fbf3b783c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-crypto.cpp @@ -0,0 +1,6 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/private/qsymbolsresolveutils_p.h> + +// No stub functions are needed for ffmpeg diff --git a/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-ssl.cpp b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-ssl.cpp new file mode 100644 index 000000000..3e38e398c --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-ssl.cpp @@ -0,0 +1,300 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/private/qsymbolsresolveutils_p.h> + +#include <qstringliteral.h> + +#include <openssl/bio.h> +#include <openssl/ssl.h> +#include <openssl/bn.h> +#include <openssl/err.h> +#include <openssl/rand.h> + +using namespace Qt::StringLiterals; + +[[maybe_unused]] static constexpr auto SHLIB_VERSION = +#if defined(OPENSSL_SHLIB_VERSION) + OPENSSL_SHLIB_VERSION; +#elif defined(SHLIB_VERSION_NUMBER) + SHLIB_VERSION_NUMBER; +#endif + + +#if !defined(Q_OS_ANDROID) +CHECK_VERSIONS("ssl", SSL_NEEDED_SOVERSION, SHLIB_VERSION); +#endif + +static std::unique_ptr<QLibrary> loadLib() +{ + auto lib = std::make_unique<QLibrary>(); + + auto tryLoad = [&](QString sslName, auto version) { + lib->setFileNameAndVersion(sslName, version); + return lib->load(); + }; + +// openssl on Android has specific suffixes +#if defined(Q_OS_ANDROID) + { + auto suffix = qEnvironmentVariable("ANDROID_OPENSSL_SUFFIX"); + if (suffix.isEmpty()) { +#if (OPENSSL_VERSION_NUMBER >> 28) < 3 // major version < 3 + suffix = "_1_1"_L1; +#elif OPENSSL_VERSION_MAJOR == 3 + suffix = "_3"_L1; +#else + static_assert(false, "Unexpected openssl version"); +#endif + } + + if (tryLoad("ssl"_L1 + suffix, -1)) + return lib; + } +#endif + + if (tryLoad("ssl"_L1, SSL_NEEDED_SOVERSION ""_L1)) + return lib; + + return {}; +}; + + +BEGIN_INIT_FUNCS("ssl", loadLib) + +// BN functions + +INIT_FUNC(BN_value_one); +INIT_FUNC(BN_mod_word); + +INIT_FUNC(BN_div_word) +INIT_FUNC(BN_mul_word) +INIT_FUNC(BN_add_word) +INIT_FUNC(BN_sub_word) +INIT_FUNC(BN_set_word) +INIT_FUNC(BN_new) +INIT_FUNC(BN_cmp) + +INIT_FUNC(BN_free); + +INIT_FUNC(BN_copy); + +INIT_FUNC(BN_CTX_new); + +INIT_FUNC(BN_CTX_free); +INIT_FUNC(BN_CTX_start); + +INIT_FUNC(BN_CTX_get); +INIT_FUNC(BN_CTX_end); + +INIT_FUNC(BN_rand); +INIT_FUNC(BN_mod_exp); + +INIT_FUNC(BN_num_bits); +INIT_FUNC(BN_num_bits_word); + +INIT_FUNC(BN_bn2hex); +INIT_FUNC(BN_bn2dec); + +INIT_FUNC(BN_hex2bn); +INIT_FUNC(BN_dec2bn); +INIT_FUNC(BN_asc2bn); + +INIT_FUNC(BN_bn2bin); +INIT_FUNC(BN_bin2bn); + +// BIO-related functions + +INIT_FUNC(BIO_new); +INIT_FUNC(BIO_free); + +INIT_FUNC(BIO_read); +INIT_FUNC(BIO_write); +INIT_FUNC(BIO_s_mem); + +INIT_FUNC(BIO_set_data); + +INIT_FUNC(BIO_get_data); +INIT_FUNC(BIO_set_init); + +INIT_FUNC(BIO_set_flags); +INIT_FUNC(BIO_test_flags); +INIT_FUNC(BIO_clear_flags); + +INIT_FUNC(BIO_meth_new); +INIT_FUNC(BIO_meth_free); + +INIT_FUNC(BIO_meth_set_write); +INIT_FUNC(BIO_meth_set_read); +INIT_FUNC(BIO_meth_set_puts); +INIT_FUNC(BIO_meth_set_gets); +INIT_FUNC(BIO_meth_set_ctrl); +INIT_FUNC(BIO_meth_set_create); +INIT_FUNC(BIO_meth_set_destroy); +INIT_FUNC(BIO_meth_set_callback_ctrl); + +// SSL functions + +INIT_FUNC(SSL_CTX_new); +INIT_FUNC(SSL_CTX_up_ref); +INIT_FUNC(SSL_CTX_free); + +INIT_FUNC(SSL_new); +INIT_FUNC(SSL_up_ref); +INIT_FUNC(SSL_free); + +INIT_FUNC(SSL_accept); +INIT_FUNC(SSL_stateless); +INIT_FUNC(SSL_connect); +INIT_FUNC(SSL_read); +INIT_FUNC(SSL_peek); +INIT_FUNC(SSL_write); +INIT_FUNC(SSL_ctrl); +INIT_FUNC(SSL_shutdown); +INIT_FUNC(SSL_set_bio); + +// options are unsigned long in openssl 1.1.1, and uint64 in 3.x.x +INIT_FUNC(SSL_CTX_set_options); + +INIT_FUNC(SSL_get_error); +INIT_FUNC(SSL_CTX_load_verify_locations); + +INIT_FUNC(SSL_CTX_set_verify); +INIT_FUNC(SSL_CTX_use_PrivateKey); + +INIT_FUNC(SSL_CTX_use_PrivateKey_file); +INIT_FUNC(SSL_CTX_use_certificate_chain_file); + +INIT_FUNC(ERR_get_error); + +INIT_FUNC(ERR_error_string); + +// TLS functions + +INIT_FUNC(TLS_client_method); +INIT_FUNC(TLS_server_method); + +// RAND functions + +INIT_FUNC(RAND_bytes); + +END_INIT_FUNCS() + +//////////// Define + +// BN functions + +DEFINE_FUNC(BN_value_one, 0); +DEFINE_FUNC(BN_mod_word, 2); + +DEFINE_FUNC(BN_div_word, 2) +DEFINE_FUNC(BN_mul_word, 2) +DEFINE_FUNC(BN_add_word, 2) +DEFINE_FUNC(BN_sub_word, 2) +DEFINE_FUNC(BN_set_word, 2) +DEFINE_FUNC(BN_new, 0) +DEFINE_FUNC(BN_cmp, 2) + +DEFINE_FUNC(BN_free, 1); + +DEFINE_FUNC(BN_copy, 2); + +DEFINE_FUNC(BN_CTX_new, 0); + +DEFINE_FUNC(BN_CTX_free, 1); +DEFINE_FUNC(BN_CTX_start, 1); + +DEFINE_FUNC(BN_CTX_get, 1); +DEFINE_FUNC(BN_CTX_end, 1); + +DEFINE_FUNC(BN_rand, 4); +DEFINE_FUNC(BN_mod_exp, 5); + +DEFINE_FUNC(BN_num_bits, 1); +DEFINE_FUNC(BN_num_bits_word, 1); + +DEFINE_FUNC(BN_bn2hex, 1); +DEFINE_FUNC(BN_bn2dec, 1); + +DEFINE_FUNC(BN_hex2bn, 2); +DEFINE_FUNC(BN_dec2bn, 2); +DEFINE_FUNC(BN_asc2bn, 2); + +DEFINE_FUNC(BN_bn2bin, 2); +DEFINE_FUNC(BN_bin2bn, 3); + +// BIO-related functions + +DEFINE_FUNC(BIO_new, 1); +DEFINE_FUNC(BIO_free, 1); + +DEFINE_FUNC(BIO_read, 3, -1); +DEFINE_FUNC(BIO_write, 3, -1); +DEFINE_FUNC(BIO_s_mem, 0); + +DEFINE_FUNC(BIO_set_data, 2); + +DEFINE_FUNC(BIO_get_data, 1); +DEFINE_FUNC(BIO_set_init, 2); + +DEFINE_FUNC(BIO_set_flags, 2); +DEFINE_FUNC(BIO_test_flags, 2); +DEFINE_FUNC(BIO_clear_flags, 2); + +DEFINE_FUNC(BIO_meth_new, 2); +DEFINE_FUNC(BIO_meth_free, 1); + +DEFINE_FUNC(BIO_meth_set_write, 2); +DEFINE_FUNC(BIO_meth_set_read, 2); +DEFINE_FUNC(BIO_meth_set_puts, 2); +DEFINE_FUNC(BIO_meth_set_gets, 2); +DEFINE_FUNC(BIO_meth_set_ctrl, 2); +DEFINE_FUNC(BIO_meth_set_create, 2); +DEFINE_FUNC(BIO_meth_set_destroy, 2); +DEFINE_FUNC(BIO_meth_set_callback_ctrl, 2); + +// SSL functions + +DEFINE_FUNC(SSL_CTX_new, 1); +DEFINE_FUNC(SSL_CTX_up_ref, 1); +DEFINE_FUNC(SSL_CTX_free, 1); + +DEFINE_FUNC(SSL_new, 1); +DEFINE_FUNC(SSL_up_ref, 1); +DEFINE_FUNC(SSL_free, 1); + +DEFINE_FUNC(SSL_accept, 1); +DEFINE_FUNC(SSL_stateless, 1); +DEFINE_FUNC(SSL_connect, 1); +DEFINE_FUNC(SSL_read, 3, -1); +DEFINE_FUNC(SSL_peek, 3); +DEFINE_FUNC(SSL_write, 3, -1); +DEFINE_FUNC(SSL_ctrl, 4); +DEFINE_FUNC(SSL_shutdown, 1); +DEFINE_FUNC(SSL_set_bio, 3); + +// options are unsigned long in openssl 1.1.1, and uint64 in 3.x.x +DEFINE_FUNC(SSL_CTX_set_options, 2); + +DEFINE_FUNC(SSL_get_error, 2); +DEFINE_FUNC(SSL_CTX_load_verify_locations, 3, -1); + +DEFINE_FUNC(SSL_CTX_set_verify, 3); +DEFINE_FUNC(SSL_CTX_use_PrivateKey, 2); + +DEFINE_FUNC(SSL_CTX_use_PrivateKey_file, 3); +DEFINE_FUNC(SSL_CTX_use_certificate_chain_file, 2); + +DEFINE_FUNC(ERR_get_error, 0); + +static char ErrorString[] = "Ssl not found"; +DEFINE_FUNC(ERR_error_string, 2, ErrorString); + +// TLS functions + +DEFINE_FUNC(TLS_client_method, 0); +DEFINE_FUNC(TLS_server_method, 0); + +// RAND functions + +DEFINE_FUNC(RAND_bytes, 2); diff --git a/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va-drm.cpp b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va-drm.cpp new file mode 100644 index 000000000..655a6b2b6 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va-drm.cpp @@ -0,0 +1,14 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/private/qsymbolsresolveutils_p.h> + +#include <va/va_drm.h> + +CHECK_VERSIONS("va-drm", VA_DRM_NEEDED_SOVERSION, VA_MAJOR_VERSION + 1); + +BEGIN_INIT_FUNCS("va-drm", VA_DRM_NEEDED_SOVERSION) +INIT_FUNC(vaGetDisplayDRM) +END_INIT_FUNCS() + +DEFINE_FUNC(vaGetDisplayDRM, 1); diff --git a/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va-x11.cpp b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va-x11.cpp new file mode 100644 index 000000000..3bada9e69 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va-x11.cpp @@ -0,0 +1,14 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/private/qsymbolsresolveutils_p.h> + +#include <va/va_x11.h> + +CHECK_VERSIONS("va-x11", VA_X11_NEEDED_SOVERSION, VA_MAJOR_VERSION + 1); + +BEGIN_INIT_FUNCS("va-x11", VA_X11_NEEDED_SOVERSION) +INIT_FUNC(vaGetDisplay) +END_INIT_FUNCS() + +DEFINE_FUNC(vaGetDisplay, 1); diff --git a/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va.cpp b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va.cpp new file mode 100644 index 000000000..cfd2e5686 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/symbolstubs/qffmpegsymbols-va.cpp @@ -0,0 +1,150 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/private/qsymbolsresolveutils_p.h> + +#include <va/va.h> +#include <va/va_str.h> + +// VAAPI generated the actual *.so name due to the rule: +// https://github.com/intel/libva/blob/master/configure.ac +// +// The library name is generated libva.<x>.<y>.0 where +// <x> = VA-API major version + 1 +// <y> = 100 * VA-API minor version + VA-API micro version +CHECK_VERSIONS("va", VA_NEEDED_SOVERSION, VA_MAJOR_VERSION + 1); + +#ifdef Q_FFMPEG_PLUGIN_STUBS_ONLY +constexpr const char *loggingName = "va(in plugin)"; +#else +constexpr const char *loggingName = nullptr; +#endif + +BEGIN_INIT_FUNCS("va", VA_NEEDED_SOVERSION, loggingName) + + +INIT_FUNC(vaExportSurfaceHandle); +INIT_FUNC(vaSyncSurface); +INIT_FUNC(vaQueryVendorString); + +#ifndef Q_FFMPEG_PLUGIN_STUBS_ONLY + +INIT_FUNC(vaInitialize); +INIT_FUNC(vaTerminate); +INIT_FUNC(vaErrorStr); +INIT_FUNC(vaSetErrorCallback); +INIT_FUNC(vaSetInfoCallback); + +INIT_FUNC(vaCreateImage); +INIT_FUNC(vaGetImage); +INIT_FUNC(vaPutImage); +INIT_FUNC(vaDeriveImage); +INIT_FUNC(vaDestroyImage); +INIT_FUNC(vaQueryImageFormats); + +INIT_FUNC(vaBeginPicture); +INIT_FUNC(vaRenderPicture); +INIT_FUNC(vaEndPicture); + +INIT_FUNC(vaCreateBuffer); +INIT_FUNC(vaMapBuffer); +INIT_FUNC(vaUnmapBuffer); +#if VA_CHECK_VERSION(1, 9, 0) +INIT_FUNC(vaSyncBuffer); +#endif +INIT_FUNC(vaDestroyBuffer); + +INIT_FUNC(vaCreateSurfaces); +INIT_FUNC(vaDestroySurfaces); + +INIT_FUNC(vaCreateConfig); +INIT_FUNC(vaGetConfigAttributes); +INIT_FUNC(vaMaxNumProfiles); +INIT_FUNC(vaMaxNumImageFormats); +INIT_FUNC(vaMaxNumEntrypoints); +INIT_FUNC(vaQueryConfigProfiles); +INIT_FUNC(vaQueryConfigEntrypoints); +INIT_FUNC(vaQuerySurfaceAttributes); +INIT_FUNC(vaDestroyConfig); + +INIT_FUNC(vaCreateContext); +INIT_FUNC(vaDestroyContext); + +INIT_FUNC(vaProfileStr); +INIT_FUNC(vaEntrypointStr); + +INIT_FUNC(vaGetDisplayAttributes); + +INIT_FUNC(vaSetDriverName); + +INIT_FUNC(vaAcquireBufferHandle); +INIT_FUNC(vaReleaseBufferHandle); + +#endif + +END_INIT_FUNCS() + +constexpr auto emptyString = ""; + +DEFINE_FUNC(vaExportSurfaceHandle, 5, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaSyncSurface, 2, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaQueryVendorString, 1, emptyString); + +#ifndef Q_FFMPEG_PLUGIN_STUBS_ONLY + +DEFINE_FUNC(vaInitialize, 3, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaTerminate, 1, VA_STATUS_ERROR_OPERATION_FAILED); + +constexpr auto errorStr = "VAAPI is not available"; +DEFINE_FUNC(vaErrorStr, 1, errorStr); +DEFINE_FUNC(vaSetErrorCallback, 3); +DEFINE_FUNC(vaSetInfoCallback, 3); + +DEFINE_FUNC(vaCreateImage, 5, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaGetImage, 7, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaPutImage, 11, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaDeriveImage, 3, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaDestroyImage, 2, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaQueryImageFormats, 3, VA_STATUS_ERROR_OPERATION_FAILED); + +DEFINE_FUNC(vaBeginPicture, 3, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaRenderPicture, 4, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaEndPicture, 2, VA_STATUS_ERROR_OPERATION_FAILED); + +DEFINE_FUNC(vaCreateBuffer, 7, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaMapBuffer, 3, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaUnmapBuffer, 2, VA_STATUS_ERROR_OPERATION_FAILED); +#if VA_CHECK_VERSION(1, 9, 0) +DEFINE_FUNC(vaSyncBuffer, 3, VA_STATUS_ERROR_OPERATION_FAILED); +#endif +DEFINE_FUNC(vaDestroyBuffer, 2, VA_STATUS_ERROR_OPERATION_FAILED); + +DEFINE_FUNC(vaCreateSurfaces, 8, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaDestroySurfaces, 3, VA_STATUS_ERROR_OPERATION_FAILED); + +DEFINE_FUNC(vaCreateConfig, 6, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaGetConfigAttributes, 5, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaMaxNumProfiles, 1); +DEFINE_FUNC(vaMaxNumImageFormats, 1); +DEFINE_FUNC(vaMaxNumEntrypoints, 1); +DEFINE_FUNC(vaQueryConfigProfiles, 3, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaQueryConfigEntrypoints, 4, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaQuerySurfaceAttributes, 4, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaDestroyConfig, 2, VA_STATUS_ERROR_OPERATION_FAILED); + +DEFINE_FUNC(vaCreateContext, 8); +DEFINE_FUNC(vaDestroyContext, 2); + + +DEFINE_FUNC(vaProfileStr, 1, emptyString); +DEFINE_FUNC(vaEntrypointStr, 1, emptyString); + +DEFINE_FUNC(vaGetDisplayAttributes, 3, VA_STATUS_ERROR_OPERATION_FAILED); + +DEFINE_FUNC(vaSetDriverName, 2, VA_STATUS_ERROR_OPERATION_FAILED); + +DEFINE_FUNC(vaAcquireBufferHandle, 3, VA_STATUS_ERROR_OPERATION_FAILED); +DEFINE_FUNC(vaReleaseBufferHandle, 2, VA_STATUS_ERROR_OPERATION_FAILED); + +#endif + diff --git a/src/plugins/multimedia/ffmpeg/symbolstubs/va.ver b/src/plugins/multimedia/ffmpeg/symbolstubs/va.ver new file mode 100644 index 000000000..80c9a6dc0 --- /dev/null +++ b/src/plugins/multimedia/ffmpeg/symbolstubs/va.ver @@ -0,0 +1,7 @@ +# Copyright (C) 2024 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +VA_API_0.33.0 { + global: + vaCreateSurfaces; +}; diff --git a/src/plugins/multimedia/gstreamer/CMakeLists.txt b/src/plugins/multimedia/gstreamer/CMakeLists.txt index 3bce143f6..1ef1f9a36 100644 --- a/src/plugins/multimedia/gstreamer/CMakeLists.txt +++ b/src/plugins/multimedia/gstreamer/CMakeLists.txt @@ -1,20 +1,24 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + qt_find_package(EGL) -qt_internal_add_plugin(QGstreamerMediaPlugin - OUTPUT_NAME gstreamermediaplugin - PLUGIN_TYPE multimedia +qt_internal_add_module(QGstreamerMediaPluginPrivate + STATIC + INTERNAL_MODULE SOURCES audio/qgstreameraudiodevice.cpp audio/qgstreameraudiodevice_p.h - audio/qgstreameraudiosource.cpp audio/qgstreameraudiosource_p.h - audio/qgstreameraudiosink.cpp audio/qgstreameraudiosink_p.h audio/qgstreameraudiodecoder.cpp audio/qgstreameraudiodecoder_p.h - common/qgst_p.h - common/qgstappsrc.cpp common/qgstappsrc_p.h + common/qglist_helper_p.h + common/qgst.cpp common/qgst_p.h + common/qgst_debug.cpp common/qgst_debug_p.h + common/qgst_handle_types_p.h + common/qgstappsource.cpp common/qgstappsource_p.h common/qgstreameraudioinput.cpp common/qgstreameraudioinput_p.h common/qgstreameraudiooutput.cpp common/qgstreameraudiooutput_p.h common/qgstreamerbufferprobe.cpp common/qgstreamerbufferprobe_p.h common/qgstreamermetadata.cpp common/qgstreamermetadata_p.h - common/qgstreamermessage.cpp common/qgstreamermessage_p.h + common/qgstreamermessage_p.h common/qgstreamermediaplayer.cpp common/qgstreamermediaplayer_p.h common/qgstreamervideooutput.cpp common/qgstreamervideooutput_p.h common/qgstreamervideooverlay.cpp common/qgstreamervideooverlay_p.h @@ -24,33 +28,46 @@ qt_internal_add_plugin(QGstreamerMediaPlugin common/qgstvideobuffer.cpp common/qgstvideobuffer_p.h common/qgstvideorenderersink.cpp common/qgstvideorenderersink_p.h common/qgstsubtitlesink.cpp common/qgstsubtitlesink_p.h - qgstreamervideodevices.cpp qgstreamervideodevices_p.h - qgstreamerformatinfo.cpp qgstreamerformatinfo_p.h qgstreamerintegration.cpp qgstreamerintegration_p.h + qgstreamerformatinfo.cpp qgstreamerformatinfo_p.h + qgstreamervideodevices.cpp qgstreamervideodevices_p.h mediacapture/qgstreamercamera.cpp mediacapture/qgstreamercamera_p.h mediacapture/qgstreamerimagecapture.cpp mediacapture/qgstreamerimagecapture_p.h mediacapture/qgstreamermediacapture.cpp mediacapture/qgstreamermediacapture_p.h mediacapture/qgstreamermediaencoder.cpp mediacapture/qgstreamermediaencoder_p.h + NO_UNITY_BUILD_SOURCES + # Conflicts with macros defined in X11.h, and Xlib.h + common/qgstvideobuffer.cpp + common/qgstreamervideosink.cpp + NO_GENERATE_CPP_EXPORTS DEFINES GLIB_VERSION_MIN_REQUIRED=GLIB_VERSION_2_26 - INCLUDE_DIRECTORIES - audio - common - mediacapture - LIBRARIES + PUBLIC_LIBRARIES Qt::MultimediaPrivate Qt::CorePrivate GStreamer::GStreamer GStreamer::App ) -qt_internal_extend_target(QGstreamerMediaPlugin CONDITION QT_FEATURE_gstreamer_photography - LIBRARIES - -lgstphotography-1.0 +qt_internal_extend_target(QGstreamerMediaPluginPrivate CONDITION QT_FEATURE_gstreamer_photography + PUBLIC_LIBRARIES + GStreamer::Photography ) -qt_internal_extend_target(QGstreamerMediaPlugin CONDITION QT_FEATURE_gstreamer_gl - LIBRARIES +qt_internal_extend_target(QGstreamerMediaPluginPrivate CONDITION QT_FEATURE_gstreamer_gl + PUBLIC_LIBRARIES GStreamer::Gl + LIBRARIES EGL::EGL ) + +qt_internal_add_plugin(QGstreamerMediaPlugin + OUTPUT_NAME gstreamermediaplugin + PLUGIN_TYPE multimedia + SOURCES + qgstreamerplugin.cpp + gstreamer.json + LIBRARIES + Qt::QGstreamerMediaPluginPrivate + Qt::MultimediaPrivate +) diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp index 779084c7f..280b43cdb 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder.cpp @@ -1,47 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2020 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2020 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only //#define DEBUG_DECODER -#include "qgstreameraudiodecoder_p.h" -#include "qgstreamermessage_p.h" +#include <audio/qgstreameraudiodecoder_p.h> -#include <qgstutils_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstutils_p.h> #include <gst/gstvalue.h> #include <gst/base/gstbasesrc.h> @@ -54,11 +19,12 @@ #include <QtCore/qdir.h> #include <QtCore/qstandardpaths.h> #include <QtCore/qurl.h> - -#define MAX_BUFFERS_IN_QUEUE 4 +#include <QtCore/qloggingcategory.h> QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLcGstreamerAudioDecoder, "qt.multimedia.gstreameraudiodecoder"); + typedef enum { GST_PLAY_FLAG_VIDEO = 0x00000001, GST_PLAY_FLAG_AUDIO = 0x00000002, @@ -72,32 +38,41 @@ typedef enum { } GstPlayFlags; +QMaybe<QPlatformAudioDecoder *> QGstreamerAudioDecoder::create(QAudioDecoder *parent) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable("audioconvert", "playbin"); + if (error) + return *error; + + return new QGstreamerAudioDecoder(parent); +} QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent) : QPlatformAudioDecoder(parent), - m_playbin(GST_PIPELINE_CAST(QGstElement("playbin", "playbin").element())) + m_playbin{ + QGstPipeline::adopt(GST_PIPELINE_CAST( + QGstElement::createFromFactory("playbin", "playbin").element())), + }, + m_audioConvert{ + QGstElement::createFromFactory("audioconvert", "audioconvert"), + } { - if (m_playbin.isNull()) { - // ### set error - return; - } - // Sort out messages m_playbin.installMessageFilter(this); // Set the rest of the pipeline up setAudioFlags(true); - m_audioConvert = QGstElement("audioconvert", "audioconvert"); - - m_outputBin = QGstBin("audio-output-bin"); + m_outputBin = QGstBin::create("audio-output-bin"); m_outputBin.add(m_audioConvert); // add ghostpad m_outputBin.addGhostPad(m_audioConvert, "sink"); g_object_set(m_playbin.object(), "audio-sink", m_outputBin.element(), NULL); - g_signal_connect(m_playbin.object(), "deep-notify::source", (GCallback) &QGstreamerAudioDecoder::configureAppSrcElement, (gpointer)this); + + m_deepNotifySourceConnection = m_playbin.connect( + "deep-notify::source", (GCallback)&configureAppSrcElement, (gpointer)this); // Set volume to 100% gdouble volume = 1.0; @@ -106,165 +81,156 @@ QGstreamerAudioDecoder::QGstreamerAudioDecoder(QAudioDecoder *parent) QGstreamerAudioDecoder::~QGstreamerAudioDecoder() { - if (m_playbin.isNull()) - return; - stop(); -#if QT_CONFIG(gstreamer_app) + m_playbin.removeMessageFilter(this); + delete m_appSrc; -#endif } -#if QT_CONFIG(gstreamer_app) -void QGstreamerAudioDecoder::configureAppSrcElement(GObject* object, GObject *orig, GParamSpec *pspec, QGstreamerAudioDecoder *self) +void QGstreamerAudioDecoder::configureAppSrcElement([[maybe_unused]] GObject *object, GObject *orig, + [[maybe_unused]] GParamSpec *pspec, + QGstreamerAudioDecoder *self) { - Q_UNUSED(object); - Q_UNUSED(pspec); - // In case we switch from appsrc to not - if (!self->appsrc()) + if (!self->m_appSrc) return; - GstElement *appsrc; + QGstElementHandle appsrc; g_object_get(orig, "source", &appsrc, NULL); - auto *qAppSrc = self->appsrc(); - qAppSrc->setExternalAppSrc(appsrc); + auto *qAppSrc = self->m_appSrc; + qAppSrc->setExternalAppSrc(QGstAppSrc{ + qGstSafeCast<GstAppSrc>(appsrc.get()), + QGstAppSrc::NeedsRef, // CHECK: can we `release()`? + }); qAppSrc->setup(self->mDevice); - - g_object_unref(G_OBJECT(appsrc)); } -#endif bool QGstreamerAudioDecoder::processBusMessage(const QGstreamerMessage &message) { - GstMessage* gm = message.rawMessage(); - if (gm) { - if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_DURATION) { - updateDuration(); - } else if (GST_MESSAGE_SRC(gm) == m_playbin.object()) { - switch (GST_MESSAGE_TYPE(gm)) { - case GST_MESSAGE_STATE_CHANGED: - { - GstState oldState; - GstState newState; - GstState pending; - - gst_message_parse_state_changed(gm, &oldState, &newState, &pending); - -#ifdef DEBUG_DECODER - QStringList states; - states << "GST_STATE_VOID_PENDING" << "GST_STATE_NULL" << "GST_STATE_READY" << "GST_STATE_PAUSED" << "GST_STATE_PLAYING"; - - qDebug() << QString("state changed: old: %1 new: %2 pending: %3") \ - .arg(states[oldState]) \ - .arg(states[newState]) \ - .arg(states[pending]) << "internal" << m_state; -#endif + qCDebug(qLcGstreamerAudioDecoder) << "received bus message:" << message; - bool isDecoding = false; - switch (newState) { - case GST_STATE_VOID_PENDING: - case GST_STATE_NULL: - case GST_STATE_READY: - break; - case GST_STATE_PLAYING: - isDecoding = true; - break; - case GST_STATE_PAUSED: - isDecoding = true; - - //gstreamer doesn't give a reliable indication the duration - //information is ready, GST_MESSAGE_DURATION is not sent by most elements - //the duration is queried up to 5 times with increasing delay - m_durationQueries = 5; - updateDuration(); - break; - } - - setIsDecoding(isDecoding); - } - break; - - case GST_MESSAGE_EOS: - finished(); - break; - - case GST_MESSAGE_ERROR: { - GError *err; - gchar *debug; - gst_message_parse_error(gm, &err, &debug); - if (err->domain == GST_STREAM_ERROR && err->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) - processInvalidMedia(QAudioDecoder::FormatError, tr("Cannot play stream of type: <unknown>")); - else - processInvalidMedia(QAudioDecoder::ResourceError, QString::fromUtf8(err->message)); - qWarning() << "Error:" << QString::fromUtf8(err->message); - g_error_free(err); - g_free(debug); - } - break; - case GST_MESSAGE_WARNING: - { - GError *err; - gchar *debug; - gst_message_parse_warning (gm, &err, &debug); - qWarning() << "Warning:" << QString::fromUtf8(err->message); - g_error_free (err); - g_free (debug); - } - break; -#ifdef DEBUG_DECODER - case GST_MESSAGE_INFO: - { - GError *err; - gchar *debug; - gst_message_parse_info (gm, &err, &debug); - qDebug() << "Info:" << QString::fromUtf8(err->message); - g_error_free (err); - g_free (debug); - } - break; -#endif - default: - break; - } - } else if (GST_MESSAGE_TYPE(gm) == GST_MESSAGE_ERROR) { - GError *err; - gchar *debug; - gst_message_parse_error(gm, &err, &debug); + GstMessage *gm = message.message(); + + switch (message.type()) { + case GST_MESSAGE_DURATION: { + updateDuration(); + return false; + } + + case GST_MESSAGE_ERROR: { + qCDebug(qLcGstreamerAudioDecoder) << " error" << QCompactGstMessageAdaptor(message); + + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(gm, &err, &debug); + + if (message.source() == m_playbin) { + if (err.get()->domain == GST_STREAM_ERROR + && err.get()->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) + processInvalidMedia(QAudioDecoder::FormatError, + tr("Cannot play stream of type: <unknown>")); + else + processInvalidMedia(QAudioDecoder::ResourceError, + QString::fromUtf8(err.get()->message)); + } else { QAudioDecoder::Error qerror = QAudioDecoder::ResourceError; - if (err->domain == GST_STREAM_ERROR) { - switch (err->code) { - case GST_STREAM_ERROR_DECRYPT: - case GST_STREAM_ERROR_DECRYPT_NOKEY: - qerror = QAudioDecoder::AccessDeniedError; - break; - case GST_STREAM_ERROR_FORMAT: - case GST_STREAM_ERROR_DEMUX: - case GST_STREAM_ERROR_DECODE: - case GST_STREAM_ERROR_WRONG_TYPE: - case GST_STREAM_ERROR_TYPE_NOT_FOUND: - case GST_STREAM_ERROR_CODEC_NOT_FOUND: - qerror = QAudioDecoder::FormatError; - break; - default: - break; + if (err.get()->domain == GST_STREAM_ERROR) { + switch (err.get()->code) { + case GST_STREAM_ERROR_DECRYPT: + case GST_STREAM_ERROR_DECRYPT_NOKEY: + qerror = QAudioDecoder::AccessDeniedError; + break; + case GST_STREAM_ERROR_FORMAT: + case GST_STREAM_ERROR_DEMUX: + case GST_STREAM_ERROR_DECODE: + case GST_STREAM_ERROR_WRONG_TYPE: + case GST_STREAM_ERROR_TYPE_NOT_FOUND: + case GST_STREAM_ERROR_CODEC_NOT_FOUND: + qerror = QAudioDecoder::FormatError; + break; + default: + break; } - } else if (err->domain == GST_CORE_ERROR) { - switch (err->code) { - case GST_CORE_ERROR_MISSING_PLUGIN: - qerror = QAudioDecoder::FormatError; - break; - default: - break; + } else if (err.get()->domain == GST_CORE_ERROR) { + switch (err.get()->code) { + case GST_CORE_ERROR_MISSING_PLUGIN: + qerror = QAudioDecoder::FormatError; + break; + default: + break; } } - processInvalidMedia(qerror, QString::fromUtf8(err->message)); - g_error_free(err); - g_free(debug); + processInvalidMedia(qerror, QString::fromUtf8(err.get()->message)); } + break; + } + + default: + if (message.source() == m_playbin) + return handlePlaybinMessage(message); + } + + return false; +} + +bool QGstreamerAudioDecoder::handlePlaybinMessage(const QGstreamerMessage &message) +{ + GstMessage *gm = message.message(); + + switch (GST_MESSAGE_TYPE(gm)) { + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(gm, &oldState, &newState, &pending); + + bool isDecoding = false; + switch (newState) { + case GST_STATE_VOID_PENDING: + case GST_STATE_NULL: + case GST_STATE_READY: + break; + case GST_STATE_PLAYING: + isDecoding = true; + break; + case GST_STATE_PAUSED: + isDecoding = true; + + // gstreamer doesn't give a reliable indication the duration + // information is ready, GST_MESSAGE_DURATION is not sent by most elements + // the duration is queried up to 5 times with increasing delay + m_durationQueries = 5; + updateDuration(); + break; + } + + setIsDecoding(isDecoding); + break; + }; + + case GST_MESSAGE_EOS: + m_playbin.setState(GST_STATE_NULL); + finished(); + break; + + case GST_MESSAGE_ERROR: + Q_UNREACHABLE_RETURN(false); // handled in processBusMessage + + case GST_MESSAGE_WARNING: + qCWarning(qLcGstreamerAudioDecoder) << "Warning:" << QCompactGstMessageAdaptor(message); + break; + + case GST_MESSAGE_INFO: { + if (qLcGstreamerAudioDecoder().isDebugEnabled()) + qCWarning(qLcGstreamerAudioDecoder) << "Info:" << QCompactGstMessageAdaptor(message); + break; + } + default: + break; } return false; @@ -285,7 +251,7 @@ void QGstreamerAudioDecoder::setSource(const QUrl &fileName) bool isSignalRequired = (mSource != fileName); mSource = fileName; if (isSignalRequired) - emit sourceChanged(); + sourceChanged(); } QIODevice *QGstreamerAudioDecoder::sourceDevice() const @@ -300,16 +266,11 @@ void QGstreamerAudioDecoder::setSourceDevice(QIODevice *device) bool isSignalRequired = (mDevice != device); mDevice = device; if (isSignalRequired) - emit sourceChanged(); + sourceChanged(); } void QGstreamerAudioDecoder::start() { - if (m_playbin.isNull()) { - processInvalidMedia(QAudioDecoder::ResourceError, QLatin1String("Playbin element is not valid")); - return; - } - addAppSink(); if (!mSource.isEmpty()) { @@ -321,8 +282,15 @@ void QGstreamerAudioDecoder::start() return; } - if (!m_appSrc) - m_appSrc = new QGstAppSrc(this); + if (!m_appSrc) { + auto maybeAppSrc = QGstAppSource::create(this); + if (maybeAppSrc) { + m_appSrc = maybeAppSrc.value(); + } else { + processInvalidMedia(QAudioDecoder::ResourceError, maybeAppSrc.error()); + return; + } + } m_playbin.set("uri", "appsrc://"); } else { @@ -333,12 +301,12 @@ void QGstreamerAudioDecoder::start() if (m_appSink) { if (mFormat.isValid()) { setAudioFlags(false); - QGstMutableCaps caps = QGstUtils::capsForAudioFormat(mFormat); - gst_app_sink_set_caps(m_appSink, caps.get()); + auto caps = QGstUtils::capsForAudioFormat(mFormat); + m_appSink.setCaps(caps); } else { // We want whatever the native audio format is setAudioFlags(true); - gst_app_sink_set_caps(m_appSink, nullptr); + m_appSink.setCaps({}); } } @@ -351,26 +319,24 @@ void QGstreamerAudioDecoder::start() void QGstreamerAudioDecoder::stop() { - if (m_playbin.isNull()) - return; - m_playbin.setState(GST_STATE_NULL); + m_currentSessionId += 1; removeAppSink(); // GStreamer thread is stopped. Can safely access m_buffersAvailable if (m_buffersAvailable != 0) { m_buffersAvailable = 0; - emit bufferAvailableChanged(false); + bufferAvailableChanged(false); } - if (m_position != -1) { - m_position = -1; - emit positionChanged(m_position); + if (m_position != invalidPosition) { + m_position = invalidPosition; + positionChanged(m_position.count()); } - if (m_duration != -1) { - m_duration = -1; - emit durationChanged(m_duration); + if (m_duration != invalidDuration) { + m_duration = invalidDuration; + durationChanged(m_duration.count()); } setIsDecoding(false); @@ -385,104 +351,94 @@ void QGstreamerAudioDecoder::setAudioFormat(const QAudioFormat &format) { if (mFormat != format) { mFormat = format; - emit formatChanged(mFormat); + formatChanged(mFormat); } } QAudioBuffer QGstreamerAudioDecoder::read() { - QAudioBuffer audioBuffer; - - int buffersAvailable; - { - QMutexLocker locker(&m_buffersMutex); - buffersAvailable = m_buffersAvailable; - - // need to decrement before pulling a buffer - // to make sure assert in QGstreamerAudioDecoderControl::new_buffer works - m_buffersAvailable--; - } + using namespace std::chrono; + QAudioBuffer audioBuffer; - if (buffersAvailable) { - if (buffersAvailable == 1) - emit bufferAvailableChanged(false); - - const char* bufferData = nullptr; - int bufferSize = 0; - - GstSample *sample = gst_app_sink_pull_sample(m_appSink); - GstBuffer *buffer = gst_sample_get_buffer(sample); - GstMapInfo mapInfo; - gst_buffer_map(buffer, &mapInfo, GST_MAP_READ); - bufferData = (const char*)mapInfo.data; - bufferSize = mapInfo.size; - QAudioFormat format = QGstUtils::audioFormatForSample(sample); - - if (format.isValid()) { - // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer. - // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer. - qint64 position = getPositionFromBuffer(buffer); - audioBuffer = QAudioBuffer(QByteArray((const char*)bufferData, bufferSize), format, position); - position /= 1000; // convert to milliseconds - if (position != m_position) { - m_position = position; - emit positionChanged(m_position); - } + if (m_buffersAvailable == 0) + return audioBuffer; + + m_buffersAvailable -= 1; + + if (m_buffersAvailable == 0) + bufferAvailableChanged(false); + + QGstSampleHandle sample = m_appSink.pullSample(); + GstBuffer *buffer = gst_sample_get_buffer(sample.get()); + GstMapInfo mapInfo; + gst_buffer_map(buffer, &mapInfo, GST_MAP_READ); + const char *bufferData = (const char *)mapInfo.data; + int bufferSize = mapInfo.size; + QAudioFormat format = QGstUtils::audioFormatForSample(sample.get()); + + if (format.isValid()) { + // XXX At the moment we have to copy data from GstBuffer into QAudioBuffer. + // We could improve performance by implementing QAbstractAudioBuffer for GstBuffer. + nanoseconds position = getPositionFromBuffer(buffer); + audioBuffer = QAudioBuffer{ + QByteArray(bufferData, bufferSize), + format, + round<microseconds>(position).count(), + }; + milliseconds positionInMs = round<milliseconds>(position); + if (position != m_position) { + m_position = positionInMs; + positionChanged(m_position.count()); } - gst_buffer_unmap(buffer, &mapInfo); - gst_sample_unref(sample); } + gst_buffer_unmap(buffer, &mapInfo); return audioBuffer; } -bool QGstreamerAudioDecoder::bufferAvailable() const -{ - QMutexLocker locker(&m_buffersMutex); - return m_buffersAvailable > 0; -} - qint64 QGstreamerAudioDecoder::position() const { - return m_position; + return m_position.count(); } qint64 QGstreamerAudioDecoder::duration() const { - return m_duration; + return m_duration.count(); } void QGstreamerAudioDecoder::processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString) { stop(); - emit error(int(errorCode), errorString); + error(int(errorCode), errorString); } -GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *, gpointer user_data) +GstFlowReturn QGstreamerAudioDecoder::newSample(GstAppSink *) { - // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()." - QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder*>(user_data); - - int buffersAvailable; - { - QMutexLocker locker(&decoder->m_buffersMutex); - buffersAvailable = decoder->m_buffersAvailable; - decoder->m_buffersAvailable++; - Q_ASSERT(decoder->m_buffersAvailable <= MAX_BUFFERS_IN_QUEUE); - } + // "Note that the preroll buffer will also be returned as the first buffer when calling + // gst_app_sink_pull_buffer()." + + QMetaObject::invokeMethod(this, [this, sessionId = m_currentSessionId] { + if (sessionId != m_currentSessionId) + return; // stop()ed before message is executed + + m_buffersAvailable += 1; + bufferAvailableChanged(true); + bufferReady(); + }); - if (!buffersAvailable) - decoder->bufferAvailableChanged(true); - decoder->bufferReady(); return GST_FLOW_OK; } -void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) +GstFlowReturn QGstreamerAudioDecoder::new_sample(GstAppSink *sink, gpointer user_data) { - if (m_playbin.isNull()) - return; + QGstreamerAudioDecoder *decoder = reinterpret_cast<QGstreamerAudioDecoder *>(user_data); + qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::new_sample"; + return decoder->newSample(sink); +} +void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) +{ int flags = m_playbin.getInt("flags"); // make sure not to use GST_PLAY_FLAG_NATIVE_AUDIO unless desired // it prevents audio format conversion @@ -495,20 +451,32 @@ void QGstreamerAudioDecoder::setAudioFlags(bool wantNativeAudio) void QGstreamerAudioDecoder::addAppSink() { + using namespace std::chrono_literals; + if (m_appSink) return; - m_appSink = (GstAppSink*)gst_element_factory_make("appsink", nullptr); + qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::addAppSink"; + m_appSink = QGstAppSink::create("decoderAppSink"); + GstAppSinkCallbacks callbacks{}; + callbacks.new_sample = new_sample; + m_appSink.setCallbacks(callbacks, this, nullptr); + +#if GST_CHECK_VERSION(1, 24, 0) + static constexpr auto maxBufferTime = 500ms; + m_appSink.setMaxBufferTime(maxBufferTime); +#else + static constexpr int maxBuffers = 16; + m_appSink.setMaxBuffers(maxBuffers); +#endif - GstAppSinkCallbacks callbacks; - memset(&callbacks, 0, sizeof(callbacks)); - callbacks.new_sample = &new_sample; - gst_app_sink_set_callbacks(m_appSink, &callbacks, this, nullptr); - gst_app_sink_set_max_buffers(m_appSink, MAX_BUFFERS_IN_QUEUE); - gst_base_sink_set_sync(GST_BASE_SINK(m_appSink), FALSE); + static constexpr bool sync = false; + m_appSink.setSync(sync); - gst_bin_add(m_outputBin.bin(), GST_ELEMENT(m_appSink)); - gst_element_link(m_audioConvert.element(), GST_ELEMENT(m_appSink)); + QGstPipeline::modifyPipelineWhileNotRunning(m_playbin.getPipeline(), [&] { + m_outputBin.add(m_appSink); + qLinkGstElements(m_audioConvert, m_appSink); + }); } void QGstreamerAudioDecoder::removeAppSink() @@ -516,43 +484,48 @@ void QGstreamerAudioDecoder::removeAppSink() if (!m_appSink) return; - gst_element_unlink(m_audioConvert.element(), GST_ELEMENT(m_appSink)); - gst_bin_remove(m_outputBin.bin(), GST_ELEMENT(m_appSink)); + qCDebug(qLcGstreamerAudioDecoder) << "QGstreamerAudioDecoder::removeAppSink"; - m_appSink = nullptr; + QGstPipeline::modifyPipelineWhileNotRunning(m_playbin.getPipeline(), [&] { + qUnlinkGstElements(m_audioConvert, m_appSink); + m_outputBin.stopAndRemoveElements(m_appSink); + }); + m_appSink = {}; } void QGstreamerAudioDecoder::updateDuration() { - int duration = -1; - - if (!m_playbin.isNull()) - duration = m_playbin.duration() / 1000000; + std::optional<std::chrono::milliseconds> duration = m_playbin.durationInMs(); + if (!duration) + duration = invalidDuration; if (m_duration != duration) { - m_duration = duration; - emit durationChanged(m_duration); + m_duration = *duration; + durationChanged(m_duration.count()); } - if (m_duration > 0) + if (m_duration.count() > 0) m_durationQueries = 0; if (m_durationQueries > 0) { //increase delay between duration requests int delay = 25 << (5 - m_durationQueries); - QTimer::singleShot(delay, this, SLOT(updateDuration())); + QTimer::singleShot(delay, this, &QGstreamerAudioDecoder::updateDuration); m_durationQueries--; } } -qint64 QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer* buffer) +std::chrono::nanoseconds QGstreamerAudioDecoder::getPositionFromBuffer(GstBuffer *buffer) { - qint64 position = GST_BUFFER_TIMESTAMP(buffer); - if (position >= 0) - position = position / G_GINT64_CONSTANT(1000); // microseconds + using namespace std::chrono; + using namespace std::chrono_literals; + nanoseconds position{ GST_BUFFER_TIMESTAMP(buffer) }; + if (position >= 0ns) + return position; else - position = -1; - return position; + return invalidPosition; } QT_END_NAMESPACE + +#include "moc_qgstreameraudiodecoder_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h index 85ff53041..d2d259dde 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodecoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIODECODERCONTROL_H #define QGSTREAMERAUDIODECODERCONTROL_H @@ -51,34 +15,30 @@ // We mean it. // +#include <QtMultimedia/private/qmultimediautils_p.h> +#include <QtMultimedia/private/qplatformaudiodecoder_p.h> #include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include <QObject> +#include <QtMultimedia/qaudiodecoder.h> +#include <QtCore/qobject.h> #include <QtCore/qmutex.h> #include <QtCore/qurl.h> -#include "private/qplatformaudiodecoder_p.h" -#include <qgstpipeline_p.h> -#include "qaudiodecoder.h" +#include <common/qgst_p.h> +#include <common/qgstappsource_p.h> +#include <common/qgstpipeline_p.h> -#if QT_CONFIG(gstreamer_app) -#include <qgstappsrc_p.h> -#endif - -#include <qgst_p.h> #include <gst/app/gstappsink.h> QT_BEGIN_NAMESPACE class QGstreamerMessage; -class QGstreamerAudioDecoder - : public QPlatformAudioDecoder, - public QGstreamerBusMessageFilter +class QGstreamerAudioDecoder final : public QPlatformAudioDecoder, public QGstreamerBusMessageFilter { Q_OBJECT public: - QGstreamerAudioDecoder(QAudioDecoder *parent); + static QMaybe<QPlatformAudioDecoder *> create(QAudioDecoder *parent); virtual ~QGstreamerAudioDecoder(); QUrl source() const override; @@ -94,7 +54,6 @@ public: void setAudioFormat(const QAudioFormat &format) override; QAudioBuffer read() override; - bool bufferAvailable() const override; qint64 position() const override; qint64 duration() const override; @@ -102,41 +61,49 @@ public: // GStreamerBusMessageFilter interface bool processBusMessage(const QGstreamerMessage &message) override; -#if QT_CONFIG(gstreamer_app) - QGstAppSrc *appsrc() const { return m_appSrc; } - static void configureAppSrcElement(GObject*, GObject*, GParamSpec*, QGstreamerAudioDecoder *_this); -#endif - - static GstFlowReturn new_sample(GstAppSink *sink, gpointer user_data); - private slots: void updateDuration(); private: + explicit QGstreamerAudioDecoder(QAudioDecoder *parent); + + static GstFlowReturn new_sample(GstAppSink *sink, gpointer user_data); + GstFlowReturn newSample(GstAppSink *sink); + + static void configureAppSrcElement(GObject *, GObject *, GParamSpec *, + QGstreamerAudioDecoder *_this); + void setAudioFlags(bool wantNativeAudio); void addAppSink(); void removeAppSink(); - void processInvalidMedia(QAudioDecoder::Error errorCode, const QString& errorString); - static qint64 getPositionFromBuffer(GstBuffer* buffer); + bool handlePlaybinMessage(const QGstreamerMessage &); + + void processInvalidMedia(QAudioDecoder::Error errorCode, const QString &errorString); + static std::chrono::nanoseconds getPositionFromBuffer(GstBuffer *buffer); QGstPipeline m_playbin; QGstBin m_outputBin; QGstElement m_audioConvert; - GstAppSink *m_appSink = nullptr; - QGstAppSrc *m_appSrc = nullptr; + QGstAppSink m_appSink; + QGstAppSource *m_appSrc = nullptr; QUrl mSource; QIODevice *mDevice = nullptr; QAudioFormat mFormat; - mutable QMutex m_buffersMutex; int m_buffersAvailable = 0; - qint64 m_position = -1; - qint64 m_duration = -1; + static constexpr auto invalidDuration = std::chrono::milliseconds{ -1 }; + static constexpr auto invalidPosition = std::chrono::milliseconds{ -1 }; + std::chrono::milliseconds m_position{ invalidPosition }; + std::chrono::milliseconds m_duration{ invalidDuration }; int m_durationQueries = 0; + + qint32 m_currentSessionId{}; + + QGObjectHandlerScopedConnection m_deepNotifySourceConnection; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp index aaac46efd..b22e40118 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice.cpp @@ -1,61 +1,28 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreameraudiodevice_p.h" -#include <qgstutils_p.h> +#include <common/qgst_p.h> +#include <common/qgstutils_p.h> #include <private/qplatformmediaintegration_p.h> QT_BEGIN_NAMESPACE -QGStreamerAudioDeviceInfo::QGStreamerAudioDeviceInfo(GstDevice *d, const QByteArray &device, QAudioDevice::Mode mode) +QGStreamerAudioDeviceInfo::QGStreamerAudioDeviceInfo(GstDevice *d, const QByteArray &device, + QAudioDevice::Mode mode) : QAudioDevicePrivate(device, mode), - gstDevice(d) + gstDevice{ + d, + QGstDeviceHandle::NeedsRef, + } { - Q_ASSERT(gstDevice); - gst_object_ref(gstDevice); + QGString name{ + gst_device_get_display_name(gstDevice.get()), + }; + description = name.toQString(); - auto *n = gst_device_get_display_name(gstDevice); - description = QString::fromUtf8(n); - g_free(n); - - QGstCaps caps = gst_device_get_caps(gstDevice); + auto caps = QGstCaps(gst_device_get_caps(gstDevice.get()), QGstCaps::HasRef); int size = caps.size(); for (int i = 0; i < size; ++i) { auto c = caps.at(i); @@ -82,10 +49,39 @@ QGStreamerAudioDeviceInfo::QGStreamerAudioDeviceInfo(GstDevice *d, const QByteAr preferredFormat.setSampleFormat(f); } -QGStreamerAudioDeviceInfo::~QGStreamerAudioDeviceInfo() +QGStreamerCustomAudioDeviceInfo::QGStreamerCustomAudioDeviceInfo( + const QByteArray &gstreamerPipeline, QAudioDevice::Mode mode) + : QAudioDevicePrivate{ + gstreamerPipeline, + mode, + } +{ +} + +QAudioDevice qMakeCustomGStreamerAudioInput(const QByteArray &gstreamerPipeline) +{ + auto deviceInfo = std::make_unique<QGStreamerCustomAudioDeviceInfo>(gstreamerPipeline, + QAudioDevice::Mode::Input); + + return deviceInfo.release()->create(); +} + +QAudioDevice qMakeCustomGStreamerAudioOutput(const QByteArray &gstreamerPipeline) +{ + auto deviceInfo = std::make_unique<QGStreamerCustomAudioDeviceInfo>(gstreamerPipeline, + QAudioDevice::Mode::Output); + + return deviceInfo.release()->create(); +} + +bool isCustomAudioDevice(const QAudioDevicePrivate *device) +{ + return dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(device); +} + +bool isCustomAudioDevice(const QAudioDevice &device) { - if (gstDevice) - gst_object_unref(gstDevice); + return isCustomAudioDevice(device.handle()); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h index 9da109e38..403fd5e74 100644 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h +++ b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiodevice_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIODEVICEINFO_H #define QGSTREAMERAUDIODEVICEINFO_H @@ -55,9 +19,11 @@ #include <QtCore/qstringlist.h> #include <QtCore/qlist.h> -#include "qaudio.h" -#include "qaudiodevice.h" -#include <private/qaudiodevice_p.h> +#include <QtMultimedia/qaudio.h> +#include <QtMultimedia/qaudiodevice.h> +#include <QtMultimedia/private/qaudiodevice_p.h> + +#include <QtQGstreamerMediaPlugin/private/qgst_handle_types_p.h> #include <gst/gst.h> @@ -67,11 +33,22 @@ class QGStreamerAudioDeviceInfo : public QAudioDevicePrivate { public: QGStreamerAudioDeviceInfo(GstDevice *gstDevice, const QByteArray &device, QAudioDevice::Mode mode); - ~QGStreamerAudioDeviceInfo(); - GstDevice *gstDevice = nullptr; + QGstDeviceHandle gstDevice; }; +class QGStreamerCustomAudioDeviceInfo : public QAudioDevicePrivate +{ +public: + QGStreamerCustomAudioDeviceInfo(const QByteArray &gstreamerPipeline, QAudioDevice::Mode mode); +}; + +bool isCustomAudioDevice(const QAudioDevicePrivate *device); +bool isCustomAudioDevice(const QAudioDevice &device); + +QAudioDevice qMakeCustomGStreamerAudioInput(const QByteArray &gstreamerPipeline); +QAudioDevice qMakeCustomGStreamerAudioOutput(const QByteArray &gstreamerPipeline); + QT_END_NAMESPACE #endif diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink.cpp deleted file mode 100644 index f3602186c..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink.cpp +++ /dev/null @@ -1,397 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QtCore/qcoreapplication.h> -#include <QtCore/qdebug.h> -#include <QtCore/qmath.h> -#include <private/qaudiohelpers_p.h> - -#include "qgstreameraudiosink_p.h" -#include "qgstreameraudiodevice_p.h" -#include <sys/types.h> -#include <unistd.h> - -#include <qgstpipeline_p.h> -#include <qgstappsrc_p.h> - -#include <qgstutils_p.h> -#include <qgstreamermessage_p.h> - -QT_BEGIN_NAMESPACE - -QGStreamerAudioSink::QGStreamerAudioSink(const QAudioDevice &device) - : m_device(device.id()), - gstPipeline("pipeline") -{ - gstPipeline.installMessageFilter(this); - - m_appSrc = new QGstAppSrc; - connect(m_appSrc, &QGstAppSrc::bytesProcessed, this, &QGStreamerAudioSink::bytesProcessedByAppSrc); - connect(m_appSrc, &QGstAppSrc::noMoreData, this, &QGStreamerAudioSink::needData); - gstAppSrc = m_appSrc->element(); - - // gstDecodeBin = gst_element_factory_make ("decodebin", "dec"); - QGstElement queue("queue", "queue"); - QGstElement conv("audioconvert", "conv"); - gstVolume = QGstElement("volume", "volume"); - if (m_volume != 1.) - gstVolume.set("volume", m_volume); - - // link decodeBin to audioconvert in a callback once we get a pad from the decoder - // g_signal_connect (gstDecodeBin, "pad-added", (GCallback) padAdded, conv); - - const auto *audioInfo = static_cast<const QGStreamerAudioDeviceInfo *>(device.handle()); - gstOutput = gst_device_create_element(audioInfo->gstDevice, nullptr); - - gstPipeline.add(gstAppSrc, queue, /*gstDecodeBin, */ conv, gstVolume, gstOutput); - gstAppSrc.link(queue, conv, gstVolume, gstOutput); -} - -QGStreamerAudioSink::~QGStreamerAudioSink() -{ - close(); - gstPipeline = {}; - gstVolume = {}; - gstAppSrc = {}; - delete m_appSrc; - m_appSrc = nullptr; -} - -void QGStreamerAudioSink::setError(QAudio::Error error) -{ - if (m_errorState == error) - return; - - m_errorState = error; - emit errorChanged(error); -} - -QAudio::Error QGStreamerAudioSink::error() const -{ - return m_errorState; -} - -void QGStreamerAudioSink::setState(QAudio::State state) -{ - if (m_deviceState == state) - return; - - m_deviceState = state; - emit stateChanged(state); -} - -QAudio::State QGStreamerAudioSink::state() const -{ - return m_deviceState; -} - -void QGStreamerAudioSink::start(QIODevice *device) -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!m_format.isValid()) { - setError(QAudio::OpenError); - return; - } - - m_pullMode = true; - m_audioSource = device; - - if (!open()) { - m_audioSource = nullptr; - setError(QAudio::OpenError); - return; - } - - setState(QAudio::ActiveState); -} - -QIODevice *QGStreamerAudioSink::start() -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!m_format.isValid()) { - setError(QAudio::OpenError); - return nullptr; - } - - m_pullMode = false; - - if (!open()) - return nullptr; - - m_audioSource = new GStreamerOutputPrivate(this); - m_audioSource->open(QIODevice::WriteOnly|QIODevice::Unbuffered); - - setState(QAudio::IdleState); - - return m_audioSource; -} - -#if 0 -static void padAdded(GstElement *element, GstPad *pad, gpointer data) -{ - GstElement *other = static_cast<GstElement *>(data); - - gchar *name = gst_pad_get_name(pad); - qDebug("A new pad %s was created for %s\n", name, gst_element_get_name(element)); - g_free(name); - - qDebug("element %s will be linked to %s\n", - gst_element_get_name(element), - gst_element_get_name(other)); - gst_element_link(element, other); -} -#endif - -bool QGStreamerAudioSink::processBusMessage(const QGstreamerMessage &message) -{ - auto *msg = message.rawMessage(); - switch (GST_MESSAGE_TYPE (msg)) { - case GST_MESSAGE_EOS: - setState(QAudio::IdleState); - break; - case GST_MESSAGE_ERROR: { - setError(QAudio::IOError); - gchar *debug; - GError *error; - - gst_message_parse_error (msg, &error, &debug); - g_free (debug); - - qDebug("Error: %s\n", error->message); - g_error_free (error); - - break; - } - default: - break; - } - - return true; -} - -bool QGStreamerAudioSink::open() -{ - if (m_opened) - return true; - - if (gstOutput.isNull()) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - -// qDebug() << "GST caps:" << gst_caps_to_string(caps); - m_appSrc->setup(m_audioSource, m_audioSource ? m_audioSource->pos() : 0); - m_appSrc->setAudioFormat(m_format); - - /* run */ - gstPipeline.setState(GST_STATE_PLAYING); - - m_opened = true; - - m_timeStamp.restart(); - m_bytesProcessed = 0; - - return true; -} - -void QGStreamerAudioSink::close() -{ - if (!m_opened) - return; - - if (!gstPipeline.setStateSync(GST_STATE_NULL)) - qWarning() << "failed to close the audio output stream"; - - if (!m_pullMode && m_audioSource) - delete m_audioSource; - m_audioSource = nullptr; - m_opened = false; -} - -qint64 QGStreamerAudioSink::write(const char *data, qint64 len) -{ - if (!len) - return 0; - if (m_errorState == QAudio::UnderrunError) - m_errorState = QAudio::NoError; - - m_appSrc->write(data, len); - return len; -} - -void QGStreamerAudioSink::stop() -{ - if (m_deviceState == QAudio::StoppedState) - return; - - close(); - - setError(QAudio::NoError); - setState(QAudio::StoppedState); -} - -qsizetype QGStreamerAudioSink::bytesFree() const -{ - if (m_deviceState != QAudio::ActiveState && m_deviceState != QAudio::IdleState) - return 0; - - return m_appSrc->canAcceptMoreData() ? 4096*4 : 0; -} - -void QGStreamerAudioSink::setBufferSize(qsizetype value) -{ - m_bufferSize = value; - if (!gstAppSrc.isNull()) - gst_app_src_set_max_bytes(GST_APP_SRC(gstAppSrc.element()), value); -} - -qsizetype QGStreamerAudioSink::bufferSize() const -{ - return m_bufferSize; -} - -qint64 QGStreamerAudioSink::processedUSecs() const -{ - qint64 result = qint64(1000000) * m_bytesProcessed / - m_format.bytesPerFrame() / - m_format.sampleRate(); - - return result; -} - -void QGStreamerAudioSink::resume() -{ - if (m_deviceState == QAudio::SuspendedState) { - m_appSrc->resume(); - gstPipeline.setState(GST_STATE_PLAYING); - - setState(m_pullMode ? QAudio::ActiveState : QAudio::IdleState); - setError(QAudio::NoError); - } -} - -void QGStreamerAudioSink::setFormat(const QAudioFormat &format) -{ - m_format = format; -} - -QAudioFormat QGStreamerAudioSink::format() const -{ - return m_format; -} - -void QGStreamerAudioSink::suspend() -{ - if (m_deviceState == QAudio::ActiveState || m_deviceState == QAudio::IdleState) { - setError(QAudio::NoError); - setState(QAudio::SuspendedState); - - gstPipeline.setState(GST_STATE_PAUSED); - m_appSrc->suspend(); - // ### elapsed time - } -} - -void QGStreamerAudioSink::reset() -{ - stop(); -} - -GStreamerOutputPrivate::GStreamerOutputPrivate(QGStreamerAudioSink *audio) -{ - m_audioDevice = audio; -} - -qint64 GStreamerOutputPrivate::readData(char *data, qint64 len) -{ - Q_UNUSED(data); - Q_UNUSED(len); - - return 0; -} - -qint64 GStreamerOutputPrivate::writeData(const char *data, qint64 len) -{ - if (m_audioDevice->state() == QAudio::IdleState) - m_audioDevice->setState(QAudio::ActiveState); - return m_audioDevice->write(data, len); -} - -void QGStreamerAudioSink::setVolume(qreal vol) -{ - if (m_volume == vol) - return; - - m_volume = vol; - if (!gstVolume.isNull()) - gstVolume.set("volume", vol); -} - -qreal QGStreamerAudioSink::volume() const -{ - return m_volume; -} - -void QGStreamerAudioSink::bytesProcessedByAppSrc(int bytes) -{ - m_bytesProcessed += bytes; - setState(QAudio::ActiveState); - setError(QAudio::NoError); -} - -void QGStreamerAudioSink::needData() -{ - if (state() != QAudio::StoppedState && state() != QAudio::IdleState) { - setState(QAudio::IdleState); - setError(m_audioSource && m_audioSource->atEnd() ? QAudio::NoError : QAudio::UnderrunError); - } -} - -QT_END_NAMESPACE - -#include "moc_qgstreameraudiosink_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink_p.h deleted file mode 100644 index 080fe1cbf..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosink_p.h +++ /dev/null @@ -1,157 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QAUDIOOUTPUTGSTREAMER_H -#define QAUDIOOUTPUTGSTREAMER_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <QtCore/qfile.h> -#include <QtCore/qtimer.h> -#include <QtCore/qstring.h> -#include <QtCore/qstringlist.h> -#include <QtCore/qelapsedtimer.h> -#include <QtCore/qiodevice.h> -#include <QtCore/private/qringbuffer_p.h> - -#include "qaudio.h" -#include "qaudiodevice.h" -#include <private/qaudiosystem_p.h> - -#include <qgst_p.h> -#include <qgstpipeline_p.h> - -QT_BEGIN_NAMESPACE - -class QGstAppSrc; - -class QGStreamerAudioSink - : public QPlatformAudioSink, - public QGstreamerBusMessageFilter -{ - friend class GStreamerOutputPrivate; - Q_OBJECT - -public: - QGStreamerAudioSink(const QAudioDevice &device); - ~QGStreamerAudioSink(); - - void start(QIODevice *device) override; - QIODevice *start() override; - void stop() override; - void reset() override; - void suspend() override; - void resume() override; - qsizetype bytesFree() const override; - void setBufferSize(qsizetype value) override; - qsizetype bufferSize() const override; - qint64 processedUSecs() const override; - QAudio::Error error() const override; - QAudio::State state() const override; - void setFormat(const QAudioFormat &format) override; - QAudioFormat format() const override; - - void setVolume(qreal volume) override; - qreal volume() const override; - -private Q_SLOTS: - void bytesProcessedByAppSrc(int bytes); - void needData(); - -private: - void setState(QAudio::State state); - void setError(QAudio::Error error); - - bool processBusMessage(const QGstreamerMessage &message) override; - - bool open(); - void close(); - qint64 write(const char *data, qint64 len); - -private: - QByteArray m_device; - QAudioFormat m_format; - QAudio::Error m_errorState = QAudio::NoError; - QAudio::State m_deviceState = QAudio::StoppedState; - bool m_pullMode = true; - bool m_opened = false; - QIODevice *m_audioSource = nullptr; - QTimer m_periodTimer; - int m_bufferSize = 0; - qint64 m_bytesProcessed = 0; - QElapsedTimer m_timeStamp; - qreal m_volume = 1.; - QByteArray pushData; - - QGstPipeline gstPipeline; - QGstElement gstOutput; - QGstElement gstVolume; - QGstElement gstAppSrc; - QGstAppSrc *m_appSrc = nullptr; -}; - -class GStreamerOutputPrivate : public QIODevice -{ - friend class QGStreamerAudioSink; - Q_OBJECT - -public: - GStreamerOutputPrivate(QGStreamerAudioSink *audio); - virtual ~GStreamerOutputPrivate() {} - -protected: - qint64 readData(char *data, qint64 len) override; - qint64 writeData(const char *data, qint64 len) override; - -private: - QGStreamerAudioSink *m_audioDevice; -}; - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource.cpp b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource.cpp deleted file mode 100644 index ffa402627..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource.cpp +++ /dev/null @@ -1,407 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QtCore/qcoreapplication.h> -#include <QtCore/qdebug.h> -#include <QtCore/qmath.h> -#include <private/qaudiohelpers_p.h> - -#include "qgstreameraudiosource_p.h" -#include "qgstreameraudiodevice_p.h" -#include <sys/types.h> -#include <unistd.h> - -#include <gst/gst.h> -Q_DECLARE_OPAQUE_POINTER(GstSample *); -Q_DECLARE_METATYPE(GstSample *); - -QT_BEGIN_NAMESPACE - - -QGStreamerAudioSource::QGStreamerAudioSource(const QAudioDevice &device) - : m_info(device), - m_device(device.id()) -{ - qRegisterMetaType<GstSample *>(); -} - -QGStreamerAudioSource::~QGStreamerAudioSource() -{ - close(); -} - -void QGStreamerAudioSource::setError(QAudio::Error error) -{ - if (m_errorState == error) - return; - - m_errorState = error; - emit errorChanged(error); -} - -QAudio::Error QGStreamerAudioSource::error() const -{ - return m_errorState; -} - -void QGStreamerAudioSource::setState(QAudio::State state) -{ - if (m_deviceState == state) - return; - - m_deviceState = state; - emit stateChanged(state); -} - -QAudio::State QGStreamerAudioSource::state() const -{ - return m_deviceState; -} - -void QGStreamerAudioSource::setFormat(const QAudioFormat &format) -{ - if (m_deviceState == QAudio::StoppedState) - m_format = format; -} - -QAudioFormat QGStreamerAudioSource::format() const -{ - return m_format; -} - -void QGStreamerAudioSource::start(QIODevice *device) -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!open()) - return; - - m_pullMode = true; - m_audioSink = device; - - setState(QAudio::ActiveState); -} - -QIODevice *QGStreamerAudioSource::start() -{ - setState(QAudio::StoppedState); - setError(QAudio::NoError); - - close(); - - if (!open()) - return nullptr; - - m_pullMode = false; - m_audioSink = new GStreamerInputPrivate(this); - m_audioSink->open(QIODevice::ReadOnly | QIODevice::Unbuffered); - - setState(QAudio::IdleState); - - return m_audioSink; -} - -void QGStreamerAudioSource::stop() -{ - if (m_deviceState == QAudio::StoppedState) - return; - - close(); - - setError(QAudio::NoError); - setState(QAudio::StoppedState); -} - -bool QGStreamerAudioSource::open() -{ - if (m_opened) - return true; - - const auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_info.handle()); - if (!deviceInfo->gstDevice) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - - gstInput = QGstElement(gst_device_create_element(deviceInfo->gstDevice, nullptr)); - if (gstInput.isNull()) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - - auto gstCaps = QGstUtils::capsForAudioFormat(m_format); - - if (gstCaps.isNull()) { - setError(QAudio::OpenError); - setState(QAudio::StoppedState); - return false; - } - - -#ifdef DEBUG_AUDIO - qDebug() << "Opening input" << QTime::currentTime(); - qDebug() << "Caps: " << gst_caps_to_string(gstCaps); -#endif - - gstPipeline = QGstPipeline("pipeline"); - - auto *gstBus = gst_pipeline_get_bus(gstPipeline.pipeline()); - gst_bus_add_watch(gstBus, &QGStreamerAudioSource::busMessage, this); - gst_object_unref (gstBus); - - gstAppSink = createAppSink(); - gstAppSink.set("caps", gstCaps); - - QGstElement conv("audioconvert", "conv"); - gstVolume = QGstElement("volume", "volume"); - if (m_volume != 1.) - gstVolume.set("volume", m_volume); - - gstPipeline.add(gstInput, gstVolume, conv, gstAppSink); - gstInput.link(gstVolume, conv, gstAppSink); - - gstPipeline.setState(GST_STATE_PLAYING); - - m_opened = true; - - m_timeStamp.restart(); - m_elapsedTimeOffset = 0; - m_bytesWritten = 0; - - return true; -} - -void QGStreamerAudioSource::close() -{ - if (!m_opened) - return; - - gstPipeline.setState(GST_STATE_NULL); - gstPipeline = {}; - gstVolume = {}; - gstAppSink = {}; - gstInput = {}; - - if (!m_pullMode && m_audioSink) { - delete m_audioSink; - } - m_audioSink = nullptr; - m_opened = false; -} - -gboolean QGStreamerAudioSource::busMessage(GstBus *, GstMessage *msg, gpointer user_data) -{ - QGStreamerAudioSource *input = static_cast<QGStreamerAudioSource *>(user_data); - switch (GST_MESSAGE_TYPE (msg)) { - case GST_MESSAGE_EOS: - input->stop(); - break; - case GST_MESSAGE_ERROR: { - input->setError(QAudio::IOError); - gchar *debug; - GError *error; - - gst_message_parse_error (msg, &error, &debug); - g_free (debug); - - qDebug("Error: %s\n", error->message); - g_error_free (error); - - break; - } - default: - break; - } - return false; -} - -qsizetype QGStreamerAudioSource::bytesReady() const -{ - return m_buffer.size(); -} - -void QGStreamerAudioSource::resume() -{ - if (m_deviceState == QAudio::SuspendedState || m_deviceState == QAudio::IdleState) { - gstPipeline.setState(GST_STATE_PLAYING); - setState(QAudio::ActiveState); - setError(QAudio::NoError); - } -} - -void QGStreamerAudioSource::setVolume(qreal vol) -{ - if (m_volume == vol) - return; - - m_volume = vol; - if (!gstVolume.isNull()) - gstVolume.set("volume", vol); -} - -qreal QGStreamerAudioSource::volume() const -{ - return m_volume; -} - -void QGStreamerAudioSource::setBufferSize(qsizetype value) -{ - m_bufferSize = value; -} - -qsizetype QGStreamerAudioSource::bufferSize() const -{ - return m_bufferSize; -} - -qint64 QGStreamerAudioSource::processedUSecs() const -{ - return m_format.durationForBytes(m_bytesWritten); -} - -void QGStreamerAudioSource::suspend() -{ - if (m_deviceState == QAudio::ActiveState) { - setError(QAudio::NoError); - setState(QAudio::SuspendedState); - - gstPipeline.setState(GST_STATE_PAUSED); - } -} - -void QGStreamerAudioSource::reset() -{ - stop(); - m_buffer.clear(); -} - -//#define MAX_BUFFERS_IN_QUEUE 4 - -QGstElement QGStreamerAudioSource::createAppSink() -{ - QGstElement sink("appsink", "appsink"); - GstAppSink *appSink = reinterpret_cast<GstAppSink *>(sink.element()); - - GstAppSinkCallbacks callbacks; - memset(&callbacks, 0, sizeof(callbacks)); - callbacks.eos = &eos; - callbacks.new_sample = &new_sample; - gst_app_sink_set_callbacks(appSink, &callbacks, this, nullptr); -// gst_app_sink_set_max_buffers(appSink, MAX_BUFFERS_IN_QUEUE); - gst_base_sink_set_sync(GST_BASE_SINK(appSink), FALSE); - - return sink; -} - -void QGStreamerAudioSource::newDataAvailable(GstSample *sample) -{ - if (m_audioSink) { - GstBuffer *buffer = gst_sample_get_buffer(sample); - GstMapInfo mapInfo; - gst_buffer_map(buffer, &mapInfo, GST_MAP_READ); - const char *bufferData = (const char*)mapInfo.data; - gsize bufferSize = mapInfo.size; - - if (!m_pullMode) { - // need to store that data in the QBuffer - m_buffer.append(bufferData, bufferSize); - m_audioSink->readyRead(); - } else { - m_bytesWritten += bufferSize; - m_audioSink->write(bufferData, bufferSize); - } - - gst_buffer_unmap(buffer, &mapInfo); - } - - gst_sample_unref(sample); -} - -GstFlowReturn QGStreamerAudioSource::new_sample(GstAppSink *sink, gpointer user_data) -{ - // "Note that the preroll buffer will also be returned as the first buffer when calling gst_app_sink_pull_buffer()." - QGStreamerAudioSource *control = static_cast<QGStreamerAudioSource*>(user_data); - - GstSample *sample = gst_app_sink_pull_sample(sink); - QMetaObject::invokeMethod(control, "newDataAvailable", Qt::AutoConnection, Q_ARG(GstSample *, sample)); - - return GST_FLOW_OK; -} - -void QGStreamerAudioSource::eos(GstAppSink *, gpointer user_data) -{ - QGStreamerAudioSource *control = static_cast<QGStreamerAudioSource*>(user_data); - control->setState(QAudio::StoppedState); -} - -GStreamerInputPrivate::GStreamerInputPrivate(QGStreamerAudioSource *audio) -{ - m_audioDevice = qobject_cast<QGStreamerAudioSource*>(audio); -} - -qint64 GStreamerInputPrivate::readData(char *data, qint64 len) -{ - if (m_audioDevice->state() == QAudio::IdleState) - m_audioDevice->setState(QAudio::ActiveState); - qint64 bytes = m_audioDevice->m_buffer.read(data, len); - m_audioDevice->m_bytesWritten += bytes; - return bytes; -} - -qint64 GStreamerInputPrivate::writeData(const char *data, qint64 len) -{ - Q_UNUSED(data); - Q_UNUSED(len); - return 0; -} - -qint64 GStreamerInputPrivate::bytesAvailable() const -{ - return m_audioDevice->m_buffer.size(); -} - - -QT_END_NAMESPACE - -#include "moc_qgstreameraudiosource_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource_p.h b/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource_p.h deleted file mode 100644 index 5a04702ca..000000000 --- a/src/plugins/multimedia/gstreamer/audio/qgstreameraudiosource_p.h +++ /dev/null @@ -1,159 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists for the convenience -// of other Qt classes. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#ifndef QAUDIOINPUTGSTREAMER_H -#define QAUDIOINPUTGSTREAMER_H - -#include <QtCore/qfile.h> -#include <QtCore/qtimer.h> -#include <QtCore/qstring.h> -#include <QtCore/qstringlist.h> -#include <QtCore/qelapsedtimer.h> -#include <QtCore/qiodevice.h> -#include <QtCore/qmutex.h> -#include <QtCore/qatomic.h> -#include <QtCore/private/qringbuffer_p.h> - -#include "qaudio.h" -#include "qaudiodevice.h" -#include <private/qaudiosystem_p.h> - -#include <qgstutils_p.h> -#include <qgstpipeline_p.h> -#include <gst/app/gstappsink.h> - -QT_BEGIN_NAMESPACE - -class GStreamerInputPrivate; - -class QGStreamerAudioSource - : public QPlatformAudioSource -{ - Q_OBJECT - friend class GStreamerInputPrivate; -public: - QGStreamerAudioSource(const QAudioDevice &device); - ~QGStreamerAudioSource(); - - void start(QIODevice *device) override; - QIODevice *start() override; - void stop() override; - void reset() override; - void suspend() override; - void resume() override; - qsizetype bytesReady() const override; - void setBufferSize(qsizetype value) override; - qsizetype bufferSize() const override; - qint64 processedUSecs() const override; - QAudio::Error error() const override; - QAudio::State state() const override; - void setFormat(const QAudioFormat &format) override; - QAudioFormat format() const override; - - void setVolume(qreal volume) override; - qreal volume() const override; - -private Q_SLOTS: - void newDataAvailable(GstSample *sample); - -private: - void setState(QAudio::State state); - void setError(QAudio::Error error); - - QGstElement createAppSink(); - static GstFlowReturn new_sample(GstAppSink *, gpointer user_data); - static void eos(GstAppSink *, gpointer user_data); - - bool open(); - void close(); - - static gboolean busMessage(GstBus *bus, GstMessage *msg, gpointer user_data); - - QAudioDevice m_info; - qint64 m_bytesWritten = 0; - QIODevice *m_audioSink = nullptr; - QAudioFormat m_format; - QAudio::Error m_errorState = QAudio::NoError; - QAudio::State m_deviceState = QAudio::StoppedState; - qreal m_volume = 1.; - - QRingBuffer m_buffer; - QAtomicInteger<bool> m_pullMode = true; - bool m_opened = false; - int m_bufferSize = 0; - qint64 m_elapsedTimeOffset = 0; - QElapsedTimer m_timeStamp; - QByteArray m_device; - QByteArray m_tempBuffer; - - QGstElement gstInput; - QGstPipeline gstPipeline; - QGstElement gstVolume; - QGstElement gstAppSink; -}; - -class GStreamerInputPrivate : public QIODevice -{ - Q_OBJECT -public: - GStreamerInputPrivate(QGStreamerAudioSource *audio); - ~GStreamerInputPrivate() {}; - - qint64 readData(char *data, qint64 len) override; - qint64 writeData(const char *data, qint64 len) override; - qint64 bytesAvailable() const override; - bool isSequential() const override { return true; } -private: - QGStreamerAudioSource *m_audioDevice; -}; - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/gstreamer/common/qglist_helper_p.h b/src/plugins/multimedia/gstreamer/common/qglist_helper_p.h new file mode 100644 index 000000000..54108e1c3 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qglist_helper_p.h @@ -0,0 +1,82 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGLIST_HELPER_P_H +#define QGLIST_HELPER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/qtconfigmacros.h> + +#include <glib.h> +#include <iterator> + +QT_BEGIN_NAMESPACE + +namespace QGstUtils { + +template <typename ListType> +struct GListIterator +{ + explicit GListIterator(const GList *element = nullptr) : element(element) { } + + const ListType &operator*() const noexcept { return *operator->(); } + const ListType *operator->() const noexcept + { + return reinterpret_cast<const ListType *>(&element->data); + } + + GListIterator &operator++() noexcept + { + if (element) + element = element->next; + + return *this; + } + GListIterator operator++(int n) noexcept + { + for (int i = 0; i != n; ++i) + operator++(); + + return *this; + } + + bool operator==(const GListIterator &r) const noexcept { return element == r.element; } + bool operator!=(const GListIterator &r) const noexcept { return element != r.element; } + + using difference_type = std::ptrdiff_t; + using value_type = ListType; + using pointer = value_type *; + using reference = value_type &; + using iterator_category = std::input_iterator_tag; + + const GList *element = nullptr; +}; + +template <typename ListType> +struct GListRangeAdaptor +{ + static_assert(std::is_pointer_v<ListType>); + + explicit GListRangeAdaptor(const GList *list) : head(list) { } + + auto begin() { return GListIterator<ListType>(head); } + auto end() { return GListIterator<ListType>(nullptr); } + + const GList *head; +}; + +} // namespace QGstUtils + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgst.cpp b/src/plugins/multimedia/gstreamer/common/qgst.cpp new file mode 100644 index 000000000..cb1f38495 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst.cpp @@ -0,0 +1,1404 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgst_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamermessage_p.h> + +#include <QtCore/qdebug.h> +#include <QtMultimedia/qcameradevice.h> + +#include <array> + +QT_BEGIN_NAMESPACE + +namespace { + +struct VideoFormat +{ + QVideoFrameFormat::PixelFormat pixelFormat; + GstVideoFormat gstFormat; +}; + +constexpr std::array<VideoFormat, 19> qt_videoFormatLookup{ { + { QVideoFrameFormat::Format_YUV420P, GST_VIDEO_FORMAT_I420 }, + { QVideoFrameFormat::Format_YUV422P, GST_VIDEO_FORMAT_Y42B }, + { QVideoFrameFormat::Format_YV12, GST_VIDEO_FORMAT_YV12 }, + { QVideoFrameFormat::Format_UYVY, GST_VIDEO_FORMAT_UYVY }, + { QVideoFrameFormat::Format_YUYV, GST_VIDEO_FORMAT_YUY2 }, + { QVideoFrameFormat::Format_NV12, GST_VIDEO_FORMAT_NV12 }, + { QVideoFrameFormat::Format_NV21, GST_VIDEO_FORMAT_NV21 }, + { QVideoFrameFormat::Format_AYUV, GST_VIDEO_FORMAT_AYUV }, + { QVideoFrameFormat::Format_Y8, GST_VIDEO_FORMAT_GRAY8 }, + { QVideoFrameFormat::Format_XRGB8888, GST_VIDEO_FORMAT_xRGB }, + { QVideoFrameFormat::Format_XBGR8888, GST_VIDEO_FORMAT_xBGR }, + { QVideoFrameFormat::Format_RGBX8888, GST_VIDEO_FORMAT_RGBx }, + { QVideoFrameFormat::Format_BGRX8888, GST_VIDEO_FORMAT_BGRx }, + { QVideoFrameFormat::Format_ARGB8888, GST_VIDEO_FORMAT_ARGB }, + { QVideoFrameFormat::Format_ABGR8888, GST_VIDEO_FORMAT_ABGR }, + { QVideoFrameFormat::Format_RGBA8888, GST_VIDEO_FORMAT_RGBA }, + { QVideoFrameFormat::Format_BGRA8888, GST_VIDEO_FORMAT_BGRA }, +#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN + { QVideoFrameFormat::Format_Y16, GST_VIDEO_FORMAT_GRAY16_LE }, + { QVideoFrameFormat::Format_P010, GST_VIDEO_FORMAT_P010_10LE }, +#else + { QVideoFrameFormat::Format_Y16, GST_VIDEO_FORMAT_GRAY16_BE }, + { QVideoFrameFormat::Format_P010, GST_VIDEO_FORMAT_P010_10BE }, +#endif +} }; + +int indexOfVideoFormat(QVideoFrameFormat::PixelFormat format) +{ + for (size_t i = 0; i < qt_videoFormatLookup.size(); ++i) + if (qt_videoFormatLookup[i].pixelFormat == format) + return int(i); + + return -1; +} + +int indexOfVideoFormat(GstVideoFormat format) +{ + for (size_t i = 0; i < qt_videoFormatLookup.size(); ++i) + if (qt_videoFormatLookup[i].gstFormat == format) + return int(i); + + return -1; +} + +} // namespace + +// QGValue + +QGValue::QGValue(const GValue *v) : value(v) { } + +bool QGValue::isNull() const +{ + return !value; +} + +std::optional<bool> QGValue::toBool() const +{ + if (!G_VALUE_HOLDS_BOOLEAN(value)) + return std::nullopt; + return g_value_get_boolean(value); +} + +std::optional<int> QGValue::toInt() const +{ + if (!G_VALUE_HOLDS_INT(value)) + return std::nullopt; + return g_value_get_int(value); +} + +std::optional<int> QGValue::toInt64() const +{ + if (!G_VALUE_HOLDS_INT64(value)) + return std::nullopt; + return g_value_get_int64(value); +} + +const char *QGValue::toString() const +{ + return value ? g_value_get_string(value) : nullptr; +} + +std::optional<float> QGValue::getFraction() const +{ + if (!GST_VALUE_HOLDS_FRACTION(value)) + return std::nullopt; + return (float)gst_value_get_fraction_numerator(value) + / (float)gst_value_get_fraction_denominator(value); +} + +std::optional<QGRange<float>> QGValue::getFractionRange() const +{ + if (!GST_VALUE_HOLDS_FRACTION_RANGE(value)) + return std::nullopt; + QGValue min = QGValue{ gst_value_get_fraction_range_min(value) }; + QGValue max = QGValue{ gst_value_get_fraction_range_max(value) }; + return QGRange<float>{ *min.getFraction(), *max.getFraction() }; +} + +std::optional<QGRange<int>> QGValue::toIntRange() const +{ + if (!GST_VALUE_HOLDS_INT_RANGE(value)) + return std::nullopt; + return QGRange<int>{ gst_value_get_int_range_min(value), gst_value_get_int_range_max(value) }; +} + +QGstStructureView QGValue::toStructure() const +{ + if (!value || !GST_VALUE_HOLDS_STRUCTURE(value)) + return QGstStructureView(nullptr); + return QGstStructureView(gst_value_get_structure(value)); +} + +QGstCaps QGValue::toCaps() const +{ + if (!value || !GST_VALUE_HOLDS_CAPS(value)) + return {}; + return QGstCaps(gst_caps_copy(gst_value_get_caps(value)), QGstCaps::HasRef); +} + +bool QGValue::isList() const +{ + return value && GST_VALUE_HOLDS_LIST(value); +} + +int QGValue::listSize() const +{ + return gst_value_list_get_size(value); +} + +QGValue QGValue::at(int index) const +{ + return QGValue{ gst_value_list_get_value(value, index) }; +} + +// QGstStructureView + +QGstStructureView::QGstStructureView(const GstStructure *s) : structure(s) { } + +QGstStructureView::QGstStructureView(const QUniqueGstStructureHandle &handle) + : QGstStructureView{ handle.get() } +{ +} + +QUniqueGstStructureHandle QGstStructureView::clone() const +{ + return QUniqueGstStructureHandle{ gst_structure_copy(structure) }; +} + +bool QGstStructureView::isNull() const +{ + return !structure; +} + +QByteArrayView QGstStructureView::name() const +{ + return gst_structure_get_name(structure); +} + +QGValue QGstStructureView::operator[](const char *fieldname) const +{ + return QGValue{ gst_structure_get_value(structure, fieldname) }; +} + +QGstCaps QGstStructureView::caps() const +{ + return operator[]("caps").toCaps(); +} + +QGstTagListHandle QGstStructureView::tags() const +{ + QGValue tags = operator[]("tags"); + if (tags.isNull()) + return {}; + + QGstTagListHandle tagList; + gst_structure_get(structure, "tags", GST_TYPE_TAG_LIST, &tagList, nullptr); + return tagList; +} + +QSize QGstStructureView::resolution() const +{ + QSize size; + + int w, h; + if (structure && gst_structure_get_int(structure, "width", &w) + && gst_structure_get_int(structure, "height", &h)) { + size.rwidth() = w; + size.rheight() = h; + } + + return size; +} + +QVideoFrameFormat::PixelFormat QGstStructureView::pixelFormat() const +{ + QVideoFrameFormat::PixelFormat pixelFormat = QVideoFrameFormat::Format_Invalid; + + if (!structure) + return pixelFormat; + + if (gst_structure_has_name(structure, "video/x-raw")) { + const gchar *s = gst_structure_get_string(structure, "format"); + if (s) { + GstVideoFormat format = gst_video_format_from_string(s); + int index = indexOfVideoFormat(format); + + if (index != -1) + pixelFormat = qt_videoFormatLookup[index].pixelFormat; + } + } else if (gst_structure_has_name(structure, "image/jpeg")) { + pixelFormat = QVideoFrameFormat::Format_Jpeg; + } + + return pixelFormat; +} + +QGRange<float> QGstStructureView::frameRateRange() const +{ + float minRate = 0.; + float maxRate = 0.; + + if (!structure) + return { 0.f, 0.f }; + + auto extractFraction = [](const GValue *v) -> float { + return (float)gst_value_get_fraction_numerator(v) + / (float)gst_value_get_fraction_denominator(v); + }; + auto extractFrameRate = [&](const GValue *v) { + auto insert = [&](float min, float max) { + if (max > maxRate) + maxRate = max; + if (min < minRate) + minRate = min; + }; + + if (GST_VALUE_HOLDS_FRACTION(v)) { + float rate = extractFraction(v); + insert(rate, rate); + } else if (GST_VALUE_HOLDS_FRACTION_RANGE(v)) { + auto *min = gst_value_get_fraction_range_max(v); + auto *max = gst_value_get_fraction_range_max(v); + insert(extractFraction(min), extractFraction(max)); + } + }; + + const GValue *gstFrameRates = gst_structure_get_value(structure, "framerate"); + if (gstFrameRates) { + if (GST_VALUE_HOLDS_LIST(gstFrameRates)) { + guint nFrameRates = gst_value_list_get_size(gstFrameRates); + for (guint f = 0; f < nFrameRates; ++f) { + extractFrameRate(gst_value_list_get_value(gstFrameRates, f)); + } + } else { + extractFrameRate(gstFrameRates); + } + } else { + const GValue *min = gst_structure_get_value(structure, "min-framerate"); + const GValue *max = gst_structure_get_value(structure, "max-framerate"); + if (min && max) { + minRate = extractFraction(min); + maxRate = extractFraction(max); + } + } + + return { minRate, maxRate }; +} + +QGstreamerMessage QGstStructureView::getMessage() +{ + GstMessage *message = nullptr; + gst_structure_get(structure, "message", GST_TYPE_MESSAGE, &message, nullptr); + return QGstreamerMessage(message, QGstreamerMessage::HasRef); +} + +std::optional<Fraction> QGstStructureView::pixelAspectRatio() const +{ + gint numerator; + gint denominator; + if (gst_structure_get_fraction(structure, "pixel-aspect-ratio", &numerator, &denominator)) { + return Fraction{ + numerator, + denominator, + }; + } + + return std::nullopt; +} + +// QTBUG-125249: gstreamer tries "to keep the input height (because of interlacing)". Can we align +// the behavior between gstreamer and ffmpeg? +static QSize qCalculateFrameSizeGStreamer(QSize resolution, Fraction par) +{ + if (par.numerator == par.denominator || par.numerator < 1 || par.denominator < 1) + return resolution; + + return QSize{ + resolution.width() * par.numerator / par.denominator, + resolution.height(), + }; +} + +QSize QGstStructureView::nativeSize() const +{ + QSize size = resolution(); + if (!size.isValid()) { + qWarning() << Q_FUNC_INFO << "invalid resolution when querying nativeSize"; + return size; + } + + std::optional<Fraction> par = pixelAspectRatio(); + if (par) + size = qCalculateFrameSizeGStreamer(size, *par); + return size; +} + +// QGstCaps + +std::optional<std::pair<QVideoFrameFormat, GstVideoInfo>> QGstCaps::formatAndVideoInfo() const +{ + GstVideoInfo vidInfo; + + bool success = gst_video_info_from_caps(&vidInfo, get()); + if (!success) + return std::nullopt; + + int index = indexOfVideoFormat(vidInfo.finfo->format); + if (index == -1) + return std::nullopt; + + QVideoFrameFormat format(QSize(vidInfo.width, vidInfo.height), + qt_videoFormatLookup[index].pixelFormat); + + if (vidInfo.fps_d > 0) + format.setStreamFrameRate(qreal(vidInfo.fps_n) / vidInfo.fps_d); + + QVideoFrameFormat::ColorRange range = QVideoFrameFormat::ColorRange_Unknown; + switch (vidInfo.colorimetry.range) { + case GST_VIDEO_COLOR_RANGE_UNKNOWN: + break; + case GST_VIDEO_COLOR_RANGE_0_255: + range = QVideoFrameFormat::ColorRange_Full; + break; + case GST_VIDEO_COLOR_RANGE_16_235: + range = QVideoFrameFormat::ColorRange_Video; + break; + } + format.setColorRange(range); + + QVideoFrameFormat::ColorSpace colorSpace = QVideoFrameFormat::ColorSpace_Undefined; + switch (vidInfo.colorimetry.matrix) { + case GST_VIDEO_COLOR_MATRIX_UNKNOWN: + case GST_VIDEO_COLOR_MATRIX_RGB: + case GST_VIDEO_COLOR_MATRIX_FCC: + break; + case GST_VIDEO_COLOR_MATRIX_BT709: + colorSpace = QVideoFrameFormat::ColorSpace_BT709; + break; + case GST_VIDEO_COLOR_MATRIX_BT601: + colorSpace = QVideoFrameFormat::ColorSpace_BT601; + break; + case GST_VIDEO_COLOR_MATRIX_SMPTE240M: + colorSpace = QVideoFrameFormat::ColorSpace_AdobeRgb; + break; + case GST_VIDEO_COLOR_MATRIX_BT2020: + colorSpace = QVideoFrameFormat::ColorSpace_BT2020; + break; + } + format.setColorSpace(colorSpace); + + QVideoFrameFormat::ColorTransfer transfer = QVideoFrameFormat::ColorTransfer_Unknown; + switch (vidInfo.colorimetry.transfer) { + case GST_VIDEO_TRANSFER_UNKNOWN: + break; + case GST_VIDEO_TRANSFER_GAMMA10: + transfer = QVideoFrameFormat::ColorTransfer_Linear; + break; + case GST_VIDEO_TRANSFER_GAMMA22: + case GST_VIDEO_TRANSFER_SMPTE240M: + case GST_VIDEO_TRANSFER_SRGB: + case GST_VIDEO_TRANSFER_ADOBERGB: + transfer = QVideoFrameFormat::ColorTransfer_Gamma22; + break; + case GST_VIDEO_TRANSFER_GAMMA18: + case GST_VIDEO_TRANSFER_GAMMA20: + // not quite, but best fit + case GST_VIDEO_TRANSFER_BT709: + case GST_VIDEO_TRANSFER_BT2020_12: + transfer = QVideoFrameFormat::ColorTransfer_BT709; + break; + case GST_VIDEO_TRANSFER_GAMMA28: + transfer = QVideoFrameFormat::ColorTransfer_Gamma28; + break; + case GST_VIDEO_TRANSFER_LOG100: + case GST_VIDEO_TRANSFER_LOG316: + break; +#if GST_CHECK_VERSION(1, 18, 0) + case GST_VIDEO_TRANSFER_SMPTE2084: + transfer = QVideoFrameFormat::ColorTransfer_ST2084; + break; + case GST_VIDEO_TRANSFER_ARIB_STD_B67: + transfer = QVideoFrameFormat::ColorTransfer_STD_B67; + break; + case GST_VIDEO_TRANSFER_BT2020_10: + transfer = QVideoFrameFormat::ColorTransfer_BT709; + break; + case GST_VIDEO_TRANSFER_BT601: + transfer = QVideoFrameFormat::ColorTransfer_BT601; + break; +#endif + } + format.setColorTransfer(transfer); + + return std::pair{ + std::move(format), + vidInfo, + }; +} + +void QGstCaps::addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, + const char *modifier) +{ + if (!gst_caps_is_writable(get())) + *this = QGstCaps(gst_caps_make_writable(release()), QGstCaps::RefMode::HasRef); + + GValue list = {}; + g_value_init(&list, GST_TYPE_LIST); + + for (QVideoFrameFormat::PixelFormat format : formats) { + int index = indexOfVideoFormat(format); + if (index == -1) + continue; + GValue item = {}; + + g_value_init(&item, G_TYPE_STRING); + g_value_set_string(&item, + gst_video_format_to_string(qt_videoFormatLookup[index].gstFormat)); + gst_value_list_append_value(&list, &item); + g_value_unset(&item); + } + + auto *structure = gst_structure_new("video/x-raw", "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, + INT_MAX, 1, "width", GST_TYPE_INT_RANGE, 1, INT_MAX, + "height", GST_TYPE_INT_RANGE, 1, INT_MAX, nullptr); + gst_structure_set_value(structure, "format", &list); + gst_caps_append_structure(get(), structure); + g_value_unset(&list); + + if (modifier) + gst_caps_set_features(get(), size() - 1, gst_caps_features_from_string(modifier)); +} + +void QGstCaps::setResolution(QSize resolution) +{ + Q_ASSERT(resolution.isValid()); + GValue width{}; + g_value_init(&width, G_TYPE_INT); + g_value_set_int(&width, resolution.width()); + GValue height{}; + g_value_init(&height, G_TYPE_INT); + g_value_set_int(&height, resolution.height()); + + gst_caps_set_value(caps(), "width", &width); + gst_caps_set_value(caps(), "height", &height); +} + +QGstCaps QGstCaps::fromCameraFormat(const QCameraFormat &format) +{ + QSize size = format.resolution(); + GstStructure *structure = nullptr; + if (format.pixelFormat() == QVideoFrameFormat::Format_Jpeg) { + structure = gst_structure_new("image/jpeg", "width", G_TYPE_INT, size.width(), "height", + G_TYPE_INT, size.height(), nullptr); + } else { + int index = indexOfVideoFormat(format.pixelFormat()); + if (index < 0) + return {}; + auto gstFormat = qt_videoFormatLookup[index].gstFormat; + structure = gst_structure_new("video/x-raw", "format", G_TYPE_STRING, + gst_video_format_to_string(gstFormat), "width", G_TYPE_INT, + size.width(), "height", G_TYPE_INT, size.height(), nullptr); + } + auto caps = QGstCaps::create(); + gst_caps_append_structure(caps.get(), structure); + return caps; +} + +QGstCaps QGstCaps::copy() const +{ + return QGstCaps{ + gst_caps_copy(caps()), + QGstCaps::HasRef, + }; +} + +QGstCaps::MemoryFormat QGstCaps::memoryFormat() const +{ + auto *features = gst_caps_get_features(get(), 0); + if (gst_caps_features_contains(features, "memory:GLMemory")) + return GLTexture; + if (gst_caps_features_contains(features, "memory:DMABuf")) + return DMABuf; + return CpuMemory; +} + +int QGstCaps::size() const +{ + return int(gst_caps_get_size(get())); +} + +QGstStructureView QGstCaps::at(int index) const +{ + return QGstStructureView{ + gst_caps_get_structure(get(), index), + }; +} + +GstCaps *QGstCaps::caps() const +{ + return get(); +} + +QGstCaps QGstCaps::create() +{ + return QGstCaps(gst_caps_new_empty(), HasRef); +} + +// QGstObject + +void QGstObject::set(const char *property, const char *str) +{ + g_object_set(get(), property, str, nullptr); +} + +void QGstObject::set(const char *property, bool b) +{ + g_object_set(get(), property, gboolean(b), nullptr); +} + +void QGstObject::set(const char *property, uint i) +{ + g_object_set(get(), property, guint(i), nullptr); +} + +void QGstObject::set(const char *property, int i) +{ + g_object_set(get(), property, gint(i), nullptr); +} + +void QGstObject::set(const char *property, qint64 i) +{ + g_object_set(get(), property, gint64(i), nullptr); +} + +void QGstObject::set(const char *property, quint64 i) +{ + g_object_set(get(), property, guint64(i), nullptr); +} + +void QGstObject::set(const char *property, double d) +{ + g_object_set(get(), property, gdouble(d), nullptr); +} + +void QGstObject::set(const char *property, const QGstObject &o) +{ + g_object_set(get(), property, o.object(), nullptr); +} + +void QGstObject::set(const char *property, const QGstCaps &c) +{ + g_object_set(get(), property, c.caps(), nullptr); +} + +QGString QGstObject::getString(const char *property) const +{ + char *s = nullptr; + g_object_get(get(), property, &s, nullptr); + return QGString(s); +} + +QGstStructureView QGstObject::getStructure(const char *property) const +{ + GstStructure *s = nullptr; + g_object_get(get(), property, &s, nullptr); + return QGstStructureView(s); +} + +bool QGstObject::getBool(const char *property) const +{ + gboolean b = false; + g_object_get(get(), property, &b, nullptr); + return b; +} + +uint QGstObject::getUInt(const char *property) const +{ + guint i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +int QGstObject::getInt(const char *property) const +{ + gint i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +quint64 QGstObject::getUInt64(const char *property) const +{ + guint64 i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +qint64 QGstObject::getInt64(const char *property) const +{ + gint64 i = 0; + g_object_get(get(), property, &i, nullptr); + return i; +} + +float QGstObject::getFloat(const char *property) const +{ + gfloat d = 0; + g_object_get(get(), property, &d, nullptr); + return d; +} + +double QGstObject::getDouble(const char *property) const +{ + gdouble d = 0; + g_object_get(get(), property, &d, nullptr); + return d; +} + +QGstObject QGstObject::getObject(const char *property) const +{ + GstObject *o = nullptr; + g_object_get(get(), property, &o, nullptr); + return QGstObject(o, HasRef); +} + +QGObjectHandlerConnection QGstObject::connect(const char *name, GCallback callback, + gpointer userData) +{ + return QGObjectHandlerConnection{ + *this, + g_signal_connect(get(), name, callback, userData), + }; +} + +void QGstObject::disconnect(gulong handlerId) +{ + g_signal_handler_disconnect(get(), handlerId); +} + +GType QGstObject::type() const +{ + return G_OBJECT_TYPE(get()); +} + +QLatin1StringView QGstObject::typeName() const +{ + return QLatin1StringView{ + g_type_name(type()), + }; +} + +GstObject *QGstObject::object() const +{ + return get(); +} + +QLatin1StringView QGstObject::name() const +{ + using namespace Qt::StringLiterals; + + return get() ? QLatin1StringView{ GST_OBJECT_NAME(get()) } : "(null)"_L1; +} + +// QGObjectHandlerConnection + +QGObjectHandlerConnection::QGObjectHandlerConnection(QGstObject object, gulong handlerId) + : object{ std::move(object) }, handlerId{ handlerId } +{ +} + +void QGObjectHandlerConnection::disconnect() +{ + if (!object) + return; + + object.disconnect(handlerId); + object = {}; + handlerId = invalidHandlerId; +} + +// QGObjectHandlerScopedConnection + +QGObjectHandlerScopedConnection::QGObjectHandlerScopedConnection( + QGObjectHandlerConnection connection) + : connection{ + std::move(connection), + } +{ +} + +QGObjectHandlerScopedConnection::~QGObjectHandlerScopedConnection() +{ + connection.disconnect(); +} + +void QGObjectHandlerScopedConnection::disconnect() +{ + connection.disconnect(); +} + +// QGstPad + +QGstPad::QGstPad(const QGstObject &o) + : QGstPad{ + qGstSafeCast<GstPad>(o.object()), + QGstElement::NeedsRef, + } +{ +} + +QGstPad::QGstPad(GstPad *pad, RefMode mode) + : QGstObject{ + qGstCheckedCast<GstObject>(pad), + mode, + } +{ +} + +QGstCaps QGstPad::currentCaps() const +{ + return QGstCaps(gst_pad_get_current_caps(pad()), QGstCaps::HasRef); +} + +QGstCaps QGstPad::queryCaps() const +{ + return QGstCaps(gst_pad_query_caps(pad(), nullptr), QGstCaps::HasRef); +} + +QGstTagListHandle QGstPad::tags() const +{ + QGstTagListHandle tagList; + g_object_get(object(), "tags", &tagList, nullptr); + return tagList; +} + +std::optional<QPlatformMediaPlayer::TrackType> QGstPad::inferTrackTypeFromName() const +{ + using namespace Qt::Literals; + QLatin1StringView padName = name(); + + if (padName.startsWith("video_"_L1)) + return QPlatformMediaPlayer::TrackType::VideoStream; + if (padName.startsWith("audio_"_L1)) + return QPlatformMediaPlayer::TrackType::AudioStream; + if (padName.startsWith("text_"_L1)) + return QPlatformMediaPlayer::TrackType::SubtitleStream; + + return std::nullopt; +} + +bool QGstPad::isLinked() const +{ + return gst_pad_is_linked(pad()); +} + +bool QGstPad::link(const QGstPad &sink) const +{ + return gst_pad_link(pad(), sink.pad()) == GST_PAD_LINK_OK; +} + +bool QGstPad::unlink(const QGstPad &sink) const +{ + return gst_pad_unlink(pad(), sink.pad()); +} + +bool QGstPad::unlinkPeer() const +{ + return unlink(peer()); +} + +QGstPad QGstPad::peer() const +{ + return QGstPad(gst_pad_get_peer(pad()), HasRef); +} + +QGstElement QGstPad::parent() const +{ + return QGstElement(gst_pad_get_parent_element(pad()), HasRef); +} + +GstPad *QGstPad::pad() const +{ + return qGstCheckedCast<GstPad>(object()); +} + +GstEvent *QGstPad::stickyEvent(GstEventType type) +{ + return gst_pad_get_sticky_event(pad(), type, 0); +} + +bool QGstPad::sendEvent(GstEvent *event) +{ + return gst_pad_send_event(pad(), event); +} + +// QGstClock + +QGstClock::QGstClock(const QGstObject &o) + : QGstClock{ + qGstSafeCast<GstClock>(o.object()), + QGstElement::NeedsRef, + } +{ +} + +QGstClock::QGstClock(GstClock *clock, RefMode mode) + : QGstObject{ + qGstCheckedCast<GstObject>(clock), + mode, + } +{ +} + +GstClock *QGstClock::clock() const +{ + return qGstCheckedCast<GstClock>(object()); +} + +GstClockTime QGstClock::time() const +{ + return gst_clock_get_time(clock()); +} + +// QGstElement + +QGstElement::QGstElement(GstElement *element, RefMode mode) + : QGstObject{ + qGstCheckedCast<GstObject>(element), + mode, + } +{ +} + +QGstElement QGstElement::createFromFactory(const char *factory, const char *name) +{ + GstElement *element = gst_element_factory_make(factory, name); + +#ifndef QT_NO_DEBUG + if (!element) { + qWarning() << "Failed to make element" << name << "from factory" << factory; + return QGstElement{}; + } +#endif + + return QGstElement{ + element, + NeedsRef, + }; +} + +QGstElement QGstElement::createFromFactory(GstElementFactory *factory, const char *name) +{ + return QGstElement{ + gst_element_factory_create(factory, name), + NeedsRef, + }; +} + +QGstElement QGstElement::createFromFactory(const QGstElementFactoryHandle &factory, + const char *name) +{ + return createFromFactory(factory.get(), name); +} + +QGstElement QGstElement::createFromDevice(const QGstDeviceHandle &device, const char *name) +{ + return createFromDevice(device.get(), name); +} + +QGstElement QGstElement::createFromDevice(GstDevice *device, const char *name) +{ + return QGstElement{ + gst_device_create_element(device, name), + QGstElement::NeedsRef, + }; +} + +QGstElement QGstElement::createFromPipelineDescription(const char *str) +{ + QUniqueGErrorHandle error; + QGstElement element{ + gst_parse_launch(str, &error), + QGstElement::NeedsRef, + }; + + if (error) // error does not mean that the element could not be constructed + qWarning() << "gst_parse_launch error:" << error; + + return element; +} + +QGstElement QGstElement::createFromPipelineDescription(const QByteArray &str) +{ + return createFromPipelineDescription(str.constData()); +} + +QGstElementFactoryHandle QGstElement::findFactory(const char *name) +{ + return QGstElementFactoryHandle{ + gst_element_factory_find(name), + QGstElementFactoryHandle::HasRef, + }; +} + +QGstElementFactoryHandle QGstElement::findFactory(const QByteArray &name) +{ + return findFactory(name.constData()); +} + +QGstPad QGstElement::staticPad(const char *name) const +{ + return QGstPad(gst_element_get_static_pad(element(), name), HasRef); +} + +QGstPad QGstElement::src() const +{ + return staticPad("src"); +} + +QGstPad QGstElement::sink() const +{ + return staticPad("sink"); +} + +QGstPad QGstElement::getRequestPad(const char *name) const +{ +#if GST_CHECK_VERSION(1, 19, 1) + return QGstPad(gst_element_request_pad_simple(element(), name), HasRef); +#else + return QGstPad(gst_element_get_request_pad(element(), name), HasRef); +#endif +} + +void QGstElement::releaseRequestPad(const QGstPad &pad) const +{ + return gst_element_release_request_pad(element(), pad.pad()); +} + +GstState QGstElement::state(std::chrono::nanoseconds timeout) const +{ + using namespace std::chrono_literals; + + GstState state; + GstStateChangeReturn change = + gst_element_get_state(element(), &state, nullptr, timeout.count()); + + if (Q_UNLIKELY(change == GST_STATE_CHANGE_ASYNC)) + qWarning() << "QGstElement::state detected an asynchronous state change. Return value not " + "reliable"; + + return state; +} + +GstStateChangeReturn QGstElement::setState(GstState state) +{ + return gst_element_set_state(element(), state); +} + +bool QGstElement::setStateSync(GstState state, std::chrono::nanoseconds timeout) +{ + if (state == GST_STATE_NULL) { + // QTBUG-125251: when changing pipeline state too quickly between NULL->PAUSED->NULL there + // may be a pending task to activate pads while we try to switch to NULL. This can cause an + // assertion failure in gstreamer. we therefore finish the state change when called on a bin + // or pipeline. + if (qIsGstObjectOfType<GstBin>(element())) + finishStateChange(); + } + + GstStateChangeReturn change = gst_element_set_state(element(), state); + if (change == GST_STATE_CHANGE_ASYNC) + change = gst_element_get_state(element(), nullptr, &state, timeout.count()); + + if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) { + qWarning() << "Could not change state of" << name() << "to" << state << change; + dumpPipelineGraph("setStateSyncFailure"); + } + return change == GST_STATE_CHANGE_SUCCESS; +} + +bool QGstElement::syncStateWithParent() +{ + Q_ASSERT(element()); + return gst_element_sync_state_with_parent(element()) == TRUE; +} + +bool QGstElement::finishStateChange(std::chrono::nanoseconds timeout) +{ + GstState state, pending; + GstStateChangeReturn change = + gst_element_get_state(element(), &state, &pending, timeout.count()); + + if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) { + qWarning() << "Could not finish change state of" << name() << change << state << pending; + dumpPipelineGraph("finishStateChangeFailure"); + } + return change == GST_STATE_CHANGE_SUCCESS; +} + +void QGstElement::lockState(bool locked) +{ + gst_element_set_locked_state(element(), locked); +} + +bool QGstElement::isStateLocked() const +{ + return gst_element_is_locked_state(element()); +} + +void QGstElement::sendEvent(GstEvent *event) const +{ + gst_element_send_event(element(), event); +} + +void QGstElement::sendEos() const +{ + sendEvent(gst_event_new_eos()); +} + +std::optional<std::chrono::nanoseconds> QGstElement::duration() const +{ + gint64 d; + if (!gst_element_query_duration(element(), GST_FORMAT_TIME, &d)) { + qDebug() << "QGstElement: failed to query duration"; + return std::nullopt; + } + return std::chrono::nanoseconds{ d }; +} + +std::optional<std::chrono::milliseconds> QGstElement::durationInMs() const +{ + using namespace std::chrono; + auto dur = duration(); + if (dur) + return round<milliseconds>(*dur); + return std::nullopt; +} + +std::optional<std::chrono::nanoseconds> QGstElement::position() const +{ + QGstQueryHandle &query = positionQuery(); + + gint64 pos; + if (gst_element_query(element(), query.get())) { + gst_query_parse_position(query.get(), nullptr, &pos); + return std::chrono::nanoseconds{ pos }; + } + + qDebug() << "QGstElement: failed to query position"; + return std::nullopt; +} + +std::optional<std::chrono::milliseconds> QGstElement::positionInMs() const +{ + using namespace std::chrono; + auto pos = position(); + if (pos) + return round<milliseconds>(*pos); + return std::nullopt; +} + +std::optional<bool> QGstElement::canSeek() const +{ + QGstQueryHandle query{ + gst_query_new_seeking(GST_FORMAT_TIME), + QGstQueryHandle::HasRef, + }; + gboolean canSeek = false; + gst_query_parse_seeking(query.get(), nullptr, &canSeek, nullptr, nullptr); + + if (gst_element_query(element(), query.get())) { + gst_query_parse_seeking(query.get(), nullptr, &canSeek, nullptr, nullptr); + return canSeek; + } + return std::nullopt; +} + +GstClockTime QGstElement::baseTime() const +{ + return gst_element_get_base_time(element()); +} + +void QGstElement::setBaseTime(GstClockTime time) const +{ + gst_element_set_base_time(element(), time); +} + +GstElement *QGstElement::element() const +{ + return GST_ELEMENT_CAST(get()); +} + +QGstElement QGstElement::getParent() const +{ + return QGstElement{ + qGstCheckedCast<GstElement>(gst_element_get_parent(object())), + QGstElement::HasRef, + }; +} + +QGstPipeline QGstElement::getPipeline() const +{ + QGstElement ancestor = *this; + for (;;) { + QGstElement greatAncestor = ancestor.getParent(); + if (greatAncestor) { + ancestor = std::move(greatAncestor); + continue; + } + + return QGstPipeline{ + qGstSafeCast<GstPipeline>(ancestor.element()), + QGstPipeline::NeedsRef, + }; + } +} + +void QGstElement::dumpPipelineGraph(const char *filename) const +{ + static const bool dumpEnabled = qEnvironmentVariableIsSet("GST_DEBUG_DUMP_DOT_DIR"); + if (dumpEnabled) { + QGstPipeline pipeline = getPipeline(); + if (pipeline) + pipeline.dumpGraph(filename); + } +} + +QGstQueryHandle &QGstElement::positionQuery() const +{ + if (Q_UNLIKELY(!m_positionQuery)) + m_positionQuery = QGstQueryHandle{ + gst_query_new_position(GST_FORMAT_TIME), + QGstQueryHandle::HasRef, + }; + + return m_positionQuery; +} + +// QGstBin + +QGstBin QGstBin::create(const char *name) +{ + return QGstBin(gst_bin_new(name), NeedsRef); +} + +QGstBin QGstBin::createFromFactory(const char *factory, const char *name) +{ + QGstElement element = QGstElement::createFromFactory(factory, name); + Q_ASSERT(GST_IS_BIN(element.element())); + return QGstBin{ + GST_BIN(element.release()), + RefMode::HasRef, + }; +} + +QGstBin QGstBin::createFromPipelineDescription(const QByteArray &pipelineDescription, + const char *name, bool ghostUnlinkedPads) +{ + return createFromPipelineDescription(pipelineDescription.constData(), name, ghostUnlinkedPads); +} + +QGstBin QGstBin::createFromPipelineDescription(const char *pipelineDescription, const char *name, + bool ghostUnlinkedPads) +{ + QUniqueGErrorHandle error; + + GstElement *element = + gst_parse_bin_from_description_full(pipelineDescription, ghostUnlinkedPads, + /*context=*/nullptr, GST_PARSE_FLAG_NONE, &error); + + if (!element) { + qWarning() << "Failed to make element from pipeline description" << pipelineDescription + << error; + return QGstBin{}; + } + + if (name) + gst_element_set_name(element, name); + + return QGstBin{ + element, + NeedsRef, + }; +} + +QGstBin::QGstBin(GstBin *bin, RefMode mode) + : QGstElement{ + qGstCheckedCast<GstElement>(bin), + mode, + } +{ +} + +GstBin *QGstBin::bin() const +{ + return qGstCheckedCast<GstBin>(object()); +} + +void QGstBin::addGhostPad(const QGstElement &child, const char *name) +{ + addGhostPad(name, child.staticPad(name)); +} + +void QGstBin::addGhostPad(const char *name, const QGstPad &pad) +{ + gst_element_add_pad(element(), gst_ghost_pad_new(name, pad.pad())); +} + +bool QGstBin::syncChildrenState() +{ + return gst_bin_sync_children_states(bin()); +} + +void QGstBin::dumpGraph(const char *fileNamePrefix) const +{ + if (isNull()) + return; + + GST_DEBUG_BIN_TO_DOT_FILE(bin(), GST_DEBUG_GRAPH_SHOW_VERBOSE, fileNamePrefix); +} + +QGstElement QGstBin::findByName(const char *name) +{ + return QGstElement{ + gst_bin_get_by_name(bin(), name), + QGstElement::NeedsRef, + }; +} + +// QGstBaseSink + +QGstBaseSink::QGstBaseSink(GstBaseSink *element, RefMode mode) + : QGstElement{ + qGstCheckedCast<GstElement>(element), + mode, + } +{ +} + +void QGstBaseSink::setSync(bool arg) +{ + gst_base_sink_set_sync(baseSink(), arg ? TRUE : FALSE); +} + +GstBaseSink *QGstBaseSink::baseSink() const +{ + return qGstCheckedCast<GstBaseSink>(element()); +} + +// QGstBaseSrc + +QGstBaseSrc::QGstBaseSrc(GstBaseSrc *element, RefMode mode) + : QGstElement{ + qGstCheckedCast<GstElement>(element), + mode, + } +{ +} + +GstBaseSrc *QGstBaseSrc::baseSrc() const +{ + return qGstCheckedCast<GstBaseSrc>(element()); +} + +// QGstAppSink + +QGstAppSink::QGstAppSink(GstAppSink *element, RefMode mode) + : QGstBaseSink{ + qGstCheckedCast<GstBaseSink>(element), + mode, + } +{ +} + +QGstAppSink QGstAppSink::create(const char *name) +{ + QGstElement created = QGstElement::createFromFactory("appsink", name); + return QGstAppSink{ + qGstCheckedCast<GstAppSink>(created.element()), + QGstAppSink::NeedsRef, + }; +} + +GstAppSink *QGstAppSink::appSink() const +{ + return qGstCheckedCast<GstAppSink>(element()); +} + +# if GST_CHECK_VERSION(1, 24, 0) +void QGstAppSink::setMaxBufferTime(std::chrono::nanoseconds ns) +{ + gst_app_sink_set_max_time(appSink(), qGstClockTimeFromChrono(ns)); +} +# endif + +void QGstAppSink::setMaxBuffers(int n) +{ + gst_app_sink_set_max_buffers(appSink(), n); +} + +void QGstAppSink::setCaps(const QGstCaps &caps) +{ + gst_app_sink_set_caps(appSink(), caps.caps()); +} + +void QGstAppSink::setCallbacks(GstAppSinkCallbacks &callbacks, gpointer user_data, + GDestroyNotify notify) +{ + gst_app_sink_set_callbacks(appSink(), &callbacks, user_data, notify); +} + +QGstSampleHandle QGstAppSink::pullSample() +{ + return QGstSampleHandle{ + gst_app_sink_pull_sample(appSink()), + QGstSampleHandle::HasRef, + }; +} + +// QGstAppSrc + +QGstAppSrc::QGstAppSrc(GstAppSrc *element, RefMode mode) + : QGstBaseSrc{ + qGstCheckedCast<GstBaseSrc>(element), + mode, + } +{ +} + +QGstAppSrc QGstAppSrc::create(const char *name) +{ + QGstElement created = QGstElement::createFromFactory("appsrc", name); + return QGstAppSrc{ + qGstCheckedCast<GstAppSrc>(created.element()), + QGstAppSrc::NeedsRef, + }; +} + +GstAppSrc *QGstAppSrc::appSrc() const +{ + return qGstCheckedCast<GstAppSrc>(element()); +} + +void QGstAppSrc::setCallbacks(GstAppSrcCallbacks &callbacks, gpointer user_data, + GDestroyNotify notify) +{ + gst_app_src_set_callbacks(appSrc(), &callbacks, user_data, notify); +} + +GstFlowReturn QGstAppSrc::pushBuffer(GstBuffer *buffer) +{ + return gst_app_src_push_buffer(appSrc(), buffer); +} + +QString qGstErrorMessageCannotFindElement(std::string_view element) +{ + return QStringLiteral("Could not find the %1 GStreamer element") + .arg(QLatin1StringView(element)); +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp b/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp new file mode 100644 index 000000000..e47515d2d --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst_debug.cpp @@ -0,0 +1,573 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qgst_debug_p.h" +#include "qgstreamermessage_p.h" + +#include <gst/gstclock.h> + +QT_BEGIN_NAMESPACE + +// NOLINTBEGIN(performance-unnecessary-value-param) + +QDebug operator<<(QDebug dbg, const QGString &str) +{ + return dbg << str.get(); +} + +QDebug operator<<(QDebug dbg, const QGstCaps &caps) +{ + return dbg << caps.caps(); +} + +QDebug operator<<(QDebug dbg, const QGstStructureView &structure) +{ + return dbg << structure.structure; +} + +QDebug operator<<(QDebug dbg, const QGValue &value) +{ + return dbg << value.value; +} + +QDebug operator<<(QDebug dbg, const QGstreamerMessage &msg) +{ + return dbg << msg.message(); +} + +QDebug operator<<(QDebug dbg, const QUniqueGErrorHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QUniqueGStringHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstStreamCollectionHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstStreamHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstTagListHandle &handle) +{ + return dbg << handle.get(); +} + +QDebug operator<<(QDebug dbg, const QGstElement &element) +{ + return dbg << element.element(); +} + +QDebug operator<<(QDebug dbg, const QGstPad &pad) +{ + return dbg << pad.pad(); +} + +QDebug operator<<(QDebug dbg, const GstCaps *caps) +{ + if (caps) + return dbg << QGString(gst_caps_to_string(caps)); + else + return dbg << "null"; +} + +QDebug operator<<(QDebug dbg, const GstVideoInfo *info) +{ +#if GST_CHECK_VERSION(1, 20, 0) + return dbg << QGstCaps{ + gst_video_info_to_caps(info), + QGstCaps::NeedsRef, + }; +#else + return dbg << QGstCaps{ + gst_video_info_to_caps(const_cast<GstVideoInfo *>(info)), + QGstCaps::NeedsRef, + }; +#endif +} + +QDebug operator<<(QDebug dbg, const GstStructure *structure) +{ + if (structure) + return dbg << QGString(gst_structure_to_string(structure)); + else + return dbg << "null"; +} + +QDebug operator<<(QDebug dbg, const GstObject *object) +{ + dbg << QGString{gst_object_get_name(const_cast<GstObject*>(object))}; + + { + QDebugStateSaver saver(dbg); + dbg.nospace(); + + dbg << "{"; + + guint numProperties; + GParamSpec **properties = g_object_class_list_properties(G_OBJECT_GET_CLASS(object), &numProperties); + + for (guint i = 0; i < numProperties; i++) { + GParamSpec *param = properties[i]; + + const gchar *name = g_param_spec_get_name(param); + constexpr bool trace_blurb = false; + if constexpr (trace_blurb) { + const gchar *blurb = g_param_spec_get_blurb(param); + dbg << name << " (" << blurb << "): "; + } else + dbg << name << ": "; + + bool readable = bool(param->flags & G_PARAM_READABLE); + if (!readable) { + dbg << "(not readable)"; + } else if (QLatin1StringView(name) == QLatin1StringView("parent")) { + if (object->parent) + dbg << QGString{ gst_object_get_name(object->parent) }; + else + dbg << "(none)"; + } else { + GValue value = {}; + g_object_get_property(&const_cast<GstObject *>(object)->object, param->name, + &value); + dbg << &value; + } + if (i != numProperties - 1) + dbg << ", "; + } + + dbg << "}"; + + g_free(properties); + } + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstElement *element) +{ + return dbg << GST_OBJECT_CAST(element); // LATER: output other members? +} + +QDebug operator<<(QDebug dbg, const GstPad *pad) +{ + return dbg << GST_OBJECT_CAST(pad); // LATER: output other members? +} + +QDebug operator<<(QDebug dbg, const GstDevice *device) +{ + GstDevice *d = const_cast<GstDevice *>(device); + QDebugStateSaver saver(dbg); + dbg.nospace(); + + dbg << gst_device_get_display_name(d) << "(" << gst_device_get_device_class(d) << ") "; + dbg << "Caps: " << QGstCaps{ gst_device_get_caps(d), QGstCaps::NeedsRef, } << ", "; + dbg << "Properties: " << QUniqueGstStructureHandle{ gst_device_get_properties(d) }.get(); + return dbg; +} + +namespace { + +struct Timepoint +{ + explicit Timepoint(guint64 us) : ts{ us } { } + guint64 ts; +}; + +QDebug operator<<(QDebug dbg, Timepoint ts) +{ + char buffer[128]; + snprintf(buffer, sizeof(buffer), "%" GST_TIME_FORMAT, GST_TIME_ARGS(ts.ts)); + dbg << buffer; + return dbg; +} + +} // namespace + +QDebug operator<<(QDebug dbg, const GstMessage *msg) +{ + QDebugStateSaver saver(dbg); + dbg.nospace(); + + dbg << GST_MESSAGE_TYPE_NAME(msg) << ", Source: " << GST_MESSAGE_SRC_NAME(msg); + if (GST_MESSAGE_TIMESTAMP(msg) != 0xFFFFFFFFFFFFFFFF) + dbg << ", Timestamp: " << GST_MESSAGE_TIMESTAMP(msg); + + switch (msg->type) { + case GST_MESSAGE_ERROR: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Error: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_WARNING: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_warning(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Warning: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_INFO: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_info(const_cast<GstMessage *>(msg), &err, &debug); + + dbg << ", Info: " << err << " (" << debug << ")"; + break; + } + + case GST_MESSAGE_TAG: { + QGstTagListHandle tagList; + gst_message_parse_tag(const_cast<GstMessage *>(msg), &tagList); + + dbg << ", Tags: " << tagList; + break; + } + + case GST_MESSAGE_QOS: { + gboolean live; + guint64 running_time; + guint64 stream_time; + guint64 timestamp; + guint64 duration; + + gst_message_parse_qos(const_cast<GstMessage *>(msg), &live, &running_time, &stream_time, + ×tamp, &duration); + + dbg << ", Live: " << bool(live) << ", Running time: " << Timepoint{ running_time } + << ", Stream time: " << Timepoint{ stream_time } + << ", Timestamp: " << Timepoint{ timestamp } << ", Duration: " << Timepoint{ duration }; + break; + } + + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(const_cast<GstMessage *>(msg), &oldState, &newState, + &pending); + + dbg << ", Transition: " << oldState << "->" << newState; + + if (pending != GST_STATE_VOID_PENDING) + dbg << ", Pending State: " << pending; + break; + } + + case GST_MESSAGE_STREAM_COLLECTION: { + QGstStreamCollectionHandle collection; + gst_message_parse_stream_collection(const_cast<GstMessage *>(msg), &collection); + + dbg << ", " << collection; + break; + } + + case GST_MESSAGE_STREAMS_SELECTED: { + QGstStreamCollectionHandle collection; + gst_message_parse_streams_selected(const_cast<GstMessage *>(msg), &collection); + + dbg << ", " << collection; + break; + } + + case GST_MESSAGE_STREAM_STATUS: { + GstStreamStatusType streamStatus; + gst_message_parse_stream_status(const_cast<GstMessage *>(msg), &streamStatus, nullptr); + + dbg << ", Stream Status: " << streamStatus; + break; + } + + case GST_MESSAGE_BUFFERING: { + int progress = 0; + gst_message_parse_buffering(const_cast<GstMessage *>(msg), &progress); + + dbg << ", Buffering: " << progress << "%"; + break; + } + + default: + break; + } + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstTagList *tagList) +{ + dbg << QGString{ gst_tag_list_to_string(tagList) }; + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstQuery *query) +{ + dbg << GST_QUERY_TYPE_NAME(query); + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstEvent *event) +{ + dbg << GST_EVENT_TYPE_NAME(event); + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstPadTemplate *padTemplate) +{ + QGstCaps caps = padTemplate + ? QGstCaps{ gst_pad_template_get_caps(const_cast<GstPadTemplate *>(padTemplate)), QGstCaps::HasRef, } + : QGstCaps{}; + + dbg << caps; + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstStreamCollection *streamCollection) +{ + GstStreamCollection *collection = const_cast<GstStreamCollection *>(streamCollection); + guint size = gst_stream_collection_get_size(collection); + + dbg << "Stream Collection: {"; + for (guint index = 0; index != size; ++index) { + dbg << gst_stream_collection_get_stream(collection, index); + if (index + 1 != size) + dbg << ", "; + } + + dbg << "}"; + return dbg; +} + +QDebug operator<<(QDebug dbg, const GstStream *cstream) +{ + GstStream *stream = const_cast<GstStream *>(cstream); + + dbg << "GstStream { "; + dbg << "Type: " << gst_stream_type_get_name(gst_stream_get_stream_type(stream)); + + QGstTagListHandle tagList{ + gst_stream_get_tags(stream), + QGstTagListHandle::HasRef, + }; + + if (tagList) + dbg << ", Tags: " << tagList; + + QGstCaps caps{ + gst_stream_get_caps(stream), + QGstCaps::HasRef, + }; + + if (caps) + dbg << ", Caps: " << caps; + + dbg << "}"; + + return dbg; +} + +QDebug operator<<(QDebug dbg, GstState state) +{ + return dbg << gst_element_state_get_name(state); +} + +QDebug operator<<(QDebug dbg, GstStateChange transition) +{ + return dbg << gst_state_change_get_name(transition); +} + +QDebug operator<<(QDebug dbg, GstStateChangeReturn stateChangeReturn) +{ + return dbg << gst_element_state_change_return_get_name(stateChangeReturn); +} + +QDebug operator<<(QDebug dbg, GstMessageType type) +{ + return dbg << gst_message_type_get_name(type); +} + +#define ADD_ENUM_SWITCH(value) \ + case value: \ + return dbg << #value; \ + static_assert(true, "enforce semicolon") + +QDebug operator<<(QDebug dbg, GstPadDirection direction) +{ + switch (direction) { + ADD_ENUM_SWITCH(GST_PAD_UNKNOWN); + ADD_ENUM_SWITCH(GST_PAD_SRC); + ADD_ENUM_SWITCH(GST_PAD_SINK); + default: + Q_UNREACHABLE_RETURN(dbg); + } +} + +QDebug operator<<(QDebug dbg, GstStreamStatusType type) +{ + switch (type) { + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_CREATE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_ENTER); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_LEAVE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_DESTROY); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_START); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_PAUSE); + ADD_ENUM_SWITCH(GST_STREAM_STATUS_TYPE_STOP); + default: + Q_UNREACHABLE_RETURN(dbg); + } + return dbg; +} + +#undef ADD_ENUM_SWITCH + +QDebug operator<<(QDebug dbg, const GValue *value) +{ + switch (G_VALUE_TYPE(value)) { + case G_TYPE_STRING: + return dbg << g_value_get_string(value); + case G_TYPE_BOOLEAN: + return dbg << g_value_get_boolean(value); + case G_TYPE_ULONG: + return dbg << g_value_get_ulong(value); + case G_TYPE_LONG: + return dbg << g_value_get_long(value); + case G_TYPE_UINT: + return dbg << g_value_get_uint(value); + case G_TYPE_INT: + return dbg << g_value_get_int(value); + case G_TYPE_UINT64: + return dbg << g_value_get_uint64(value); + case G_TYPE_INT64: + return dbg << g_value_get_int64(value); + case G_TYPE_FLOAT: + return dbg << g_value_get_float(value); + case G_TYPE_DOUBLE: + return dbg << g_value_get_double(value); + default: + break; + } + + if (GST_VALUE_HOLDS_BITMASK(value)) { + QDebugStateSaver saver(dbg); + return dbg << Qt::hex << gst_value_get_bitmask(value); + } + + if (GST_VALUE_HOLDS_FRACTION(value)) + return dbg << gst_value_get_fraction_numerator(value) << "/" + << gst_value_get_fraction_denominator(value); + + if (GST_VALUE_HOLDS_CAPS(value)) + return dbg << gst_value_get_caps(value); + + if (GST_VALUE_HOLDS_STRUCTURE(value)) + return dbg << gst_value_get_structure(value); + + if (GST_VALUE_HOLDS_ARRAY(value)) { + const guint size = gst_value_array_get_size(value); + const guint last = size - 1; + dbg << "["; + for (guint index = 0; index != size; ++index) { + dbg << gst_value_array_get_value(value, index); + if (index != last) + dbg << ", "; + } + dbg << "}"; + return dbg; + } + + if (G_VALUE_TYPE(value) == GST_TYPE_PAD_DIRECTION) { + GstPadDirection direction = static_cast<GstPadDirection>(g_value_get_enum(value)); + return dbg << direction; + } + + if (G_VALUE_TYPE(value) == GST_TYPE_PAD_TEMPLATE) { + GstPadTemplate *padTemplate = static_cast<GstPadTemplate *>(g_value_get_object(value)); + return dbg << padTemplate; + } + + dbg << "(not implemented: " << G_VALUE_TYPE_NAME(value) << ")"; + + return dbg; +} + +QDebug operator<<(QDebug dbg, const GError *error) +{ + return dbg << error->message; +} + +QCompactGstMessageAdaptor::QCompactGstMessageAdaptor(const QGstreamerMessage &m) + : QCompactGstMessageAdaptor{ + m.message(), + } +{ +} + +QCompactGstMessageAdaptor::QCompactGstMessageAdaptor(GstMessage *m) + : msg{ + m, + } +{ +} + +QDebug operator<<(QDebug dbg, const QCompactGstMessageAdaptor &m) +{ + std::optional<QDebugStateSaver> saver(dbg); + dbg.nospace(); + + switch (GST_MESSAGE_TYPE(m.msg)) { + case GST_MESSAGE_ERROR: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(m.msg, &err, &debug); + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_WARNING: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_warning(m.msg, &err, &debug); + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_INFO: { + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_info(m.msg, &err, &debug); + + dbg << err << " (" << debug << ")"; + return dbg; + } + + case GST_MESSAGE_STATE_CHANGED: { + GstState oldState; + GstState newState; + GstState pending; + + gst_message_parse_state_changed(m.msg, &oldState, &newState, &pending); + + dbg << oldState << " -> " << newState; + if (pending != GST_STATE_VOID_PENDING) + dbg << " (pending: " << pending << ")"; + return dbg; + } + + default: { + saver.reset(); + return dbg << m.msg; + } + } +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h b/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h new file mode 100644 index 000000000..df13c6c13 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst_debug_p.h @@ -0,0 +1,74 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGST_DEBUG_P_H +#define QGST_DEBUG_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qgst_p.h" +#include <qdebug.h> + +QT_BEGIN_NAMESPACE + +class QGstreamerMessage; + +QDebug operator<<(QDebug, const QGstCaps &); +QDebug operator<<(QDebug, const QGstStructureView &); +QDebug operator<<(QDebug, const QGstElement &); +QDebug operator<<(QDebug, const QGstPad &); +QDebug operator<<(QDebug, const QGString &); +QDebug operator<<(QDebug, const QGValue &); +QDebug operator<<(QDebug, const QGstreamerMessage &); +QDebug operator<<(QDebug, const QUniqueGErrorHandle &); +QDebug operator<<(QDebug, const QUniqueGStringHandle &); +QDebug operator<<(QDebug, const QGstStreamCollectionHandle &); +QDebug operator<<(QDebug, const QGstStreamHandle &); +QDebug operator<<(QDebug, const QGstTagListHandle &); + +QDebug operator<<(QDebug, const GstCaps *); +QDebug operator<<(QDebug, const GstVideoInfo *); +QDebug operator<<(QDebug, const GstStructure *); +QDebug operator<<(QDebug, const GstObject *); +QDebug operator<<(QDebug, const GstElement *); +QDebug operator<<(QDebug, const GstPad *); +QDebug operator<<(QDebug, const GstDevice *); +QDebug operator<<(QDebug, const GstMessage *); +QDebug operator<<(QDebug, const GstTagList *); +QDebug operator<<(QDebug, const GstQuery *); +QDebug operator<<(QDebug, const GstEvent *); +QDebug operator<<(QDebug, const GstPadTemplate *); +QDebug operator<<(QDebug, const GstStreamCollection *); +QDebug operator<<(QDebug, const GstStream *); + +QDebug operator<<(QDebug, GstState); +QDebug operator<<(QDebug, GstStateChange); +QDebug operator<<(QDebug, GstStateChangeReturn); +QDebug operator<<(QDebug, GstMessageType); +QDebug operator<<(QDebug, GstPadDirection); +QDebug operator<<(QDebug, GstStreamStatusType); + +QDebug operator<<(QDebug, const GValue *); +QDebug operator<<(QDebug, const GError *); + +struct QCompactGstMessageAdaptor +{ + explicit QCompactGstMessageAdaptor(const QGstreamerMessage &m); + explicit QCompactGstMessageAdaptor(GstMessage *m); + GstMessage *msg; +}; + +QDebug operator<<(QDebug, const QCompactGstMessageAdaptor &); + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h b/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h new file mode 100644 index 000000000..e813f4181 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgst_handle_types_p.h @@ -0,0 +1,270 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGST_HANDLE_TYPES_P_H +#define QGST_HANDLE_TYPES_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/private/qcore_unix_p.h> +#include <QtCore/private/quniquehandle_p.h> +#include <QtCore/qtconfigmacros.h> + +#include <QtMultimedia/private/qtmultimedia-config_p.h> + +#include <gst/gst.h> + +#if QT_CONFIG(gstreamer_gl) +# include <gst/gl/gstglcontext.h> +#endif + +QT_BEGIN_NAMESPACE + +namespace QGstImpl { + +template <typename HandleTraits> +struct QSharedHandle : private QUniqueHandle<HandleTraits> +{ + using BaseClass = QUniqueHandle<HandleTraits>; + + enum RefMode { HasRef, NeedsRef }; + + QSharedHandle() = default; + + explicit QSharedHandle(typename HandleTraits::Type object, RefMode mode) + : BaseClass{ mode == NeedsRef ? HandleTraits::ref(object) : object } + { + } + + QSharedHandle(const QSharedHandle &o) + : BaseClass{ + HandleTraits::ref(o.get()), + } + { + } + + QSharedHandle(QSharedHandle &&) noexcept = default; + + QSharedHandle &operator=(const QSharedHandle &o) // NOLINT: bugprone-unhandled-self-assign + { + if (BaseClass::get() != o.get()) + reset(HandleTraits::ref(o.get())); + return *this; + }; + + QSharedHandle &operator=(QSharedHandle &&) noexcept = default; + + [[nodiscard]] friend bool operator==(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() == rhs.get(); + } + + [[nodiscard]] friend bool operator!=(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() != rhs.get(); + } + + [[nodiscard]] friend bool operator<(const QSharedHandle &lhs, const QSharedHandle &rhs) noexcept + { + return lhs.get() < rhs.get(); + } + + [[nodiscard]] friend bool operator<=(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() <= rhs.get(); + } + + [[nodiscard]] friend bool operator>(const QSharedHandle &lhs, const QSharedHandle &rhs) noexcept + { + return lhs.get() > rhs.get(); + } + + [[nodiscard]] friend bool operator>=(const QSharedHandle &lhs, + const QSharedHandle &rhs) noexcept + { + return lhs.get() >= rhs.get(); + } + + using BaseClass::get; + using BaseClass::isValid; + using BaseClass::operator bool; + using BaseClass::release; + using BaseClass::reset; + using BaseClass::operator&; + using BaseClass::close; +}; + +struct QGstTagListHandleTraits +{ + using Type = GstTagList *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_tag_list_unref(handle); + return true; + } + static Type ref(Type handle) noexcept { return gst_tag_list_ref(handle); } +}; + +struct QGstSampleHandleTraits +{ + using Type = GstSample *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_sample_unref(handle); + return true; + } + static Type ref(Type handle) noexcept { return gst_sample_ref(handle); } +}; + +struct QUniqueGstStructureHandleTraits +{ + using Type = GstStructure *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_structure_free(handle); + return true; + } +}; + +struct QUniqueGStringHandleTraits +{ + using Type = gchar *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + g_free(handle); + return true; + } +}; + +struct QUniqueGErrorHandleTraits +{ + using Type = GError *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + g_error_free(handle); + return true; + } +}; + + +struct QUniqueGstDateTimeHandleTraits +{ + using Type = GstDateTime *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_date_time_unref(handle); + return true; + } +}; + +struct QFileDescriptorHandleTraits +{ + using Type = int; + static constexpr Type invalidValue() noexcept { return -1; } + static bool close(Type fd) noexcept + { + int closeResult = qt_safe_close(fd); + return closeResult == 0; + } +}; + +template <typename GstType> +struct QGstHandleHelper +{ + struct QGstSafeObjectHandleTraits + { + using Type = GstType *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_object_unref(G_OBJECT(handle)); + return true; + } + + static Type ref(Type handle) noexcept + { + gst_object_ref_sink(G_OBJECT(handle)); + return handle; + } + }; + + using SharedHandle = QSharedHandle<QGstSafeObjectHandleTraits>; + using UniqueHandle = QUniqueHandle<QGstSafeObjectHandleTraits>; +}; + +template <typename GstType> +struct QGstMiniObjectHandleHelper +{ + struct Traits + { + using Type = GstType *; + static constexpr Type invalidValue() noexcept { return nullptr; } + static bool close(Type handle) noexcept + { + gst_mini_object_unref(GST_MINI_OBJECT_CAST(handle)); + return true; + } + + static Type ref(Type handle) noexcept + { + if (GST_MINI_OBJECT_CAST(handle)) + gst_mini_object_ref(GST_MINI_OBJECT_CAST(handle)); + return handle; + } + }; + + using SharedHandle = QSharedHandle<Traits>; + using UniqueHandle = QUniqueHandle<Traits>; +}; + +} // namespace QGstImpl + +using QGstClockHandle = QGstImpl::QGstHandleHelper<GstClock>::UniqueHandle; +using QGstElementHandle = QGstImpl::QGstHandleHelper<GstElement>::UniqueHandle; +using QGstElementFactoryHandle = QGstImpl::QGstHandleHelper<GstElementFactory>::SharedHandle; +using QGstDeviceHandle = QGstImpl::QGstHandleHelper<GstDevice>::SharedHandle; +using QGstDeviceMonitorHandle = QGstImpl::QGstHandleHelper<GstDeviceMonitor>::UniqueHandle; +using QGstBusHandle = QGstImpl::QGstHandleHelper<GstBus>::UniqueHandle; +using QGstStreamCollectionHandle = QGstImpl::QGstHandleHelper<GstStreamCollection>::SharedHandle; +using QGstStreamHandle = QGstImpl::QGstHandleHelper<GstStream>::SharedHandle; + +using QGstTagListHandle = QGstImpl::QSharedHandle<QGstImpl::QGstTagListHandleTraits>; +using QGstSampleHandle = QGstImpl::QSharedHandle<QGstImpl::QGstSampleHandleTraits>; + +using QUniqueGstStructureHandle = QUniqueHandle<QGstImpl::QUniqueGstStructureHandleTraits>; +using QUniqueGStringHandle = QUniqueHandle<QGstImpl::QUniqueGStringHandleTraits>; +using QUniqueGErrorHandle = QUniqueHandle<QGstImpl::QUniqueGErrorHandleTraits>; +using QUniqueGstDateTimeHandle = QUniqueHandle<QGstImpl::QUniqueGstDateTimeHandleTraits>; +using QFileDescriptorHandle = QUniqueHandle<QGstImpl::QFileDescriptorHandleTraits>; +using QGstBufferHandle = QGstImpl::QGstMiniObjectHandleHelper<GstBuffer>::SharedHandle; +using QGstContextHandle = QGstImpl::QGstMiniObjectHandleHelper<GstContext>::UniqueHandle; +using QGstGstDateTimeHandle = QGstImpl::QGstMiniObjectHandleHelper<GstDateTime>::SharedHandle; +using QGstPluginFeatureHandle = QGstImpl::QGstHandleHelper<GstPluginFeature>::SharedHandle; +using QGstQueryHandle = QGstImpl::QGstMiniObjectHandleHelper<GstQuery>::SharedHandle; + +#if QT_CONFIG(gstreamer_gl) +using QGstGLContextHandle = QGstImpl::QGstHandleHelper<GstGLContext>::UniqueHandle; +using QGstGLDisplayHandle = QGstImpl::QGstHandleHelper<GstGLDisplay>::UniqueHandle; +#endif + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgst_p.h b/src/plugins/multimedia/gstreamer/common/qgst_p.h index 66b1b156f..bf5290d5d 100644 --- a/src/plugins/multimedia/gstreamer/common/qgst_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgst_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGST_P_H #define QGST_P_H @@ -51,32 +15,159 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> - -#include <QSemaphore> +#include <QtCore/qdebug.h> #include <QtCore/qlist.h> +#include <QtCore/qsemaphore.h> #include <QtMultimedia/qaudioformat.h> #include <QtMultimedia/qvideoframe.h> +#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtMultimedia/private/qmultimediautils_p.h> +#include <QtMultimedia/private/qplatformmediaplayer_p.h> #include <gst/gst.h> +#include <gst/app/gstappsink.h> +#include <gst/app/gstappsrc.h> #include <gst/video/video-info.h> -#include <functional> +#include "qgst_handle_types_p.h" + +#include <type_traits> #if QT_CONFIG(gstreamer_photography) -#define GST_USE_UNSTABLE_API -#include <gst/interfaces/photography.h> -#undef GST_USE_UNSTABLE_API -#endif -#ifndef QT_NO_DEBUG -#include <qdebug.h> +# define GST_USE_UNSTABLE_API +# include <gst/interfaces/photography.h> +# undef GST_USE_UNSTABLE_API #endif + QT_BEGIN_NAMESPACE +namespace QGstImpl { + +template <typename T> +struct GstObjectTraits +{ + // using Type = T; + // template <typename U> + // static bool isObjectOfType(U *); + // template <typename U> + // static T *cast(U *); +}; + +#define QGST_DEFINE_CAST_TRAITS(ClassName, MACRO_LABEL) \ + template <> \ + struct GstObjectTraits<ClassName> \ + { \ + using Type = ClassName; \ + template <typename U> \ + static bool isObjectOfType(U *arg) \ + { \ + return GST_IS_##MACRO_LABEL(arg); \ + } \ + template <typename U> \ + static Type *cast(U *arg) \ + { \ + return GST_##MACRO_LABEL##_CAST(arg); \ + } \ + template <typename U> \ + static Type *checked_cast(U *arg) \ + { \ + return GST_##MACRO_LABEL(arg); \ + } \ + }; \ + static_assert(true, "ensure semicolon") + +#define QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE(ClassName, MACRO_LABEL) \ + template <> \ + struct GstObjectTraits<ClassName> \ + { \ + using Type = ClassName; \ + template <typename U> \ + static bool isObjectOfType(U *arg) \ + { \ + return GST_IS_##MACRO_LABEL(arg); \ + } \ + template <typename U> \ + static Type *cast(U *arg) \ + { \ + return checked_cast(arg); \ + } \ + template <typename U> \ + static Type *checked_cast(U *arg) \ + { \ + return GST_##MACRO_LABEL(arg); \ + } \ + }; \ + static_assert(true, "ensure semicolon") + +QGST_DEFINE_CAST_TRAITS(GstBin, BIN); +QGST_DEFINE_CAST_TRAITS(GstClock, CLOCK); +QGST_DEFINE_CAST_TRAITS(GstElement, ELEMENT); +QGST_DEFINE_CAST_TRAITS(GstObject, OBJECT); +QGST_DEFINE_CAST_TRAITS(GstPad, PAD); +QGST_DEFINE_CAST_TRAITS(GstPipeline, PIPELINE); +QGST_DEFINE_CAST_TRAITS(GstBaseSink, BASE_SINK); +QGST_DEFINE_CAST_TRAITS(GstBaseSrc, BASE_SRC); +QGST_DEFINE_CAST_TRAITS(GstAppSink, APP_SINK); +QGST_DEFINE_CAST_TRAITS(GstAppSrc, APP_SRC); + +QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE(GstTagSetter, TAG_SETTER); + + +template <> +struct GstObjectTraits<GObject> +{ + using Type = GObject; + template <typename U> + static bool isObjectOfType(U *arg) + { + return G_IS_OBJECT(arg); + } + template <typename U> + static Type *cast(U *arg) + { + return G_OBJECT(arg); + } + template <typename U> + static Type *checked_cast(U *arg) + { + return G_OBJECT(arg); + } +}; + +#undef QGST_DEFINE_CAST_TRAITS +#undef QGST_DEFINE_CAST_TRAITS_FOR_INTERFACE + +} // namespace QGstImpl + +template <typename DestinationType, typename SourceType> +bool qIsGstObjectOfType(SourceType *arg) +{ + using Traits = QGstImpl::GstObjectTraits<DestinationType>; + return arg && Traits::isObjectOfType(arg); +} + +template <typename DestinationType, typename SourceType> +DestinationType *qGstSafeCast(SourceType *arg) +{ + using Traits = QGstImpl::GstObjectTraits<DestinationType>; + if (arg && Traits::isObjectOfType(arg)) + return Traits::cast(arg); + return nullptr; +} + +template <typename DestinationType, typename SourceType> +DestinationType *qGstCheckedCast(SourceType *arg) +{ + using Traits = QGstImpl::GstObjectTraits<DestinationType>; + if (arg) + Q_ASSERT(Traits::isObjectOfType(arg)); + return Traits::cast(arg); +} + class QSize; -class QGstStructure; +class QGstStructureView; class QGstCaps; class QGstPipelinePrivate; class QCameraFormat; @@ -87,265 +178,284 @@ template <typename T> struct QGRange T max; }; -class QGString +struct QGString : QUniqueGStringHandle { - char *str; -public: - QGString(char *string) : str(string) {} - ~QGString() { g_free(str); } - operator QByteArray() { return QByteArray(str); } - operator const char *() { return str; } + using QUniqueGStringHandle::QUniqueGStringHandle; + + QLatin1StringView asStringView() const { return QLatin1StringView{ get() }; } + QString toQString() const { return QString::fromUtf8(get()); } }; class QGValue { public: - QGValue(const GValue *v) : value(v) {} + explicit QGValue(const GValue *v); const GValue *value; - bool isNull() const { return !value; } + bool isNull() const; - std::optional<bool> toBool() const + std::optional<bool> toBool() const; + std::optional<int> toInt() const; + std::optional<int> toInt64() const; + template<typename T> + T *getPointer() const { - if (!G_VALUE_HOLDS_BOOLEAN(value)) - return std::nullopt; - return g_value_get_boolean(value); + return value ? static_cast<T *>(g_value_get_pointer(value)) : nullptr; } - std::optional<int> toInt() const + + const char *toString() const; + std::optional<float> getFraction() const; + std::optional<QGRange<float>> getFractionRange() const; + std::optional<QGRange<int>> toIntRange() const; + + QGstStructureView toStructure() const; + QGstCaps toCaps() const; + + bool isList() const; + int listSize() const; + QGValue at(int index) const; + + QList<QAudioFormat::SampleFormat> getSampleFormats() const; +}; + +namespace QGstPointerImpl { + +template <typename RefcountedObject> +struct QGstRefcountingAdaptor; + +template <typename GstType> +class QGstObjectWrapper +{ + using Adaptor = QGstRefcountingAdaptor<GstType>; + + GstType *m_object = nullptr; + +public: + enum RefMode { HasRef, NeedsRef }; + + constexpr QGstObjectWrapper() = default; + + explicit QGstObjectWrapper(GstType *object, RefMode mode) : m_object(object) { - if (!G_VALUE_HOLDS_INT(value)) - return std::nullopt; - return g_value_get_int(value); + if (m_object && mode == NeedsRef) + Adaptor::ref(m_object); } - std::optional<int> toInt64() const + + QGstObjectWrapper(const QGstObjectWrapper &other) : m_object(other.m_object) { - if (!G_VALUE_HOLDS_INT64(value)) - return std::nullopt; - return g_value_get_int64(value); + if (m_object) + Adaptor::ref(m_object); } - template<typename T> - T *getPointer() const + + ~QGstObjectWrapper() { - return value ? static_cast<T *>(g_value_get_pointer(value)) : nullptr; + if (m_object) + Adaptor::unref(m_object); } - const char *toString() const + QGstObjectWrapper(QGstObjectWrapper &&other) noexcept + : m_object(std::exchange(other.m_object, nullptr)) { - return value ? g_value_get_string(value) : nullptr; } - std::optional<float> getFraction() const + + QGstObjectWrapper & + operator=(const QGstObjectWrapper &other) // NOLINT: bugprone-unhandled-self-assign { - if (!GST_VALUE_HOLDS_FRACTION(value)) - return std::nullopt; - return (float)gst_value_get_fraction_numerator(value)/(float)gst_value_get_fraction_denominator(value); + if (m_object != other.m_object) { + GstType *originalObject = m_object; + + m_object = other.m_object; + if (m_object) + Adaptor::ref(m_object); + if (originalObject) + Adaptor::unref(originalObject); + } + return *this; } - std::optional<QGRange<float>> getFractionRange() const + QGstObjectWrapper &operator=(QGstObjectWrapper &&other) noexcept { - if (!GST_VALUE_HOLDS_FRACTION_RANGE(value)) - return std::nullopt; - QGValue min = gst_value_get_fraction_range_min(value); - QGValue max = gst_value_get_fraction_range_max(value); - return QGRange<float>{ *min.getFraction(), *max.getFraction() }; + if (this != &other) { + GstType *originalObject = m_object; + m_object = std::exchange(other.m_object, nullptr); + + if (originalObject) + Adaptor::unref(originalObject); + } + return *this; } - std::optional<QGRange<int>> toIntRange() const + friend bool operator==(const QGstObjectWrapper &a, const QGstObjectWrapper &b) { - if (!GST_VALUE_HOLDS_INT_RANGE(value)) - return std::nullopt; - return QGRange<int>{ gst_value_get_int_range_min(value), gst_value_get_int_range_max(value) }; + return a.m_object == b.m_object; + } + friend bool operator!=(const QGstObjectWrapper &a, const QGstObjectWrapper &b) + { + return a.m_object != b.m_object; } - inline QGstStructure toStructure() const; - inline QGstCaps toCaps() const; - - inline bool isList() const { return value && GST_VALUE_HOLDS_LIST(value); } - inline int listSize() const { return gst_value_list_get_size(value); } - inline QGValue at(int index) const { return gst_value_list_get_value(value, index); } + explicit operator bool() const { return bool(m_object); } + bool isNull() const { return !m_object; } + GstType *release() { return std::exchange(m_object, nullptr); } - Q_MULTIMEDIA_EXPORT QList<QAudioFormat::SampleFormat> getSampleFormats() const; +protected: + GstType *get() const { return m_object; } }; -class QGstStructure { +} // namespace QGstPointerImpl + +class QGstreamerMessage; + +class QGstStructureView +{ public: const GstStructure *structure = nullptr; - QGstStructure() = default; - QGstStructure(const GstStructure *s) : structure(s) {} - void free() { if (structure) gst_structure_free(const_cast<GstStructure *>(structure)); structure = nullptr; } + explicit QGstStructureView(const GstStructure *); + explicit QGstStructureView(const QUniqueGstStructureHandle &); - bool isNull() const { return !structure; } + QUniqueGstStructureHandle clone() const; - QByteArrayView name() const { return gst_structure_get_name(structure); } + bool isNull() const; + QByteArrayView name() const; + QGValue operator[](const char *fieldname) const; - QGValue operator[](const char *name) const { return gst_structure_get_value(structure, name); } + QGstCaps caps() const; + QGstTagListHandle tags() const; - Q_MULTIMEDIA_EXPORT QSize resolution() const; - Q_MULTIMEDIA_EXPORT QVideoFrameFormat::PixelFormat pixelFormat() const; - Q_MULTIMEDIA_EXPORT QGRange<float> frameRateRange() const; + QSize resolution() const; + QVideoFrameFormat::PixelFormat pixelFormat() const; + QGRange<float> frameRateRange() const; + QGstreamerMessage getMessage(); + std::optional<Fraction> pixelAspectRatio() const; + QSize nativeSize() const; +}; - QByteArray toString() const - { - char *s = gst_structure_to_string(structure); - QByteArray str(s); - g_free(s); - return str; - } - QGstStructure copy() const { return gst_structure_copy(structure); } +template <> +struct QGstPointerImpl::QGstRefcountingAdaptor<GstCaps> +{ + static void ref(GstCaps *arg) noexcept { gst_caps_ref(arg); } + static void unref(GstCaps *arg) noexcept { gst_caps_unref(arg); } }; -class QGstCaps { - const GstCaps *caps = nullptr; +class QGstCaps : public QGstPointerImpl::QGstObjectWrapper<GstCaps> +{ + using BaseClass = QGstPointerImpl::QGstObjectWrapper<GstCaps>; + public: - enum MemoryFormat { - CpuMemory, - GLTexture, - DMABuf - }; + using BaseClass::BaseClass; + QGstCaps(const QGstCaps &) = default; + QGstCaps(QGstCaps &&) noexcept = default; + QGstCaps &operator=(const QGstCaps &) = default; + QGstCaps &operator=(QGstCaps &&) noexcept = default; - QGstCaps() = default; - QGstCaps(const GstCaps *c) : caps(c) {} + enum MemoryFormat { CpuMemory, GLTexture, DMABuf }; - bool isNull() const { return !caps; } + int size() const; + QGstStructureView at(int index) const; + GstCaps *caps() const; - int size() const { return gst_caps_get_size(caps); } - QGstStructure at(int index) { return gst_caps_get_structure(caps, index); } - const GstCaps *get() const { return caps; } - QByteArray toString() const - { - gchar *c = gst_caps_to_string(caps); - QByteArray b(c); - g_free(c); - return b; - } - MemoryFormat memoryFormat() const { - auto *features = gst_caps_get_features(caps, 0); - if (gst_caps_features_contains(features, "memory:GLMemory")) - return GLTexture; - else if (gst_caps_features_contains(features, "memory:DMABuf")) - return DMABuf; - return CpuMemory; - } - QVideoFrameFormat formatForCaps(GstVideoInfo *info) const; + MemoryFormat memoryFormat() const; + std::optional<std::pair<QVideoFrameFormat, GstVideoInfo>> formatAndVideoInfo() const; + + void addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, const char *modifier = nullptr); + void setResolution(QSize); + + static QGstCaps create(); + + static QGstCaps fromCameraFormat(const QCameraFormat &format); + + QGstCaps copy() const; +}; + +template <> +struct QGstPointerImpl::QGstRefcountingAdaptor<GstObject> +{ + static void ref(GstObject *arg) noexcept { gst_object_ref_sink(arg); } + static void unref(GstObject *arg) noexcept { gst_object_unref(arg); } }; -class QGstMutableCaps : public QGstCaps { - GstCaps *caps = nullptr; +class QGObjectHandlerConnection; + +class QGstObject : public QGstPointerImpl::QGstObjectWrapper<GstObject> +{ + using BaseClass = QGstPointerImpl::QGstObjectWrapper<GstObject>; + public: - enum RefMode { HasRef, NeedsRef }; - QGstMutableCaps() = default; - QGstMutableCaps(GstCaps *c, RefMode mode = HasRef) - : QGstCaps(c), caps(c) - { - Q_ASSERT(QGstCaps::get() == caps); - if (mode == NeedsRef) - gst_caps_ref(caps); - } - QGstMutableCaps(const QGstMutableCaps &other) - : QGstCaps(other), caps(other.caps) - { - Q_ASSERT(QGstCaps::get() == caps); - if (caps) - gst_caps_ref(caps); - } - QGstMutableCaps &operator=(const QGstMutableCaps &other) - { - QGstCaps::operator=(other); - if (other.caps) - gst_caps_ref(other.caps); - if (caps) - gst_caps_unref(caps); - caps = other.caps; - Q_ASSERT(QGstCaps::get() == caps); - return *this; - } - ~QGstMutableCaps() { - Q_ASSERT(QGstCaps::get() == caps); - if (caps) - gst_caps_unref(caps); - } + using BaseClass::BaseClass; + QGstObject(const QGstObject &) = default; + QGstObject(QGstObject &&) noexcept = default; + + QGstObject &operator=(const QGstObject &) = default; + QGstObject &operator=(QGstObject &&) noexcept = default; + + void set(const char *property, const char *str); + void set(const char *property, bool b); + void set(const char *property, uint i); + void set(const char *property, int i); + void set(const char *property, qint64 i); + void set(const char *property, quint64 i); + void set(const char *property, double d); + void set(const char *property, const QGstObject &o); + void set(const char *property, const QGstCaps &c); + + QGString getString(const char *property) const; + QGstStructureView getStructure(const char *property) const; + bool getBool(const char *property) const; + uint getUInt(const char *property) const; + int getInt(const char *property) const; + quint64 getUInt64(const char *property) const; + qint64 getInt64(const char *property) const; + float getFloat(const char *property) const; + double getDouble(const char *property) const; + QGstObject getObject(const char *property) const; + + QGObjectHandlerConnection connect(const char *name, GCallback callback, gpointer userData); + void disconnect(gulong handlerId); + + GType type() const; + QLatin1StringView typeName() const; + GstObject *object() const; + QLatin1StringView name() const; +}; - void create() { - caps = gst_caps_new_empty(); - QGstCaps::operator=(QGstCaps(caps)); - } +class QGObjectHandlerConnection +{ +public: + QGObjectHandlerConnection(QGstObject object, gulong handler); - void addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, const char *modifier = nullptr); - static QGstMutableCaps fromCameraFormat(const QCameraFormat &format); + QGObjectHandlerConnection() = default; + QGObjectHandlerConnection(const QGObjectHandlerConnection &) = default; + QGObjectHandlerConnection(QGObjectHandlerConnection &&) = default; + QGObjectHandlerConnection &operator=(const QGObjectHandlerConnection &) = default; + QGObjectHandlerConnection &operator=(QGObjectHandlerConnection &&) = default; + + void disconnect(); + +private: + static constexpr gulong invalidHandlerId = std::numeric_limits<gulong>::max(); - GstCaps *get() const { return caps; } + QGstObject object; + gulong handlerId = invalidHandlerId; }; -class QGstObject +// disconnects in dtor +class QGObjectHandlerScopedConnection { -protected: - GstObject *m_object = nullptr; public: - enum RefMode { HasRef, NeedsRef }; + QGObjectHandlerScopedConnection(QGObjectHandlerConnection connection); - QGstObject() = default; - explicit QGstObject(GstObject *o, RefMode mode = HasRef) - : m_object(o) - { - if (o && mode == NeedsRef) - // Use ref_sink to remove any floating references - gst_object_ref_sink(m_object); - } - QGstObject(const QGstObject &other) - : m_object(other.m_object) - { - if (m_object) - gst_object_ref(m_object); - } - QGstObject &operator=(const QGstObject &other) - { - if (this == &other) - return *this; - if (other.m_object) - gst_object_ref(other.m_object); - if (m_object) - gst_object_unref(m_object); - m_object = other.m_object; - return *this; - } - virtual ~QGstObject() { - if (m_object) - gst_object_unref(m_object); - } + QGObjectHandlerScopedConnection() = default; + QGObjectHandlerScopedConnection(const QGObjectHandlerScopedConnection &) = delete; + QGObjectHandlerScopedConnection &operator=(const QGObjectHandlerScopedConnection &) = delete; + QGObjectHandlerScopedConnection(QGObjectHandlerScopedConnection &&) = default; + QGObjectHandlerScopedConnection &operator=(QGObjectHandlerScopedConnection &&) = default; - friend bool operator==(const QGstObject &a, const QGstObject &b) - { return a.m_object == b.m_object; } - friend bool operator!=(const QGstObject &a, const QGstObject &b) - { return a.m_object != b.m_object; } + ~QGObjectHandlerScopedConnection(); - bool isNull() const { return !m_object; } + void disconnect(); - void set(const char *property, const char *str) { g_object_set(m_object, property, str, nullptr); } - void set(const char *property, bool b) { g_object_set(m_object, property, gboolean(b), nullptr); } - void set(const char *property, uint i) { g_object_set(m_object, property, guint(i), nullptr); } - void set(const char *property, int i) { g_object_set(m_object, property, gint(i), nullptr); } - void set(const char *property, qint64 i) { g_object_set(m_object, property, gint64(i), nullptr); } - void set(const char *property, quint64 i) { g_object_set(m_object, property, guint64(i), nullptr); } - void set(const char *property, double d) { g_object_set(m_object, property, gdouble(d), nullptr); } - void set(const char *property, const QGstObject &o) { g_object_set(m_object, property, o.object(), nullptr); } - void set(const char *property, const QGstMutableCaps &c) { g_object_set(m_object, property, c.get(), nullptr); } - - QGString getString(const char *property) const - { char *s = nullptr; g_object_get(m_object, property, &s, nullptr); return s; } - QGstStructure getStructure(const char *property) const - { GstStructure *s = nullptr; g_object_get(m_object, property, &s, nullptr); return QGstStructure(s); } - bool getBool(const char *property) const { gboolean b = false; g_object_get(m_object, property, &b, nullptr); return b; } - uint getUInt(const char *property) const { guint i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - int getInt(const char *property) const { gint i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - quint64 getUInt64(const char *property) const { guint64 i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - qint64 getInt64(const char *property) const { gint64 i = 0; g_object_get(m_object, property, &i, nullptr); return i; } - float getFloat(const char *property) const { gfloat d = 0; g_object_get(m_object, property, &d, nullptr); return d; } - double getDouble(const char *property) const { gdouble d = 0; g_object_get(m_object, property, &d, nullptr); return d; } - QGstObject getObject(const char *property) const { GstObject *o = nullptr; g_object_get(m_object, property, &o, nullptr); return QGstObject(o, HasRef); } - - void connect(const char *name, GCallback callback, gpointer userData) { g_signal_connect(m_object, name, callback, userData); } - - GstObject *object() const { return m_object; } - const char *name() const { return m_object ? GST_OBJECT_NAME(m_object) : "(null)"; } +private: + QGObjectHandlerConnection connection; }; class QGstElement; @@ -353,48 +463,57 @@ class QGstElement; class QGstPad : public QGstObject { public: - QGstPad() = default; - QGstPad(const QGstObject &o) - : QGstPad(GST_PAD(o.object()), NeedsRef) - {} - QGstPad(GstPad *pad, RefMode mode = NeedsRef) - : QGstObject(&pad->object, mode) - {} - - QGstMutableCaps currentCaps() const - { return QGstMutableCaps(gst_pad_get_current_caps(pad())); } - QGstCaps queryCaps() const - { return QGstCaps(gst_pad_query_caps(pad(), nullptr)); } - - bool isLinked() const { return gst_pad_is_linked(pad()); } - bool link(const QGstPad &sink) const { return gst_pad_link(pad(), sink.pad()) == GST_PAD_LINK_OK; } - bool unlink(const QGstPad &sink) const { return gst_pad_unlink(pad(), sink.pad()); } - bool unlinkPeer() const { return unlink(peer()); } - QGstPad peer() const { return QGstPad(gst_pad_get_peer(pad()), HasRef); } - inline QGstElement parent() const; - - GstPad *pad() const { return GST_PAD_CAST(object()); } - - GstEvent *stickyEvent(GstEventType type) { return gst_pad_get_sticky_event(pad(), type, 0); } - bool sendEvent(GstEvent *event) { return gst_pad_send_event (pad(), event); } + using QGstObject::QGstObject; + QGstPad(const QGstPad &) = default; + QGstPad(QGstPad &&) noexcept = default; + + explicit QGstPad(const QGstObject &o); + explicit QGstPad(GstPad *pad, RefMode mode); + + QGstPad &operator=(const QGstPad &) = default; + QGstPad &operator=(QGstPad &&) noexcept = default; + + QGstCaps currentCaps() const; + QGstCaps queryCaps() const; + + QGstTagListHandle tags() const; + + std::optional<QPlatformMediaPlayer::TrackType> + inferTrackTypeFromName() const; // for decodebin3 etc + + bool isLinked() const; + bool link(const QGstPad &sink) const; + bool unlink(const QGstPad &sink) const; + bool unlinkPeer() const; + QGstPad peer() const; + QGstElement parent() const; + + GstPad *pad() const; + + GstEvent *stickyEvent(GstEventType type); + bool sendEvent(GstEvent *event); template<auto Member, typename T> void addProbe(T *instance, GstPadProbeType type) { - struct Impl { - static GstPadProbeReturn callback(GstPad *pad, GstPadProbeInfo *info, gpointer userData) { - return (static_cast<T *>(userData)->*Member)(QGstPad(pad, NeedsRef), info); - }; + auto callback = [](GstPad *pad, GstPadProbeInfo *info, gpointer userData) { + return (static_cast<T *>(userData)->*Member)(QGstPad(pad, NeedsRef), info); }; - gst_pad_add_probe (pad(), type, Impl::callback, instance, nullptr); + gst_pad_add_probe(pad(), type, callback, instance, nullptr); } - void doInIdleProbe(std::function<void()> work) { + template <typename Functor> + void doInIdleProbe(Functor &&work) + { struct CallbackData { QSemaphore waitDone; - std::function<void()> work; - } cd; - cd.work = work; + Functor work; + }; + + CallbackData cd{ + .waitDone = QSemaphore{}, + .work = std::forward<Functor>(work), + }; auto callback= [](GstPad *, GstPadProbeInfo *, gpointer p) { auto cd = reinterpret_cast<CallbackData*>(p); @@ -409,16 +528,14 @@ public: template<auto Member, typename T> void addEosProbe(T *instance) { - struct Impl { - static GstPadProbeReturn callback(GstPad */*pad*/, GstPadProbeInfo *info, gpointer userData) { - if (GST_EVENT_TYPE(GST_PAD_PROBE_INFO_DATA(info)) != GST_EVENT_EOS) - return GST_PAD_PROBE_PASS; - (static_cast<T *>(userData)->*Member)(); - return GST_PAD_PROBE_REMOVE; - }; + auto callback = [](GstPad *, GstPadProbeInfo *info, gpointer userData) { + if (GST_EVENT_TYPE(GST_PAD_PROBE_INFO_DATA(info)) != GST_EVENT_EOS) + return GST_PAD_PROBE_PASS; + (static_cast<T *>(userData)->*Member)(); + return GST_PAD_PROBE_REMOVE; }; - gst_pad_add_probe (pad(), GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, Impl::callback, instance, nullptr); + gst_pad_add_probe(pad(), GST_PAD_PROBE_TYPE_EVENT_DOWNSTREAM, callback, instance, nullptr); } }; @@ -426,195 +543,303 @@ class QGstClock : public QGstObject { public: QGstClock() = default; - QGstClock(const QGstObject &o) - : QGstClock(GST_CLOCK(o.object())) - {} - QGstClock(GstClock *clock, RefMode mode = NeedsRef) - : QGstObject(&clock->object, mode) - {} - - GstClock *clock() const { return GST_CLOCK_CAST(object()); } + explicit QGstClock(const QGstObject &o); + explicit QGstClock(GstClock *clock, RefMode mode); - GstClockTime time() const { return gst_clock_get_time(clock()); } + GstClock *clock() const; + GstClockTime time() const; }; +class QGstPipeline; + class QGstElement : public QGstObject { public: - QGstElement() = default; - QGstElement(const QGstObject &o) - : QGstElement(GST_ELEMENT(o.object()), NeedsRef) - {} - QGstElement(GstElement *element, RefMode mode = NeedsRef) - : QGstObject(&element->object, mode) - {} - - QGstElement(const char *factory, const char *name = nullptr) - : QGstElement(gst_element_factory_make(factory, name), NeedsRef) - { - } - - bool linkFiltered(const QGstElement &next, const QGstMutableCaps &caps) - { return gst_element_link_filtered(element(), next.element(), caps.get()); } - bool link(const QGstElement &next) - { return gst_element_link(element(), next.element()); } - bool link(const QGstElement &n1, const QGstElement &n2) - { return gst_element_link_many(element(), n1.element(), n2.element(), nullptr); } - bool link(const QGstElement &n1, const QGstElement &n2, const QGstElement &n3) - { return gst_element_link_many(element(), n1.element(), n2.element(), n3.element(), nullptr); } - bool link(const QGstElement &n1, const QGstElement &n2, const QGstElement &n3, const QGstElement &n4) - { return gst_element_link_many(element(), n1.element(), n2.element(), n3.element(), n4.element(), nullptr); } - bool link(const QGstElement &n1, const QGstElement &n2, const QGstElement &n3, const QGstElement &n4, const QGstElement &n5) - { return gst_element_link_many(element(), n1.element(), n2.element(), n3.element(), n4.element(), n5.element(), nullptr); } - - void unlink(const QGstElement &next) - { gst_element_unlink(element(), next.element()); } - - QGstPad staticPad(const char *name) const { return QGstPad(gst_element_get_static_pad(element(), name), HasRef); } - QGstPad src() const { return staticPad("src"); } - QGstPad sink() const { return staticPad("sink"); } - QGstPad getRequestPad(const char *name) const - { -#if GST_CHECK_VERSION(1,19,1) - return QGstPad(gst_element_request_pad_simple(element(), name), HasRef); -#else - return QGstPad(gst_element_get_request_pad(element(), name), HasRef); -#endif - } - void releaseRequestPad(const QGstPad &pad) const { return gst_element_release_request_pad(element(), pad.pad()); } - - GstState state() const - { - GstState state; - gst_element_get_state(element(), &state, nullptr, 0); - return state; - } - GstStateChangeReturn setState(GstState state) { return gst_element_set_state(element(), state); } - bool setStateSync(GstState state) + using QGstObject::QGstObject; + + QGstElement(const QGstElement &) = default; + QGstElement(QGstElement &&) noexcept = default; + QGstElement &operator=(const QGstElement &) = default; + QGstElement &operator=(QGstElement &&) noexcept = default; + + explicit QGstElement(GstElement *element, RefMode mode); + static QGstElement createFromFactory(const char *factory, const char *name = nullptr); + static QGstElement createFromFactory(GstElementFactory *, const char *name = nullptr); + static QGstElement createFromFactory(const QGstElementFactoryHandle &, + const char *name = nullptr); + static QGstElement createFromDevice(const QGstDeviceHandle &, const char *name = nullptr); + static QGstElement createFromDevice(GstDevice *, const char *name = nullptr); + static QGstElement createFromPipelineDescription(const char *); + static QGstElement createFromPipelineDescription(const QByteArray &); + + static QGstElementFactoryHandle findFactory(const char *); + static QGstElementFactoryHandle findFactory(const QByteArray &name); + + QGstPad staticPad(const char *name) const; + QGstPad src() const; + QGstPad sink() const; + QGstPad getRequestPad(const char *name) const; + void releaseRequestPad(const QGstPad &pad) const; + + GstState state(std::chrono::nanoseconds timeout = std::chrono::seconds(0)) const; + GstStateChangeReturn setState(GstState state); + bool setStateSync(GstState state, std::chrono::nanoseconds timeout = std::chrono::seconds(1)); + bool syncStateWithParent(); + bool finishStateChange(std::chrono::nanoseconds timeout = std::chrono::seconds(5)); + + void lockState(bool locked); + bool isStateLocked() const; + + void sendEvent(GstEvent *event) const; + void sendEos() const; + + std::optional<std::chrono::nanoseconds> duration() const; + std::optional<std::chrono::milliseconds> durationInMs() const; + std::optional<std::chrono::nanoseconds> position() const; + std::optional<std::chrono::milliseconds> positionInMs() const; + std::optional<bool> canSeek() const; + + template <auto Member, typename T> + QGObjectHandlerConnection onPadAdded(T *instance) { - auto change = gst_element_set_state(element(), state); - if (change == GST_STATE_CHANGE_ASYNC) { - change = gst_element_get_state(element(), nullptr, &state, 1000*1e6 /*nano seconds*/); - } -#ifndef QT_NO_DEBUG - if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) - qWarning() << "Could not change state of" << name() << "to" << state << change; -#endif - return change == GST_STATE_CHANGE_SUCCESS; - } - bool syncStateWithParent() { return gst_element_sync_state_with_parent(element()) == TRUE; } - bool finishStateChange() - { - auto change = gst_element_get_state(element(), nullptr, nullptr, 1000*1e6 /*nano seconds*/); -#ifndef QT_NO_DEBUG - if (change != GST_STATE_CHANGE_SUCCESS && change != GST_STATE_CHANGE_NO_PREROLL) - qWarning() << "Could finish change state of" << name(); -#endif - return change == GST_STATE_CHANGE_SUCCESS; - } - - void lockState(bool locked) { gst_element_set_locked_state(element(), locked); } - bool isStateLocked() const { return gst_element_is_locked_state(element()); } - - void sendEvent(GstEvent *event) const { gst_element_send_event(element(), event); } - void sendEos() const { sendEvent(gst_event_new_eos()); } - - template<auto Member, typename T> - void onPadAdded(T *instance) { - struct Impl { - static void callback(GstElement *e, GstPad *pad, gpointer userData) { - (static_cast<T *>(userData)->*Member)(QGstElement(e), QGstPad(pad, NeedsRef)); + struct Impl + { + static void callback(GstElement *e, GstPad *pad, gpointer userData) + { + (static_cast<T *>(userData)->*Member)(QGstElement(e, NeedsRef), + QGstPad(pad, NeedsRef)); }; }; - connect("pad-added", G_CALLBACK(Impl::callback), instance); + return connect("pad-added", G_CALLBACK(Impl::callback), instance); } - template<auto Member, typename T> - void onPadRemoved(T *instance) { - struct Impl { - static void callback(GstElement *e, GstPad *pad, gpointer userData) { - (static_cast<T *>(userData)->*Member)(QGstElement(e), QGstPad(pad, NeedsRef)); + template <auto Member, typename T> + QGObjectHandlerConnection onPadRemoved(T *instance) + { + struct Impl + { + static void callback(GstElement *e, GstPad *pad, gpointer userData) + { + (static_cast<T *>(userData)->*Member)(QGstElement(e, NeedsRef), + QGstPad(pad, NeedsRef)); }; }; - connect("pad-removed", G_CALLBACK(Impl::callback), instance); + return connect("pad-removed", G_CALLBACK(Impl::callback), instance); } - template<auto Member, typename T> - void onNoMorePads(T *instance) { - struct Impl { - static void callback(GstElement *e, gpointer userData) { - (static_cast<T *>(userData)->*Member)(QGstElement(e)); + template <auto Member, typename T> + QGObjectHandlerConnection onNoMorePads(T *instance) + { + struct Impl + { + static void callback(GstElement *e, gpointer userData) + { + (static_cast<T *>(userData)->*Member)(QGstElement(e, NeedsRef)); }; }; - connect("no-more-pads", G_CALLBACK(Impl::callback), instance); + return connect("no-more-pads", G_CALLBACK(Impl::callback), instance); } - GstClockTime baseTime() const { return gst_element_get_base_time(element()); } - void setBaseTime(GstClockTime time) const { gst_element_set_base_time(element(), time); } + GstClockTime baseTime() const; + void setBaseTime(GstClockTime time) const; + + GstElement *element() const; + + QGstElement getParent() const; + QGstPipeline getPipeline() const; + void dumpPipelineGraph(const char *filename) const; - GstElement *element() const { return GST_ELEMENT_CAST(m_object); } +private: + QGstQueryHandle &positionQuery() const; + mutable QGstQueryHandle m_positionQuery; }; -inline QGstElement QGstPad::parent() const +template <typename... Ts> +std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> +qLinkGstElements(const Ts &...ts) { - return QGstElement(gst_pad_get_parent_element(pad()), HasRef); + bool link_success = [&] { + if constexpr (sizeof...(Ts) == 2) + return gst_element_link(ts.element()...); + else + return gst_element_link_many(ts.element()..., nullptr); + }(); + + if (Q_UNLIKELY(!link_success)) { + qWarning() << "qLinkGstElements: could not link elements: " + << std::initializer_list<const char *>{ + (GST_ELEMENT_NAME(ts.element()))..., + }; + } +} + +template <typename... Ts> +std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> +qUnlinkGstElements(const Ts &...ts) +{ + if constexpr (sizeof...(Ts) == 2) + gst_element_unlink(ts.element()...); + else + gst_element_unlink_many(ts.element()..., nullptr); } class QGstBin : public QGstElement { public: - QGstBin() = default; - QGstBin(const QGstObject &o) - : QGstBin(GST_BIN(o.object()), NeedsRef) - {} - QGstBin(const char *name) - : QGstElement(gst_bin_new(name), NeedsRef) + using QGstElement::QGstElement; + QGstBin(const QGstBin &) = default; + QGstBin(QGstBin &&) noexcept = default; + QGstBin &operator=(const QGstBin &) = default; + QGstBin &operator=(QGstBin &&) noexcept = default; + + explicit QGstBin(GstBin *bin, RefMode mode = NeedsRef); + static QGstBin create(const char *name); + static QGstBin createFromFactory(const char *factory, const char *name); + static QGstBin createFromPipelineDescription(const QByteArray &pipelineDescription, + const char *name = nullptr, + bool ghostUnlinkedPads = false); + static QGstBin createFromPipelineDescription(const char *pipelineDescription, + const char *name = nullptr, + bool ghostUnlinkedPads = false); + + template <typename... Ts> + std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> add(const Ts &...ts) { + if constexpr (sizeof...(Ts) == 1) + gst_bin_add(bin(), ts.element()...); + else + gst_bin_add_many(bin(), ts.element()..., nullptr); } - QGstBin(GstBin *bin, RefMode mode = NeedsRef) - : QGstElement(&bin->element, mode) - {} - - void add(const QGstElement &element) - { gst_bin_add(bin(), element.element()); } - void add(const QGstElement &e1, const QGstElement &e2) - { gst_bin_add_many(bin(), e1.element(), e2.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3, const QGstElement &e4) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), e4.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3, const QGstElement &e4, const QGstElement &e5) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), e4.element(), e5.element(), nullptr); } - void add(const QGstElement &e1, const QGstElement &e2, const QGstElement &e3, const QGstElement &e4, const QGstElement &e5, const QGstElement &e6) - { gst_bin_add_many(bin(), e1.element(), e2.element(), e3.element(), e4.element(), e5.element(), e6.element(), nullptr); } - void remove(const QGstElement &element) - { gst_bin_remove(bin(), element.element()); } - - GstBin *bin() const { return GST_BIN_CAST(m_object); } - - void addGhostPad(const QGstElement &child, const char *name) + + template <typename... Ts> + std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> remove(const Ts &...ts) { - addGhostPad(name, child.staticPad(name)); + if constexpr (sizeof...(Ts) == 1) + gst_bin_remove(bin(), ts.element()...); + else + gst_bin_remove_many(bin(), ts.element()..., nullptr); } - void addGhostPad(const char *name, const QGstPad &pad) + + template <typename... Ts> + std::enable_if_t<(std::is_base_of_v<QGstElement, Ts> && ...), void> + stopAndRemoveElements(Ts... ts) { - gst_element_add_pad(element(), gst_ghost_pad_new(name, pad.pad())); + bool stateChangeSuccessful = (ts.setStateSync(GST_STATE_NULL) && ...); + Q_ASSERT(stateChangeSuccessful); + remove(ts...); } + + GstBin *bin() const; + + void addGhostPad(const QGstElement &child, const char *name); + void addGhostPad(const char *name, const QGstPad &pad); + + bool syncChildrenState(); + + void dumpGraph(const char *fileNamePrefix) const; + + QGstElement findByName(const char *); +}; + +class QGstBaseSink : public QGstElement +{ +public: + using QGstElement::QGstElement; + + explicit QGstBaseSink(GstBaseSink *, RefMode); + + QGstBaseSink(const QGstBaseSink &) = default; + QGstBaseSink(QGstBaseSink &&) noexcept = default; + QGstBaseSink &operator=(const QGstBaseSink &) = default; + QGstBaseSink &operator=(QGstBaseSink &&) noexcept = default; + + void setSync(bool); + + GstBaseSink *baseSink() const; +}; + +class QGstBaseSrc : public QGstElement +{ +public: + using QGstElement::QGstElement; + + explicit QGstBaseSrc(GstBaseSrc *, RefMode); + + QGstBaseSrc(const QGstBaseSrc &) = default; + QGstBaseSrc(QGstBaseSrc &&) noexcept = default; + QGstBaseSrc &operator=(const QGstBaseSrc &) = default; + QGstBaseSrc &operator=(QGstBaseSrc &&) noexcept = default; + + GstBaseSrc *baseSrc() const; +}; + +class QGstAppSink : public QGstBaseSink +{ +public: + using QGstBaseSink::QGstBaseSink; + + explicit QGstAppSink(GstAppSink *, RefMode); + + QGstAppSink(const QGstAppSink &) = default; + QGstAppSink(QGstAppSink &&) noexcept = default; + QGstAppSink &operator=(const QGstAppSink &) = default; + QGstAppSink &operator=(QGstAppSink &&) noexcept = default; + + static QGstAppSink create(const char *name); + + GstAppSink *appSink() const; + + void setMaxBuffers(int); +# if GST_CHECK_VERSION(1, 24, 0) + void setMaxBufferTime(std::chrono::nanoseconds); +# endif + + void setCaps(const QGstCaps &caps); + void setCallbacks(GstAppSinkCallbacks &callbacks, gpointer user_data, GDestroyNotify notify); + + QGstSampleHandle pullSample(); }; -inline QGstStructure QGValue::toStructure() const +class QGstAppSrc : public QGstBaseSrc { - if (!value || !GST_VALUE_HOLDS_STRUCTURE(value)) - return QGstStructure(); - return QGstStructure(gst_value_get_structure(value)); +public: + using QGstBaseSrc::QGstBaseSrc; + + explicit QGstAppSrc(GstAppSrc *, RefMode); + + QGstAppSrc(const QGstAppSrc &) = default; + QGstAppSrc(QGstAppSrc &&) noexcept = default; + QGstAppSrc &operator=(const QGstAppSrc &) = default; + QGstAppSrc &operator=(QGstAppSrc &&) noexcept = default; + + static QGstAppSrc create(const char *name); + + GstAppSrc *appSrc() const; + + void setCallbacks(GstAppSrcCallbacks &callbacks, gpointer user_data, GDestroyNotify notify); + + GstFlowReturn pushBuffer(GstBuffer *); // take ownership +}; + +inline GstClockTime qGstClockTimeFromChrono(std::chrono::nanoseconds ns) +{ + return ns.count(); } -inline QGstCaps QGValue::toCaps() const +QString qGstErrorMessageCannotFindElement(std::string_view element); + +template <typename Arg, typename... Args> +std::optional<QString> qGstErrorMessageIfElementsNotAvailable(const Arg &arg, Args... args) { - if (!value || !GST_VALUE_HOLDS_CAPS(value)) - return QGstCaps(); - return QGstCaps(gst_value_get_caps(value)); + QGstElementFactoryHandle factory = QGstElement::findFactory(arg); + if (!factory) + return qGstErrorMessageCannotFindElement(arg); + + if constexpr (sizeof...(args) != 0) + return qGstErrorMessageIfElementsNotAvailable(args...); + else + return std::nullopt; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp b/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp new file mode 100644 index 000000000..5779ba8b1 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgstappsource.cpp @@ -0,0 +1,240 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qgstappsource_p.h" + +#include <QtCore/qdebug.h> +#include <QtCore/qloggingcategory.h> + +#include <common/qgstutils_p.h> + +static Q_LOGGING_CATEGORY(qLcAppSrc, "qt.multimedia.appsrc") + +QT_BEGIN_NAMESPACE + +QMaybe<QGstAppSource *> QGstAppSource::create(QObject *parent) +{ + QGstAppSrc appsrc = QGstAppSrc::create("appsrc"); + if (!appsrc) + return qGstErrorMessageCannotFindElement("appsrc"); + + return new QGstAppSource(appsrc, parent); +} + +QGstAppSource::QGstAppSource(QGstAppSrc appsrc, QObject *parent) + : QObject(parent), m_appSrc(std::move(appsrc)) +{ + m_appSrc.set("emit-signals", false); +} + +QGstAppSource::~QGstAppSource() +{ + m_appSrc.setStateSync(GST_STATE_NULL); + streamDestroyed(); + qCDebug(qLcAppSrc) << "~QGstAppSrc"; +} + +bool QGstAppSource::setup(QIODevice *stream, qint64 offset) +{ + QMutexLocker locker(&m_mutex); + + if (m_appSrc.isNull()) + return false; + + if (!setStream(stream, offset)) + return false; + + GstAppSrcCallbacks callbacks{}; + callbacks.need_data = QGstAppSource::on_need_data; + callbacks.enough_data = QGstAppSource::on_enough_data; + callbacks.seek_data = QGstAppSource::on_seek_data; + + m_appSrc.setCallbacks(callbacks, this, nullptr); + + GstAppSrc *appSrc = m_appSrc.appSrc(); + m_maxBytes = gst_app_src_get_max_bytes(appSrc); + + if (m_sequential) + m_streamType = GST_APP_STREAM_TYPE_STREAM; + else + m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; + gst_app_src_set_stream_type(appSrc, m_streamType); + gst_app_src_set_size(appSrc, m_sequential ? -1 : m_stream->size() - m_offset); + + return true; +} + +void QGstAppSource::setExternalAppSrc(QGstAppSrc appsrc) +{ + QMutexLocker locker(&m_mutex); + m_appSrc = std::move(appsrc); +} + +bool QGstAppSource::setStream(QIODevice *stream, qint64 offset) +{ + if (m_stream) { + disconnect(m_stream, &QIODevice::readyRead, this, &QGstAppSource::onDataReady); + disconnect(m_stream, &QIODevice::destroyed, this, &QGstAppSource::streamDestroyed); + m_stream = nullptr; + } + + m_dataRequestSize = 0; + m_sequential = true; + m_maxBytes = 0; + + if (stream) { + if (!stream->isOpen() && !stream->open(QIODevice::ReadOnly)) + return false; + m_stream = stream; + connect(m_stream, &QIODevice::destroyed, this, &QGstAppSource::streamDestroyed); + connect(m_stream, &QIODevice::readyRead, this, &QGstAppSource::onDataReady); + m_sequential = m_stream->isSequential(); + m_offset = offset; + } + return true; +} + +bool QGstAppSource::isStreamValid() const +{ + return m_stream != nullptr && m_stream->isOpen(); +} + +QGstElement QGstAppSource::element() const +{ + return m_appSrc; +} + +void QGstAppSource::onDataReady() +{ + qCDebug(qLcAppSrc) << "onDataReady" << m_stream->bytesAvailable() << m_stream->size(); + pushData(); +} + +void QGstAppSource::streamDestroyed() +{ + qCDebug(qLcAppSrc) << "stream destroyed"; + m_stream = nullptr; + m_dataRequestSize = 0; + sendEOS(); +} + +void QGstAppSource::pushData() +{ + if (m_appSrc.isNull() || !m_dataRequestSize) { + qCDebug(qLcAppSrc) << "push data: return immediately" << m_appSrc.isNull() + << m_dataRequestSize; + return; + } + + Q_ASSERT(m_stream); + + qCDebug(qLcAppSrc) << "pushData" << m_stream; + if ((m_stream && m_stream->atEnd())) { + sendEOS(); + qCDebug(qLcAppSrc) << "end pushData" << m_stream; + return; + } + + qint64 size = m_stream->bytesAvailable(); + + if (!m_dataRequestSize) + m_dataRequestSize = m_maxBytes; + size = qMin(size, (qint64)m_dataRequestSize); + qCDebug(qLcAppSrc) << " reading" << size << "bytes" << size << m_dataRequestSize; + + GstBuffer* buffer = gst_buffer_new_and_alloc(size); + + if (m_sequential) + buffer->offset = bytesReadSoFar; + else + buffer->offset = m_stream->pos(); + + GstMapInfo mapInfo; + gst_buffer_map(buffer, &mapInfo, GST_MAP_WRITE); + void* bufferData = mapInfo.data; + + qint64 bytesRead; + bytesRead = m_stream->read((char *)bufferData, size); + + buffer->offset_end = buffer->offset + bytesRead - 1; + bytesReadSoFar += bytesRead; + + gst_buffer_unmap(buffer, &mapInfo); + qCDebug(qLcAppSrc) << "pushing bytes into gstreamer" << buffer->offset << bytesRead; + if (bytesRead == 0) { + gst_buffer_unref(buffer); + sendEOS(); + qCDebug(qLcAppSrc) << "end pushData" << m_stream; + return; + } + + GstFlowReturn ret = m_appSrc.pushBuffer(buffer); + switch (ret) { + case GST_FLOW_OK: + break; + + default: + qWarning() << "QGstAppSrc: push buffer error -" << gst_flow_get_name(ret); + break; + } + + qCDebug(qLcAppSrc) << "end pushData" << m_stream; +} + +bool QGstAppSource::doSeek(qint64 value) +{ + if (isStreamValid()) + return m_stream->seek(value + m_offset); + return false; +} + +gboolean QGstAppSource::on_seek_data(GstAppSrc *, guint64 arg0, gpointer userdata) +{ + // we do get some spurious seeks to INT_MAX, ignore those + if (arg0 == std::numeric_limits<quint64>::max()) + return true; + + QGstAppSource *self = reinterpret_cast<QGstAppSource *>(userdata); + Q_ASSERT(self); + + QMutexLocker locker(&self->m_mutex); + + if (self->m_sequential) + return false; + + self->doSeek(arg0); + return true; +} + +void QGstAppSource::on_enough_data(GstAppSrc *, gpointer userdata) +{ + qCDebug(qLcAppSrc) << "on_enough_data"; + QGstAppSource *self = static_cast<QGstAppSource *>(userdata); + Q_ASSERT(self); + QMutexLocker locker(&self->m_mutex); + self->m_dataRequestSize = 0; +} + +void QGstAppSource::on_need_data(GstAppSrc *, guint arg0, gpointer userdata) +{ + qCDebug(qLcAppSrc) << "on_need_data requesting bytes" << arg0; + QGstAppSource *self = static_cast<QGstAppSource *>(userdata); + Q_ASSERT(self); + QMutexLocker locker(&self->m_mutex); + self->m_dataRequestSize = arg0; + self->pushData(); + qCDebug(qLcAppSrc) << "done on_need_data"; +} + +void QGstAppSource::sendEOS() +{ + qCDebug(qLcAppSrc) << "sending EOS"; + if (m_appSrc.isNull()) + return; + + gst_app_src_end_of_stream(GST_APP_SRC(m_appSrc.element())); +} + +QT_END_NAMESPACE + +#include "moc_qgstappsource_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsource_p.h b/src/plugins/multimedia/gstreamer/common/qgstappsource_p.h new file mode 100644 index 000000000..b181212d2 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/common/qgstappsource_p.h @@ -0,0 +1,77 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGSTAPPSRC_H +#define QGSTAPPSRC_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + + +#include <QtCore/qobject.h> +#include <QtCore/qiodevice.h> +#include <QtCore/qatomic.h> +#include <QtCore/qmutex.h> + +#include <QtMultimedia/private/qtmultimediaglobal_p.h> + +#include <common/qgst_p.h> +#include <gst/app/gstappsrc.h> + +QT_BEGIN_NAMESPACE + +class QGstAppSource : public QObject +{ + Q_OBJECT +public: + static QMaybe<QGstAppSource *> create(QObject *parent = nullptr); + ~QGstAppSource(); + + bool setup(QIODevice *stream = nullptr, qint64 offset = 0); + + void setExternalAppSrc(QGstAppSrc); + QGstElement element() const; + +private Q_SLOTS: + void onDataReady(); + void streamDestroyed(); + +private: + bool doSeek(qint64); + void pushData(); + + QGstAppSource(QGstAppSrc appsrc, QObject *parent); + + bool setStream(QIODevice *, qint64 offset); + bool isStreamValid() const; + + static gboolean on_seek_data(GstAppSrc *element, guint64 arg0, gpointer userdata); + static void on_enough_data(GstAppSrc *element, gpointer userdata); + static void on_need_data(GstAppSrc *element, uint arg0, gpointer userdata); + + void sendEOS(); + + mutable QMutex m_mutex; + + QIODevice *m_stream = nullptr; + + QGstAppSrc m_appSrc; + bool m_sequential = true; + GstAppStreamType m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; + qint64 m_offset = 0; + qint64 m_maxBytes = 0; + qint64 bytesReadSoFar = 0; + QAtomicInteger<unsigned int> m_dataRequestSize = 0; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsrc.cpp b/src/plugins/multimedia/gstreamer/common/qgstappsrc.cpp deleted file mode 100644 index 992e3d5a9..000000000 --- a/src/plugins/multimedia/gstreamer/common/qgstappsrc.cpp +++ /dev/null @@ -1,308 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QDebug> - -#include "qgstappsrc_p.h" -#include "qgstutils_p.h" -#include "qnetworkreply.h" -#include "qloggingcategory.h" - -Q_LOGGING_CATEGORY(qLcAppSrc, "qt.multimedia.appsrc") - -QGstAppSrc::QGstAppSrc(QObject *parent) - : QObject(parent) -{ - m_appSrc = QGstElement("appsrc", "appsrc"); - if (m_appSrc.isNull()) - qWarning() << "Could not create GstAppSrc."; -} - -QGstAppSrc::~QGstAppSrc() -{ - m_appSrc.setStateSync(GST_STATE_NULL); - streamDestroyed(); - qCDebug(qLcAppSrc) << "~QGstAppSrc"; -} - -bool QGstAppSrc::setup(QIODevice *stream, qint64 offset) -{ - if (m_appSrc.isNull()) - return false; - - if (!setStream(stream, offset)) - return false; - - auto *appSrc = GST_APP_SRC(m_appSrc.element()); - GstAppSrcCallbacks m_callbacks; - memset(&m_callbacks, 0, sizeof(GstAppSrcCallbacks)); - m_callbacks.need_data = &QGstAppSrc::on_need_data; - m_callbacks.enough_data = &QGstAppSrc::on_enough_data; - m_callbacks.seek_data = &QGstAppSrc::on_seek_data; - gst_app_src_set_callbacks(appSrc, (GstAppSrcCallbacks*)&m_callbacks, this, nullptr); - - m_maxBytes = gst_app_src_get_max_bytes(appSrc); - m_suspended = false; - - if (m_sequential) - m_streamType = GST_APP_STREAM_TYPE_STREAM; - else - m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; - gst_app_src_set_stream_type(appSrc, m_streamType); - gst_app_src_set_size(appSrc, m_sequential ? -1 : m_stream->size() - m_offset); - - m_networkReply = qobject_cast<QNetworkReply *>(m_stream); - m_noMoreData = true; - - return true; -} - -void QGstAppSrc::setAudioFormat(const QAudioFormat &f) -{ - m_format = f; - if (!m_format.isValid()) - return; - - auto caps = QGstUtils::capsForAudioFormat(m_format); - Q_ASSERT(!caps.isNull()); - m_appSrc.set("caps", caps); - m_appSrc.set("format", GST_FORMAT_TIME); -} - -void QGstAppSrc::setExternalAppSrc(const QGstElement &appsrc) -{ - m_appSrc = appsrc; -} - -bool QGstAppSrc::setStream(QIODevice *stream, qint64 offset) -{ - if (m_stream) { - disconnect(m_stream, SIGNAL(readyRead()), this, SLOT(onDataReady())); - disconnect(m_stream, SIGNAL(destroyed()), this, SLOT(streamDestroyed())); - m_stream = nullptr; - } - - m_dataRequestSize = 0; - m_sequential = true; - m_maxBytes = 0; - streamedSamples = 0; - - if (stream) { - if (!stream->isOpen() && !stream->open(QIODevice::ReadOnly)) - return false; - m_stream = stream; - connect(m_stream, SIGNAL(destroyed()), SLOT(streamDestroyed())); - connect(m_stream, SIGNAL(readyRead()), this, SLOT(onDataReady())); - m_sequential = m_stream->isSequential(); - m_offset = offset; - } - return true; -} - -QGstElement QGstAppSrc::element() -{ - return m_appSrc; -} - -void QGstAppSrc::write(const char *data, qsizetype size) -{ - qCDebug(qLcAppSrc) << "write" << size << m_noMoreData << m_dataRequestSize; - if (!size) - return; - Q_ASSERT(!m_stream); - m_buffer.append(data, size); - m_noMoreData = false; - pushData(); -} - -void QGstAppSrc::onDataReady() -{ - qCDebug(qLcAppSrc) << "onDataReady" << m_stream->bytesAvailable() << m_stream->size(); - pushData(); -} - -void QGstAppSrc::streamDestroyed() -{ - qCDebug(qLcAppSrc) << "stream destroyed"; - m_stream = nullptr; - m_dataRequestSize = 0; - streamedSamples = 0; - sendEOS(); -} - -void QGstAppSrc::pushData() -{ - if (m_appSrc.isNull() || !m_dataRequestSize || m_suspended) { - qCDebug(qLcAppSrc) << "push data: return immediately" << m_appSrc.isNull() << m_dataRequestSize << m_suspended; - return; - } - - qCDebug(qLcAppSrc) << "pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - if ((m_stream && m_stream->atEnd())) { - eosOrIdle(); - qCDebug(qLcAppSrc) << "end pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - return; - } - - qint64 size; - if (m_stream) - size = m_stream->bytesAvailable(); - else - size = m_buffer.size(); - - if (!m_dataRequestSize) - m_dataRequestSize = m_maxBytes; - size = qMin(size, (qint64)m_dataRequestSize); - qCDebug(qLcAppSrc) << " reading" << size << "bytes" << size << m_dataRequestSize; - - GstBuffer* buffer = gst_buffer_new_and_alloc(size); - - if (m_sequential || !m_stream) - buffer->offset = bytesReadSoFar; - else - buffer->offset = m_stream->pos(); - - if (m_format.isValid()) { - // timestamp raw audio data - uint nSamples = size/m_format.bytesPerFrame(); - - GST_BUFFER_TIMESTAMP(buffer) = gst_util_uint64_scale(streamedSamples, GST_SECOND, m_format.sampleRate()); - GST_BUFFER_DURATION(buffer) = gst_util_uint64_scale(nSamples, GST_SECOND, m_format.sampleRate()); - streamedSamples += nSamples; - } - - GstMapInfo mapInfo; - gst_buffer_map(buffer, &mapInfo, GST_MAP_WRITE); - void* bufferData = mapInfo.data; - - qint64 bytesRead; - if (m_stream) - bytesRead = m_stream->read((char*)bufferData, size); - else - bytesRead = m_buffer.read((char*)bufferData, size); - buffer->offset_end = buffer->offset + bytesRead - 1; - bytesReadSoFar += bytesRead; - - gst_buffer_unmap(buffer, &mapInfo); - qCDebug(qLcAppSrc) << "pushing bytes into gstreamer" << buffer->offset << bytesRead; - if (bytesRead == 0) { - gst_buffer_unref(buffer); - eosOrIdle(); - qCDebug(qLcAppSrc) << "end pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - return; - } - m_noMoreData = false; - emit bytesProcessed(bytesRead); - - GstFlowReturn ret = gst_app_src_push_buffer(GST_APP_SRC(m_appSrc.element()), buffer); - if (ret == GST_FLOW_ERROR) { - qWarning() << "QGstAppSrc: push buffer error"; - } else if (ret == GST_FLOW_FLUSHING) { - qWarning() << "QGstAppSrc: push buffer wrong state"; - } - qCDebug(qLcAppSrc) << "end pushData" << (m_stream ? m_stream : nullptr) << m_buffer.size(); - -} - -bool QGstAppSrc::doSeek(qint64 value) -{ - if (isStreamValid()) - return m_stream->seek(value + m_offset); - return false; -} - - -gboolean QGstAppSrc::on_seek_data(GstAppSrc *, guint64 arg0, gpointer userdata) -{ - // we do get some spurious seeks to INT_MAX, ignore those - if (arg0 == std::numeric_limits<quint64>::max()) - return true; - - QGstAppSrc *self = reinterpret_cast<QGstAppSrc*>(userdata); - Q_ASSERT(self); - if (self->m_sequential) - return false; - - QMetaObject::invokeMethod(self, "doSeek", Qt::AutoConnection, Q_ARG(qint64, arg0)); - return true; -} - -void QGstAppSrc::on_enough_data(GstAppSrc *, gpointer userdata) -{ - qCDebug(qLcAppSrc) << "on_enough_data"; - QGstAppSrc *self = static_cast<QGstAppSrc*>(userdata); - Q_ASSERT(self); - self->m_dataRequestSize = 0; -} - -void QGstAppSrc::on_need_data(GstAppSrc *, guint arg0, gpointer userdata) -{ - qCDebug(qLcAppSrc) << "on_need_data requesting bytes" << arg0; - QGstAppSrc *self = static_cast<QGstAppSrc*>(userdata); - Q_ASSERT(self); - self->m_dataRequestSize = arg0; - QMetaObject::invokeMethod(self, "pushData", Qt::AutoConnection); - qCDebug(qLcAppSrc) << "done on_need_data"; -} - -void QGstAppSrc::sendEOS() -{ - qCDebug(qLcAppSrc) << "sending EOS"; - if (m_appSrc.isNull()) - return; - - gst_app_src_end_of_stream(GST_APP_SRC(m_appSrc.element())); -} - -void QGstAppSrc::eosOrIdle() -{ - qCDebug(qLcAppSrc) << "eosOrIdle"; - if (m_appSrc.isNull()) - return; - - if (!m_sequential) { - sendEOS(); - return; - } - if (m_noMoreData) - return; - qCDebug(qLcAppSrc) << " idle!"; - m_noMoreData = true; - emit noMoreData(); -} diff --git a/src/plugins/multimedia/gstreamer/common/qgstappsrc_p.h b/src/plugins/multimedia/gstreamer/common/qgstappsrc_p.h deleted file mode 100644 index 373dbea65..000000000 --- a/src/plugins/multimedia/gstreamer/common/qgstappsrc_p.h +++ /dev/null @@ -1,132 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QGSTAPPSRC_H -#define QGSTAPPSRC_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <private/qtmultimediaglobal_p.h> -#include <qaudioformat.h> - -#include <QtCore/qobject.h> -#include <QtCore/qiodevice.h> -#include <QtCore/private/qringbuffer_p.h> -#include <QtCore/qatomic.h> - -#include <qgst_p.h> -#include <gst/app/gstappsrc.h> - -QT_BEGIN_NAMESPACE - -class QNetworkReply; - -class Q_MULTIMEDIA_EXPORT QGstAppSrc : public QObject -{ - Q_OBJECT -public: - QGstAppSrc(QObject *parent = 0); - ~QGstAppSrc(); - - bool setup(QIODevice *stream = nullptr, qint64 offset = 0); - void setAudioFormat(const QAudioFormat &f); - - void setExternalAppSrc(const QGstElement &appsrc); - QGstElement element(); - - void write(const char *data, qsizetype size); - - bool canAcceptMoreData() { return m_noMoreData || m_dataRequestSize != 0; } - - void suspend() { m_suspended = true; } - void resume() { m_suspended = false; m_noMoreData = true; } - -Q_SIGNALS: - void bytesProcessed(int bytes); - void noMoreData(); - -private Q_SLOTS: - void pushData(); - bool doSeek(qint64); - void onDataReady(); - - void streamDestroyed(); -private: - bool setStream(QIODevice *, qint64 offset); - bool isStreamValid() const - { - return m_stream != nullptr && m_stream->isOpen(); - } - - static gboolean on_seek_data(GstAppSrc *element, guint64 arg0, gpointer userdata); - static void on_enough_data(GstAppSrc *element, gpointer userdata); - static void on_need_data(GstAppSrc *element, uint arg0, gpointer userdata); - - void sendEOS(); - void eosOrIdle(); - - QIODevice *m_stream = nullptr; - QNetworkReply *m_networkReply = nullptr; - QRingBuffer m_buffer; - QAudioFormat m_format; - - QGstElement m_appSrc; - bool m_sequential = true; - bool m_suspended = false; - bool m_noMoreData = false; - GstAppStreamType m_streamType = GST_APP_STREAM_TYPE_RANDOM_ACCESS; - qint64 m_offset = 0; - qint64 m_maxBytes = 0; - qint64 bytesReadSoFar = 0; - QAtomicInteger<unsigned int> m_dataRequestSize = 0; - int streamedSamples = 0; -}; - -QT_END_NAMESPACE - -#endif diff --git a/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp b/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp index 1f20fe4ec..a2f8e91c2 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstpipeline.cpp @@ -1,94 +1,73 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -#include <QtCore/qmap.h> -#include <QtCore/qtimer.h> -#include <QtCore/qmutex.h> -#include <QtCore/qlist.h> #include <QtCore/qabstracteventdispatcher.h> #include <QtCore/qcoreapplication.h> +#include <QtCore/qlist.h> +#include <QtCore/qloggingcategory.h> +#include <QtCore/qmap.h> +#include <QtCore/qmutex.h> #include <QtCore/qproperty.h> +#include <QtCore/qtimer.h> #include "qgstpipeline_p.h" #include "qgstreamermessage_p.h" QT_BEGIN_NAMESPACE -class QGstPipelinePrivate : public QObject +static Q_LOGGING_CATEGORY(qLcGstPipeline, "qt.multimedia.gstpipeline"); + +static constexpr GstSeekFlags rateChangeSeekFlags = +#if GST_CHECK_VERSION(1, 18, 0) + GST_SEEK_FLAG_INSTANT_RATE_CHANGE; +#else + GST_SEEK_FLAG_FLUSH; +#endif + +class QGstPipelinePrivate { - Q_OBJECT public: - - int m_ref = 0; - guint m_tag = 0; + guint m_eventSourceID = 0; GstBus *m_bus = nullptr; - QTimer *m_intervalTimer = nullptr; + std::unique_ptr<QTimer> m_intervalTimer; QMutex filterMutex; QList<QGstreamerSyncMessageFilter*> syncFilters; QList<QGstreamerBusMessageFilter*> busFilters; - bool inStoppedState = true; - mutable qint64 m_position = 0; + mutable std::chrono::nanoseconds m_position{}; double m_rate = 1.; - bool m_flushOnConfigChanges = false; - bool m_pendingFlush = false; int m_configCounter = 0; GstState m_savedState = GST_STATE_NULL; - QGstPipelinePrivate(GstBus* bus, QObject* parent = 0); + explicit QGstPipelinePrivate(GstBus *bus); ~QGstPipelinePrivate(); - void ref() { ++ m_ref; } - void deref() { if (!--m_ref) delete this; } - void installMessageFilter(QGstreamerSyncMessageFilter *filter); void removeMessageFilter(QGstreamerSyncMessageFilter *filter); void installMessageFilter(QGstreamerBusMessageFilter *filter); void removeMessageFilter(QGstreamerBusMessageFilter *filter); - static GstBusSyncReply syncGstBusFilter(GstBus* bus, GstMessage* message, QGstPipelinePrivate *d) + void processMessage(const QGstreamerMessage &msg) { + for (QGstreamerBusMessageFilter *filter : std::as_const(busFilters)) { + if (filter->processBusMessage(msg)) + break; + } + } + +private: + static GstBusSyncReply syncGstBusFilter(GstBus *bus, GstMessage *message, + QGstPipelinePrivate *d) + { + if (!message) + return GST_BUS_PASS; + Q_UNUSED(bus); QMutexLocker lock(&d->filterMutex); - for (QGstreamerSyncMessageFilter *filter : qAsConst(d->syncFilters)) { - if (filter->processSyncMessage(QGstreamerMessage(message))) { + for (QGstreamerSyncMessageFilter *filter : std::as_const(d->syncFilters)) { + if (filter->processSyncMessage( + QGstreamerMessage{ message, QGstreamerMessage::NeedsRef })) { gst_message_unref(message); return GST_BUS_DROP; } @@ -97,59 +76,45 @@ public: return GST_BUS_PASS; } -private Q_SLOTS: - void interval() - { - GstMessage* message; - while ((message = gst_bus_poll(m_bus, GST_MESSAGE_ANY, 0)) != nullptr) { - processMessage(message); - gst_message_unref(message); - } - } - void doProcessMessage(const QGstreamerMessage& msg) + void processMessage(GstMessage *message) { - for (QGstreamerBusMessageFilter *filter : qAsConst(busFilters)) { - if (filter->processBusMessage(msg)) - break; - } - } + if (!message) + return; -private: - void processMessage(GstMessage* message) - { - QGstreamerMessage msg(message); - doProcessMessage(msg); - } + QGstreamerMessage msg{ + message, + QGstreamerMessage::NeedsRef, + }; - void queueMessage(GstMessage* message) - { - QGstreamerMessage msg(message); - QMetaObject::invokeMethod(this, "doProcessMessage", Qt::QueuedConnection, - Q_ARG(QGstreamerMessage, msg)); + processMessage(msg); } - static gboolean busCallback(GstBus *bus, GstMessage *message, gpointer data) + static gboolean busCallback(GstBus *, GstMessage *message, gpointer data) { - Q_UNUSED(bus); - static_cast<QGstPipelinePrivate *>(data)->queueMessage(message); + static_cast<QGstPipelinePrivate *>(data)->processMessage(message); return TRUE; } }; -QGstPipelinePrivate::QGstPipelinePrivate(GstBus* bus, QObject* parent) - : QObject(parent), - m_bus(bus) +QGstPipelinePrivate::QGstPipelinePrivate(GstBus *bus) : m_bus(bus) { // glib event loop can be disabled either by env variable or QT_NO_GLIB define, so check the dispacher QAbstractEventDispatcher *dispatcher = QCoreApplication::eventDispatcher(); const bool hasGlib = dispatcher && dispatcher->inherits("QEventDispatcherGlib"); if (!hasGlib) { - m_intervalTimer = new QTimer(this); + m_intervalTimer = std::make_unique<QTimer>(); m_intervalTimer->setInterval(250); - connect(m_intervalTimer, SIGNAL(timeout()), SLOT(interval())); + QObject::connect(m_intervalTimer.get(), &QTimer::timeout, m_intervalTimer.get(), [this] { + GstMessage *message; + while ((message = gst_bus_poll(m_bus, GST_MESSAGE_ANY, 0)) != nullptr) { + processMessage(message); + gst_message_unref(message); + } + }); m_intervalTimer->start(); } else { - m_tag = gst_bus_add_watch_full(bus, G_PRIORITY_DEFAULT, busCallback, this, nullptr); + m_eventSourceID = + gst_bus_add_watch_full(bus, G_PRIORITY_DEFAULT, busCallback, this, nullptr); } gst_bus_set_sync_handler(bus, (GstBusSyncHandler)syncGstBusFilter, this, nullptr); @@ -157,9 +122,9 @@ QGstPipelinePrivate::QGstPipelinePrivate(GstBus* bus, QObject* parent) QGstPipelinePrivate::~QGstPipelinePrivate() { - delete m_intervalTimer; + m_intervalTimer.reset(); - if (m_tag) + if (m_eventSourceID) gst_bus_remove_watch(m_bus); gst_bus_set_sync_handler(m_bus, nullptr, nullptr, nullptr); @@ -195,125 +160,121 @@ void QGstPipelinePrivate::removeMessageFilter(QGstreamerBusMessageFilter *filter busFilters.removeAll(filter); } -QGstPipeline::QGstPipeline(const QGstPipeline &o) - : QGstBin(o.bin(), NeedsRef), - d(o.d) -{ - if (d) - d->ref(); -} - -QGstPipeline &QGstPipeline::operator=(const QGstPipeline &o) -{ - if (this == &o) - return *this; - if (o.d) - o.d->ref(); - if (d) - d->deref(); - QGstBin::operator=(o); - d = o.d; - return *this; -} - -QGstPipeline::QGstPipeline(const char *name) - : QGstBin(GST_BIN(gst_pipeline_new(name)), NeedsRef) -{ - d = new QGstPipelinePrivate(gst_pipeline_get_bus(pipeline())); - d->ref(); -} - -QGstPipeline::QGstPipeline(GstPipeline *p) - : QGstBin(&p->bin, NeedsRef) -{ - d = new QGstPipelinePrivate(gst_pipeline_get_bus(pipeline())); - d->ref(); -} - -QGstPipeline::~QGstPipeline() +QGstPipeline QGstPipeline::create(const char *name) { - if (d) - d->deref(); + GstPipeline *pipeline = qGstCheckedCast<GstPipeline>(gst_pipeline_new(name)); + return adopt(pipeline); } -bool QGstPipeline::inStoppedState() const +QGstPipeline QGstPipeline::adopt(GstPipeline *pipeline) { - Q_ASSERT(d); - return d->inStoppedState; + QGstPipelinePrivate *d = new QGstPipelinePrivate(gst_pipeline_get_bus(pipeline)); + g_object_set_data_full(qGstCheckedCast<GObject>(pipeline), "pipeline-private", d, + [](gpointer ptr) { + delete reinterpret_cast<QGstPipelinePrivate *>(ptr); + return; + }); + + return QGstPipeline{ + pipeline, + QGstPipeline::NeedsRef, + }; } -void QGstPipeline::setInStoppedState(bool stopped) +QGstPipeline::QGstPipeline(GstPipeline *p, RefMode mode) : QGstBin(qGstCheckedCast<GstBin>(p), mode) { - Q_ASSERT(d); - d->inStoppedState = stopped; } -void QGstPipeline::setFlushOnConfigChanges(bool flush) -{ - d->m_flushOnConfigChanges = flush; -} +QGstPipeline::~QGstPipeline() = default; void QGstPipeline::installMessageFilter(QGstreamerSyncMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->installMessageFilter(filter); } void QGstPipeline::removeMessageFilter(QGstreamerSyncMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->removeMessageFilter(filter); } void QGstPipeline::installMessageFilter(QGstreamerBusMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->installMessageFilter(filter); } void QGstPipeline::removeMessageFilter(QGstreamerBusMessageFilter *filter) { - Q_ASSERT(d); + QGstPipelinePrivate *d = getPrivate(); d->removeMessageFilter(filter); } GstStateChangeReturn QGstPipeline::setState(GstState state) { - auto retval = gst_element_set_state(element(), state); - if (d->m_pendingFlush) { - d->m_pendingFlush = false; - flush(); - } - return retval; + return gst_element_set_state(element(), state); +} + +void QGstPipeline::processMessages(GstMessageType types) +{ + QGstPipelinePrivate *d = getPrivate(); + QGstreamerMessage message{ + gst_bus_pop_filtered(d->m_bus, types), + QGstreamerMessage::HasRef, + }; + d->processMessage(message); } void QGstPipeline::beginConfig() { - if (!d) - return; + QGstPipelinePrivate *d = getPrivate(); Q_ASSERT(!isNull()); ++d->m_configCounter; if (d->m_configCounter > 1) return; - d->m_savedState = state(); + GstState state; + GstState pending; + GstStateChangeReturn stateChangeReturn = gst_element_get_state(element(), &state, &pending, 0); + switch (stateChangeReturn) { + case GST_STATE_CHANGE_ASYNC: { + if (state == GST_STATE_PLAYING) { + // playing->paused transition in progress. wait for it to finish + bool stateChangeSuccessful = this->finishStateChange(); + if (!stateChangeSuccessful) + qWarning() << "QGstPipeline::beginConfig: timeout when waiting for state change"; + } + + state = pending; + break; + } + case GST_STATE_CHANGE_FAILURE: { + qDebug() << "QGstPipeline::beginConfig: state change failure"; + dumpGraph("beginConfigFailure"); + break; + } + + case GST_STATE_CHANGE_NO_PREROLL: + case GST_STATE_CHANGE_SUCCESS: + break; + } + + d->m_savedState = state; if (d->m_savedState == GST_STATE_PLAYING) setStateSync(GST_STATE_PAUSED); } void QGstPipeline::endConfig() { - if (!d) - return; + QGstPipelinePrivate *d = getPrivate(); Q_ASSERT(!isNull()); --d->m_configCounter; if (d->m_configCounter) return; - if (d->m_flushOnConfigChanges) - d->m_pendingFlush = true; if (d->m_savedState == GST_STATE_PLAYING) setState(GST_STATE_PLAYING); d->m_savedState = GST_STATE_NULL; @@ -321,60 +282,108 @@ void QGstPipeline::endConfig() void QGstPipeline::flush() { - seek(position(), d->m_rate); + seek(position()); } -bool QGstPipeline::seek(qint64 pos, double rate) +void QGstPipeline::seek(std::chrono::nanoseconds pos, double rate) { - // always adjust the rate, so it can be set before playback starts + using namespace std::chrono_literals; + + QGstPipelinePrivate *d = getPrivate(); + // always adjust the rate, so it can be set before playback starts // setting position needs a loaded media file that's seekable - d->m_rate = rate; - bool success = gst_element_seek(element(), rate, GST_FORMAT_TIME, - GstSeekFlags(GST_SEEK_FLAG_FLUSH), - GST_SEEK_TYPE_SET, pos, - GST_SEEK_TYPE_SET, -1); - if (!success) - return false; + + qCDebug(qLcGstPipeline) << "QGstPipeline::seek to" << pos << "rate:" << rate; + + bool success = (rate > 0) + ? gst_element_seek(element(), rate, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, pos.count(), GST_SEEK_TYPE_END, 0) + : gst_element_seek(element(), rate, GST_FORMAT_TIME, GST_SEEK_FLAG_FLUSH, + GST_SEEK_TYPE_SET, 0, GST_SEEK_TYPE_SET, pos.count()); + + if (!success) { + qDebug() << "seek: gst_element_seek failed" << pos; + return; + } d->m_position = pos; - return true; } -bool QGstPipeline::setPlaybackRate(double rate) +void QGstPipeline::seek(std::chrono::nanoseconds pos) +{ + qCDebug(qLcGstPipeline) << "QGstPipeline::seek to" << pos; + seek(pos, getPrivate()->m_rate); +} + +void QGstPipeline::setPlaybackRate(double rate) { + QGstPipelinePrivate *d = getPrivate(); if (rate == d->m_rate) - return false; - seek(position(), rate); - return true; + return; + + d->m_rate = rate; + + qCDebug(qLcGstPipeline) << "QGstPipeline::setPlaybackRate to" << rate; + + applyPlaybackRate(/*instantRateChange =*/true); } double QGstPipeline::playbackRate() const { + QGstPipelinePrivate *d = getPrivate(); return d->m_rate; } -bool QGstPipeline::setPosition(qint64 pos) +void QGstPipeline::applyPlaybackRate(bool instantRateChange) +{ + QGstPipelinePrivate *d = getPrivate(); + + // do not GST_SEEK_FLAG_FLUSH with GST_SEEK_TYPE_NONE + // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3604 + if (instantRateChange && GST_CHECK_VERSION(1, 18, 0)) { + qCDebug(qLcGstPipeline) << "QGstPipeline::applyPlaybackRate instantly"; + bool success = gst_element_seek( + element(), d->m_rate, GST_FORMAT_UNDEFINED, rateChangeSeekFlags, GST_SEEK_TYPE_NONE, + GST_CLOCK_TIME_NONE, GST_SEEK_TYPE_NONE, GST_CLOCK_TIME_NONE); + if (!success) + qDebug() << "setPlaybackRate: gst_element_seek failed"; + } else { + seek(position(), d->m_rate); + } +} + +void QGstPipeline::setPosition(std::chrono::nanoseconds pos) { - return seek(pos, d->m_rate); + seek(pos); } -qint64 QGstPipeline::position() const +std::chrono::nanoseconds QGstPipeline::position() const { - gint64 pos; - if (gst_element_query_position(element(), GST_FORMAT_TIME, &pos)) - d->m_position = pos; + QGstPipelinePrivate *d = getPrivate(); + std::optional<std::chrono::nanoseconds> pos = QGstElement::position(); + if (pos) { + d->m_position = *pos; + qCDebug(qLcGstPipeline) << "QGstPipeline::position:" + << std::chrono::round<std::chrono::milliseconds>(*pos); + } else { + qDebug() << "QGstPipeline: failed to query position, using previous position"; + } + return d->m_position; } -qint64 QGstPipeline::duration() const +std::chrono::milliseconds QGstPipeline::positionInMs() const { - gint64 d; - if (!gst_element_query_duration(element(), GST_FORMAT_TIME, &d)) - return 0.; + using namespace std::chrono; + return round<milliseconds>(position()); +} + +QGstPipelinePrivate *QGstPipeline::getPrivate() const +{ + gpointer p = g_object_get_data(qGstCheckedCast<GObject>(object()), "pipeline-private"); + auto *d = reinterpret_cast<QGstPipelinePrivate *>(p); + Q_ASSERT(d); return d; } QT_END_NAMESPACE - -#include "qgstpipeline.moc" - diff --git a/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h b/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h index d1f585cc2..bd711e553 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstpipeline_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef qgstpipeline_p_H #define qgstpipeline_p_H @@ -51,10 +15,10 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <QObject> +#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtCore/qobject.h> -#include <qgst_p.h> +#include "qgst_p.h" QT_BEGIN_NAMESPACE @@ -77,24 +41,18 @@ class QGstPipelinePrivate; class QGstPipeline : public QGstBin { - QGstPipelinePrivate *d = nullptr; public: constexpr QGstPipeline() = default; - QGstPipeline(const QGstPipeline &o); - QGstPipeline &operator=(const QGstPipeline &o); - QGstPipeline(const char *name); - QGstPipeline(GstPipeline *p); - ~QGstPipeline() override; - - // This is needed to help us avoid sending QVideoFrames or audio buffers to the - // application while we're prerolling the pipeline. - // QMediaPlayer is still in a stopped state, while we put the gstreamer pipeline into a - // Paused state so that we can get the required metadata of the stream and also have a fast - // transition to play. - bool inStoppedState() const; - void setInStoppedState(bool stopped); - - void setFlushOnConfigChanges(bool flush); + QGstPipeline(const QGstPipeline &) = default; + QGstPipeline(QGstPipeline &&) = default; + QGstPipeline &operator=(const QGstPipeline &) = default; + QGstPipeline &operator=(QGstPipeline &&) noexcept = default; + QGstPipeline(GstPipeline *, RefMode mode); + ~QGstPipeline(); + + // installs QGstPipelinePrivate as "pipeline-private" gobject property + static QGstPipeline create(const char *name); + static QGstPipeline adopt(GstPipeline *); void installMessageFilter(QGstreamerSyncMessageFilter *filter); void removeMessageFilter(QGstreamerSyncMessageFilter *filter); @@ -103,36 +61,45 @@ public: GstStateChangeReturn setState(GstState state); - GstPipeline *pipeline() const { return GST_PIPELINE_CAST(m_object); } + GstPipeline *pipeline() const { return GST_PIPELINE_CAST(get()); } + + void processMessages(GstMessageType = GST_MESSAGE_ANY); - void dumpGraph(const char *fileName) + template <typename Functor> + void modifyPipelineWhileNotRunning(Functor &&fn) { - if (isNull()) - return; - -#if 1 //def QT_GST_CAPTURE_DEBUG - GST_DEBUG_BIN_TO_DOT_FILE(bin(), - GstDebugGraphDetails(GST_DEBUG_GRAPH_SHOW_ALL | - GST_DEBUG_GRAPH_SHOW_MEDIA_TYPE | GST_DEBUG_GRAPH_SHOW_NON_DEFAULT_PARAMS | GST_DEBUG_GRAPH_SHOW_STATES), - fileName); -#else - Q_UNUSED(fileName); -#endif + beginConfig(); + fn(); + endConfig(); } - void beginConfig(); - void endConfig(); + template <typename Functor> + static void modifyPipelineWhileNotRunning(QGstPipeline &&pipeline, Functor &&fn) + { + if (pipeline) + pipeline.modifyPipelineWhileNotRunning(fn); + else + fn(); + } void flush(); - bool seek(qint64 pos, double rate); - bool setPlaybackRate(double rate); + void setPlaybackRate(double rate); double playbackRate() const; + void applyPlaybackRate(bool instantRateChange); + + void setPosition(std::chrono::nanoseconds pos); + std::chrono::nanoseconds position() const; + std::chrono::milliseconds positionInMs() const; + +private: + void seek(std::chrono::nanoseconds pos, double rate); + void seek(std::chrono::nanoseconds pos); - bool setPosition(qint64 pos); - qint64 position() const; + QGstPipelinePrivate *getPrivate() const; - qint64 duration() const; + void beginConfig(); + void endConfig(); }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp index 402b3c825..a2f60eaa1 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput.cpp @@ -1,148 +1,156 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreameraudioinput_p.h> -#include <qgstreameraudiodevice_p.h> -#include <qaudiodevice.h> -#include <qaudioinput.h> +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreameraudioinput_p.h> #include <QtCore/qloggingcategory.h> -#include <QtNetwork/qnetworkaccessmanager.h> -#include <QtNetwork/qnetworkreply.h> +#include <QtMultimedia/qaudiodevice.h> +#include <QtMultimedia/qaudioinput.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> +#include <audio/qgstreameraudiodevice_p.h> +#include <common/qgstpipeline_p.h> -Q_LOGGING_CATEGORY(qLcMediaAudioInput, "qt.multimedia.audioInput") QT_BEGIN_NAMESPACE -QGstreamerAudioInput::QGstreamerAudioInput(QAudioInput *parent) - : QObject(parent), - QPlatformAudioInput(parent), - gstAudioInput("audioInput") +namespace { + +Q_LOGGING_CATEGORY(qLcMediaAudioInput, "qt.multimedia.audioinput") + +constexpr QLatin1String defaultSrcName = [] { + using namespace Qt::Literals; + + if constexpr (QT_CONFIG(pulseaudio)) + return "pulsesrc"_L1; + else if constexpr (QT_CONFIG(alsa)) + return "alsasrc"_L1; + else + return "autoaudiosrc"_L1; +}(); + +bool hasDeviceProperty(const QGstElement &element) { - audioSrc = QGstElement("autoaudiosrc", "autoaudiosrc"); - audioVolume = QGstElement("volume", "volume"); - gstAudioInput.add(audioSrc, audioVolume); - audioSrc.link(audioVolume); + using namespace Qt::Literals; + QLatin1String elementType = element.typeName(); + + if constexpr (QT_CONFIG(pulseaudio)) + return elementType == "GstPulseSrc"_L1; + + if constexpr (0 && QT_CONFIG(alsa)) // alsasrc has a "device" property, but it cannot be changed + // during playback + return elementType == "GstAlsaSrc"_L1; - gstAudioInput.addGhostPad(audioVolume, "src"); + return false; } -QGstreamerAudioInput::~QGstreamerAudioInput() +} // namespace + +QMaybe<QPlatformAudioInput *> QGstreamerAudioInput::create(QAudioInput *parent) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable("autoaudiosrc", "volume"); + if (error) + return *error; + + return new QGstreamerAudioInput(parent); +} + +QGstreamerAudioInput::QGstreamerAudioInput(QAudioInput *parent) + : QObject(parent), + QPlatformAudioInput(parent), + m_audioInputBin(QGstBin::create("audioInput")), + m_audioSrc{ + QGstElement::createFromFactory(defaultSrcName.constData(), "autoaudiosrc"), + }, + m_audioVolume{ + QGstElement::createFromFactory("volume", "volume"), + } { - gstAudioInput.setStateSync(GST_STATE_NULL); + m_audioInputBin.add(m_audioSrc, m_audioVolume); + qLinkGstElements(m_audioSrc, m_audioVolume); + + m_audioInputBin.addGhostPad(m_audioVolume, "src"); } -int QGstreamerAudioInput::volume() const +QGstElement QGstreamerAudioInput::createGstElement() { - return m_volume; + const auto *customDeviceInfo = + dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(m_audioDevice.handle()); + + if (customDeviceInfo) { + qCDebug(qLcMediaAudioInput) + << "requesting custom audio src element: " << customDeviceInfo->id; + + QGstElement element = QGstBin::createFromPipelineDescription(customDeviceInfo->id, + /*name=*/nullptr, + /*ghostUnlinkedPads=*/true); + if (element) + return element; + + qCWarning(qLcMediaAudioInput) + << "Cannot create audio source element:" << customDeviceInfo->id; + } + + const QByteArray &id = m_audioDevice.id(); + if constexpr (QT_CONFIG(pulseaudio) || QT_CONFIG(alsa)) { + QGstElement newSrc = QGstElement::createFromFactory(defaultSrcName.constData(), "audiosrc"); + if (newSrc) { + newSrc.set("device", id.constData()); + return newSrc; + } + + qWarning() << "Cannot create" << defaultSrcName; + + } else { + auto *deviceInfo = dynamic_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); + if (deviceInfo && deviceInfo->gstDevice) { + QGstElement element = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosrc"); + if (element) + return element; + } + } + qCWarning(qLcMediaAudioInput) << "Invalid audio device"; + qCWarning(qLcMediaAudioInput) + << "Failed to create a gst element for the audio device, using a default audio source"; + return QGstElement::createFromFactory("autoaudiosrc", "audiosrc"); } -bool QGstreamerAudioInput::isMuted() const +QGstreamerAudioInput::~QGstreamerAudioInput() { - return m_muted; + m_audioInputBin.setStateSync(GST_STATE_NULL); } -void QGstreamerAudioInput::setVolume(float vol) +void QGstreamerAudioInput::setVolume(float volume) { - if (vol == m_volume) - return; - m_volume = vol; - audioVolume.set("volume", vol); - emit volumeChanged(m_volume); + m_audioVolume.set("volume", volume); } void QGstreamerAudioInput::setMuted(bool muted) { - if (muted == m_muted) - return; - m_muted = muted; - audioVolume.set("mute", muted); - emit mutedChanged(muted); + m_audioVolume.set("mute", muted); } void QGstreamerAudioInput::setAudioDevice(const QAudioDevice &device) { if (device == m_audioDevice) return; - qCDebug(qLcMediaAudioInput) << "setAudioInput" << device.description() << device.isNull(); + qCDebug(qLcMediaAudioInput) << "setAudioDevice" << device.description() << device.isNull(); m_audioDevice = device; - QGstElement newSrc; -#if QT_CONFIG(pulseaudio) - auto id = m_audioDevice.id(); - newSrc = QGstElement("pulsesrc", "audiosrc"); - if (!newSrc.isNull()) - newSrc.set("device", id.constData()); - else - qCWarning(qLcMediaAudioInput) << "Invalid audio device"; -#else - auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); - if (deviceInfo && deviceInfo->gstDevice) - newSrc = gst_device_create_element(deviceInfo->gstDevice, "audiosrc"); - else - qCWarning(qLcMediaAudioInput) << "Invalid audio device"; -#endif - - if (newSrc.isNull()) { - qCWarning(qLcMediaAudioInput) << "Failed to create a gst element for the audio device, using a default audio source"; - newSrc = QGstElement("autoaudiosrc", "audiosrc"); + if (hasDeviceProperty(m_audioSrc) && !isCustomAudioDevice(m_audioDevice)) { + m_audioSrc.set("device", m_audioDevice.id().constData()); + return; } - // FIXME: most probably source can be disconnected outside of idle probe - audioSrc.staticPad("src").doInIdleProbe([&](){ - audioSrc.unlink(audioVolume); - }); - audioSrc.setStateSync(GST_STATE_NULL); - gstAudioInput.remove(audioSrc); - audioSrc = newSrc; - gstAudioInput.add(audioSrc); - audioSrc.link(audioVolume); - audioSrc.syncStateWithParent(); -} + QGstElement newSrc = createGstElement(); -QAudioDevice QGstreamerAudioInput::audioInput() const -{ - return m_audioDevice; + QGstPipeline::modifyPipelineWhileNotRunning(m_audioInputBin.getPipeline(), [&] { + qUnlinkGstElements(m_audioSrc, m_audioVolume); + m_audioInputBin.stopAndRemoveElements(m_audioSrc); + m_audioSrc = std::move(newSrc); + m_audioInputBin.add(m_audioSrc); + qLinkGstElements(m_audioSrc, m_audioVolume); + m_audioSrc.syncStateWithParent(); + }); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h index 3b4ad3475..4b01b53a6 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudioinput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIOINPUT_P_H #define QGSTREAMERAUDIOINPUT_P_H @@ -51,55 +15,39 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <qaudiodevice.h> - #include <QtCore/qobject.h> +#include <QtMultimedia/private/qplatformaudioinput_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> -#include <private/qplatformaudioinput_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE -class QGstreamerMessage; class QAudioDevice; -class Q_MULTIMEDIA_EXPORT QGstreamerAudioInput : public QObject, public QPlatformAudioInput +class QGstreamerAudioInput : public QObject, public QPlatformAudioInput { - Q_OBJECT - public: - QGstreamerAudioInput(QAudioInput *parent); + static QMaybe<QPlatformAudioInput *> create(QAudioInput *parent); ~QGstreamerAudioInput(); - int volume() const; - bool isMuted() const; - - bool setAudioInput(const QAudioDevice &); - QAudioDevice audioInput() const; - void setAudioDevice(const QAudioDevice &) override; - void setVolume(float volume) override; - void setMuted(bool muted) override; - - QGstElement gstElement() const { return gstAudioInput; } + void setVolume(float) override; + void setMuted(bool) override; -Q_SIGNALS: - void mutedChanged(bool); - void volumeChanged(int); + QGstElement gstElement() const { return m_audioInputBin; } private: - float m_volume = 1.; - bool m_muted = false; + explicit QGstreamerAudioInput(QAudioInput *parent); + + QGstElement createGstElement(); QAudioDevice m_audioDevice; // Gst elements - QGstBin gstAudioInput; + QGstBin m_audioInputBin; - QGstElement audioSrc; - QGstElement audioVolume; + QGstElement m_audioSrc; + QGstElement m_audioVolume; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp index 0c9f84fbc..1a8c6976c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput.cpp @@ -1,133 +1,170 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreameraudiooutput_p.h> -#include <qgstreameraudiodevice_p.h> -#include <qaudiodevice.h> -#include <qaudiooutput.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreameraudiooutput_p.h> #include <QtCore/qloggingcategory.h> -#include <QtNetwork/qnetworkaccessmanager.h> -#include <QtNetwork/qnetworkreply.h> +#include <QtMultimedia/qaudiodevice.h> +#include <QtMultimedia/qaudiooutput.h> -#include <sys/types.h> -#include <sys/stat.h> -#include <fcntl.h> +#include <common/qgstpipeline_p.h> +#include <audio/qgstreameraudiodevice_p.h> -Q_LOGGING_CATEGORY(qLcMediaAudioOutput, "qt.multimedia.audiooutput") QT_BEGIN_NAMESPACE +namespace { + +Q_LOGGING_CATEGORY(qLcMediaAudioOutput, "qt.multimedia.audiooutput") + +constexpr QLatin1String defaultSinkName = [] { + using namespace Qt::Literals; + + if constexpr (QT_CONFIG(pulseaudio)) + return "pulsesink"_L1; + else if constexpr (QT_CONFIG(alsa)) + return "alsasink"_L1; + else + return "autoaudiosink"_L1; +}(); + +bool hasDeviceProperty(const QGstElement &element) +{ + using namespace Qt::Literals; + QLatin1String elementType = element.typeName(); + + if constexpr (QT_CONFIG(pulseaudio)) + return elementType == "GstPulseSink"_L1; + if constexpr (0 && QT_CONFIG(alsa)) // alsasrc has a "device" property, but it cannot be changed + // during playback + return elementType == "GstAlsaSink"_L1; + + return false; +} + +} // namespace + +QMaybe<QPlatformAudioOutput *> QGstreamerAudioOutput::create(QAudioOutput *parent) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable( + "audioconvert", "audioresample", "volume", "autoaudiosink"); + if (error) + return *error; + + return new QGstreamerAudioOutput(parent); +} + QGstreamerAudioOutput::QGstreamerAudioOutput(QAudioOutput *parent) - : QObject(parent), - QPlatformAudioOutput(parent), - gstAudioOutput("audioOutput") + : QObject(parent), + QPlatformAudioOutput(parent), + m_audioOutputBin(QGstBin::create("audioOutput")), + m_audioQueue{ + QGstElement::createFromFactory("queue", "audioQueue"), + }, + m_audioConvert{ + QGstElement::createFromFactory("audioconvert", "audioConvert"), + }, + m_audioResample{ + QGstElement::createFromFactory("audioresample", "audioResample"), + }, + m_audioVolume{ + QGstElement::createFromFactory("volume", "volume"), + }, + m_audioSink{ + QGstElement::createFromFactory(defaultSinkName.constData(), "audiosink"), + } { - audioQueue = QGstElement("queue", "audioQueue"); - audioConvert = QGstElement("audioconvert", "audioConvert"); - audioResample = QGstElement("audioresample", "audioResample"); - audioVolume = QGstElement("volume", "volume"); - audioSink = QGstElement("autoaudiosink", "autoAudioSink"); - gstAudioOutput.add(audioQueue, audioConvert, audioResample, audioVolume, audioSink); - audioQueue.link(audioConvert, audioResample, audioVolume, audioSink); - - gstAudioOutput.addGhostPad(audioQueue, "sink"); + m_audioOutputBin.add(m_audioQueue, m_audioConvert, m_audioResample, m_audioVolume, m_audioSink); + qLinkGstElements(m_audioQueue, m_audioConvert, m_audioResample, m_audioVolume, m_audioSink); + + m_audioOutputBin.addGhostPad(m_audioQueue, "sink"); } -QGstreamerAudioOutput::~QGstreamerAudioOutput() +QGstElement QGstreamerAudioOutput::createGstElement() { - gstAudioOutput.setStateSync(GST_STATE_NULL); + const auto *customDeviceInfo = + dynamic_cast<const QGStreamerCustomAudioDeviceInfo *>(m_audioDevice.handle()); + + if (customDeviceInfo) { + qCDebug(qLcMediaAudioOutput) + << "requesting custom audio sink element: " << customDeviceInfo->id; + + QGstElement element = + QGstBin::createFromPipelineDescription(customDeviceInfo->id, /*name=*/nullptr, + /*ghostUnlinkedPads=*/true); + if (element) + return element; + + qCWarning(qLcMediaAudioOutput) + << "Cannot create audio sink element:" << customDeviceInfo->id; + } + + const QByteArray &id = m_audioDevice.id(); + if constexpr (QT_CONFIG(pulseaudio) || QT_CONFIG(alsa)) { + QGstElement newSink = + QGstElement::createFromFactory(defaultSinkName.constData(), "audiosink"); + if (newSink) { + newSink.set("device", id.constData()); + return newSink; + } + + qWarning() << "Cannot create" << defaultSinkName; + } else { + auto *deviceInfo = dynamic_cast<const QGStreamerAudioDeviceInfo *>(m_audioDevice.handle()); + if (deviceInfo && deviceInfo->gstDevice) { + QGstElement element = QGstElement::createFromDevice(deviceInfo->gstDevice, "audiosink"); + if (element) + return element; + } + } + qCWarning(qLcMediaAudioOutput) << "Invalid audio device:" << m_audioDevice.id(); + qCWarning(qLcMediaAudioOutput) + << "Failed to create a gst element for the audio device, using a default audio sink"; + return QGstElement::createFromFactory("autoaudiosink", "audiosink"); } -void QGstreamerAudioOutput::setVolume(float vol) +QGstreamerAudioOutput::~QGstreamerAudioOutput() { - audioVolume.set("volume", vol); + m_audioOutputBin.setStateSync(GST_STATE_NULL); } -void QGstreamerAudioOutput::setMuted(bool muted) +void QGstreamerAudioOutput::setVolume(float volume) { - audioVolume.set("mute", muted); + m_audioVolume.set("volume", volume); } -void QGstreamerAudioOutput::setPipeline(const QGstPipeline &pipeline) +void QGstreamerAudioOutput::setMuted(bool muted) { - gstPipeline = pipeline; + m_audioVolume.set("mute", muted); } -void QGstreamerAudioOutput::setAudioDevice(const QAudioDevice &info) +void QGstreamerAudioOutput::setAudioDevice(const QAudioDevice &device) { - if (info == m_audioOutput) + if (device == m_audioDevice) return; - qCDebug(qLcMediaAudioOutput) << "setAudioOutput" << info.description() << info.isNull(); - m_audioOutput = info; - - QGstElement newSink; -#if QT_CONFIG(pulseaudio) - auto id = m_audioOutput.id(); - newSink = QGstElement("pulsesink", "audiosink"); - if (!newSink.isNull()) - newSink.set("device", id.constData()); - else - qCWarning(qLcMediaAudioOutput) << "Invalid audio device"; -#else - auto *deviceInfo = static_cast<const QGStreamerAudioDeviceInfo *>(m_audioOutput.handle()); - if (deviceInfo && deviceInfo->gstDevice) - newSink = gst_device_create_element(deviceInfo->gstDevice , "audiosink"); - else - qCWarning(qLcMediaAudioOutput) << "Invalid audio device"; -#endif + qCDebug(qLcMediaAudioOutput) << "setAudioDevice" << device.description() << device.isNull(); + + m_audioDevice = device; - if (newSink.isNull()) { - qCWarning(qLcMediaAudioOutput) << "Failed to create a gst element for the audio device, using a default audio sink"; - newSink = QGstElement("autoaudiosink", "audiosink"); + if (hasDeviceProperty(m_audioSink) && !isCustomAudioDevice(m_audioDevice)) { + m_audioSink.set("device", m_audioDevice.id().constData()); + return; } - audioVolume.staticPad("src").doInIdleProbe([&](){ - audioVolume.unlink(audioSink); - gstAudioOutput.remove(audioSink); - gstAudioOutput.add(newSink); - newSink.syncStateWithParent(); - audioVolume.link(newSink); + QGstElement newSink = createGstElement(); + + QGstPipeline::modifyPipelineWhileNotRunning(m_audioOutputBin.getPipeline(), [&] { + qUnlinkGstElements(m_audioVolume, m_audioSink); + m_audioOutputBin.stopAndRemoveElements(m_audioSink); + m_audioSink = std::move(newSink); + m_audioOutputBin.add(m_audioSink); + m_audioSink.syncStateWithParent(); + qLinkGstElements(m_audioVolume, m_audioSink); }); - audioSink.setStateSync(GST_STATE_NULL); - audioSink = newSink; + // we need to flush the pipeline, otherwise, the new sink doesn't always reach the new state + if (m_audioOutputBin.getPipeline()) + m_audioOutputBin.getPipeline().flush(); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h index 318419247..da11c39d2 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreameraudiooutput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERAUDIOOUTPUT_P_H #define QGSTREAMERAUDIOOUTPUT_P_H @@ -51,52 +15,42 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <qaudiodevice.h> - #include <QtCore/qobject.h> +#include <QtMultimedia/private/qplatformaudiooutput_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> -#include <private/qplatformaudiooutput_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE -class QGstreamerMessage; class QAudioDevice; -class Q_MULTIMEDIA_EXPORT QGstreamerAudioOutput : public QObject, public QPlatformAudioOutput +class QGstreamerAudioOutput : public QObject, public QPlatformAudioOutput { - Q_OBJECT - public: - QGstreamerAudioOutput(QAudioOutput *parent); + static QMaybe<QPlatformAudioOutput *> create(QAudioOutput *parent); ~QGstreamerAudioOutput(); void setAudioDevice(const QAudioDevice &) override; - void setVolume(float volume) override; - void setMuted(bool muted) override; + void setVolume(float) override; + void setMuted(bool) override; - void setPipeline(const QGstPipeline &pipeline); + QGstElement gstElement() const { return m_audioOutputBin; } - QGstElement gstElement() const { return gstAudioOutput; } +private: + explicit QGstreamerAudioOutput(QAudioOutput *parent); -Q_SIGNALS: - void mutedChanged(bool); - void volumeChanged(int); + QGstElement createGstElement(); -private: - QAudioDevice m_audioOutput; + QAudioDevice m_audioDevice; // Gst elements - QGstPipeline gstPipeline; - QGstBin gstAudioOutput; - - QGstElement audioQueue; - QGstElement audioConvert; - QGstElement audioResample; - QGstElement audioVolume; - QGstElement audioSink; + QGstBin m_audioOutputBin; + + QGstElement m_audioQueue; + QGstElement m_audioConvert; + QGstElement m_audioResample; + QGstElement m_audioVolume; + QGstElement m_audioSink; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp index 3af8000bb..9cba810db 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe.cpp @@ -1,44 +1,9 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamerbufferprobe_p.h" -#include "qgstutils_p.h" +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreamerbufferprobe_p.h> + +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE @@ -51,10 +16,14 @@ QGstreamerBufferProbe::~QGstreamerBufferProbe() = default; void QGstreamerBufferProbe::addProbeToPad(GstPad *pad, bool downstream) { - if (GstCaps *caps = gst_pad_get_current_caps(pad)) { - probeCaps(caps); - gst_caps_unref(caps); - } + QGstCaps caps{ + gst_pad_get_current_caps(pad), + QGstCaps::HasRef, + }; + + if (caps) + probeCaps(caps.caps()); + if (m_flags & ProbeCaps) { m_capsProbeId = gst_pad_add_probe( pad, diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h index d3ca9db2e..71996a0cc 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamerbufferprobe_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERBUFFERPROBE_H #define QGSTREAMERBUFFERPROBE_H @@ -59,7 +23,7 @@ QT_BEGIN_NAMESPACE -class Q_MULTIMEDIA_EXPORT QGstreamerBufferProbe +class QGstreamerBufferProbe { public: enum Flags diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp index c8e730c3d..e4b3d7393 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer.cpp @@ -1,73 +1,41 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreamermediaplayer_p.h> -#include <qgstpipeline_p.h> -#include <qgstreamermetadata_p.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreamermediaplayer_p.h> + +#include <audio/qgstreameraudiodevice_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstappsource_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreameraudiooutput_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgstreamermetadata_p.h> +#include <common/qgstreamervideooutput_p.h> +#include <common/qgstreamervideosink_p.h> #include <qgstreamerformatinfo_p.h> -#include <qgstreameraudiooutput_p.h> -#include <qgstreamervideooutput_p.h> -#include <qgstreamervideosink_p.h> -#include "qgstreamermessage_p.h" -#include <qgstreameraudiodevice_p.h> -#include <qgstappsrc_p.h> -#include <qaudiodevice.h> +#include <QtMultimedia/qaudiodevice.h> #include <QtCore/qdir.h> #include <QtCore/qsocketnotifier.h> #include <QtCore/qurl.h> #include <QtCore/qdebug.h> #include <QtCore/qloggingcategory.h> -#include <QtNetwork/qnetworkaccessmanager.h> -#include <QtNetwork/qnetworkreply.h> +#include <QtCore/private/quniquehandle_p.h> #include <sys/types.h> #include <sys/stat.h> #include <fcntl.h> -Q_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player") +#if QT_CONFIG(gstreamer_gl) +# include <gst/gl/gl.h> +#endif + +static Q_LOGGING_CATEGORY(qLcMediaPlayer, "qt.multimedia.player") QT_BEGIN_NAMESPACE -QGstreamerMediaPlayer::TrackSelector::TrackSelector(TrackType type, const char *name) - : selector(QGstElement("input-selector", name)), - type(type) +QGstreamerMediaPlayer::TrackSelector::TrackSelector(TrackType type, QGstElement selector) + : selector(selector), type(type) { selector.set("sync-streams", true); selector.set("sync-mode", 1 /*clock*/); @@ -110,35 +78,87 @@ QGstreamerMediaPlayer::TrackSelector &QGstreamerMediaPlayer::trackSelector(Track return ts; } -QGstreamerMediaPlayer::QGstreamerMediaPlayer(QMediaPlayer *parent) +void QGstreamerMediaPlayer::mediaStatusChanged(QMediaPlayer::MediaStatus status) +{ + if (status != QMediaPlayer::StalledMedia) + m_stalledMediaNotifier.stop(); + + QPlatformMediaPlayer::mediaStatusChanged(status); +} + +void QGstreamerMediaPlayer::updateBufferProgress(float newProgress) +{ + if (qFuzzyIsNull(newProgress - m_bufferProgress)) + return; + + m_bufferProgress = newProgress; + bufferProgressChanged(m_bufferProgress); +} + +void QGstreamerMediaPlayer::disconnectDecoderHandlers() +{ + auto handlers = std::initializer_list<QGObjectHandlerScopedConnection *>{ + &padAdded, &padRemoved, &sourceSetup, &uridecodebinElementAdded, + &unknownType, &elementAdded, &elementRemoved, + }; + for (QGObjectHandlerScopedConnection *handler : handlers) + handler->disconnect(); + + decodeBinQueues = 0; +} + +QMaybe<QPlatformMediaPlayer *> QGstreamerMediaPlayer::create(QMediaPlayer *parent) +{ + auto videoOutput = QGstreamerVideoOutput::create(); + if (!videoOutput) + return videoOutput.error(); + + static const auto error = + qGstErrorMessageIfElementsNotAvailable("input-selector", "decodebin", "uridecodebin"); + if (error) + return *error; + + return new QGstreamerMediaPlayer(videoOutput.value(), parent); +} + +QGstreamerMediaPlayer::QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, + QMediaPlayer *parent) : QObject(parent), QPlatformMediaPlayer(parent), - trackSelectors{{{ VideoStream, "videoInputSelector" }, - { AudioStream, "audioInputSelector" }, - { SubtitleStream, "subTitleInputSelector" }}}, - playerPipeline("playerPipeline") + trackSelectors{ { + { VideoStream, + QGstElement::createFromFactory("input-selector", "videoInputSelector") }, + { AudioStream, + QGstElement::createFromFactory("input-selector", "audioInputSelector") }, + { SubtitleStream, + QGstElement::createFromFactory("input-selector", "subTitleInputSelector") }, + } }, + playerPipeline(QGstPipeline::create("playerPipeline")), + gstVideoOutput(videoOutput) { - playerPipeline.setFlushOnConfigChanges(true); - - gstVideoOutput = new QGstreamerVideoOutput(this); + gstVideoOutput->setParent(this); gstVideoOutput->setPipeline(playerPipeline); for (auto &ts : trackSelectors) playerPipeline.add(ts.selector); - playerPipeline.setState(GST_STATE_NULL); playerPipeline.installMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this)); playerPipeline.installMessageFilter(static_cast<QGstreamerSyncMessageFilter *>(this)); - gst_pipeline_use_clock(playerPipeline.pipeline(), gst_system_clock_obtain()); + QGstClockHandle systemClock{ + gst_system_clock_obtain(), + }; + + gst_pipeline_use_clock(playerPipeline.pipeline(), systemClock.get()); + + connect(&positionUpdateTimer, &QTimer::timeout, this, [this] { + updatePositionFromPipeline(); + }); - /* Taken from gstdicoverer.c: - * This is ugly. We get the GType of decodebin so we can quickly detect - * when a decodebin is added to uridecodebin so we can set the - * post-stream-topology setting to TRUE */ - auto decodebin = QGstElement("decodebin", nullptr); - decodebinType = G_OBJECT_TYPE(decodebin.element()); - connect(&positionUpdateTimer, &QTimer::timeout, this, &QGstreamerMediaPlayer::updatePosition); + m_stalledMediaNotifier.setSingleShot(true); + connect(&m_stalledMediaNotifier, &QTimer::timeout, this, [this] { + mediaStatusChanged(QMediaPlayer::StalledMedia); + }); } QGstreamerMediaPlayer::~QGstreamerMediaPlayer() @@ -146,25 +166,45 @@ QGstreamerMediaPlayer::~QGstreamerMediaPlayer() playerPipeline.removeMessageFilter(static_cast<QGstreamerBusMessageFilter *>(this)); playerPipeline.removeMessageFilter(static_cast<QGstreamerSyncMessageFilter *>(this)); playerPipeline.setStateSync(GST_STATE_NULL); - topology.free(); } -qint64 QGstreamerMediaPlayer::position() const +std::chrono::nanoseconds QGstreamerMediaPlayer::pipelinePosition() const { - if (playerPipeline.isNull() || m_url.isEmpty()) - return 0; + if (!hasMedia()) + return {}; - return playerPipeline.position()/1e6; + Q_ASSERT(playerPipeline); + return playerPipeline.position(); +} + +void QGstreamerMediaPlayer::updatePositionFromPipeline() +{ + using namespace std::chrono; + + positionChanged(round<milliseconds>(pipelinePosition())); +} + +void QGstreamerMediaPlayer::updateDurationFromPipeline() +{ + std::optional<std::chrono::milliseconds> duration = playerPipeline.durationInMs(); + if (!duration) + duration = std::chrono::milliseconds{ -1 }; + + if (duration != m_duration) { + qCDebug(qLcMediaPlayer) << "updateDurationFromPipeline" << *duration; + m_duration = *duration; + durationChanged(m_duration); + } } qint64 QGstreamerMediaPlayer::duration() const { - return m_duration; + return m_duration.count(); } float QGstreamerMediaPlayer::bufferProgress() const { - return m_bufferProgress/100.; + return m_bufferProgress; } QMediaTimeRange QGstreamerMediaPlayer::availablePlaybackRanges() const @@ -179,18 +219,28 @@ qreal QGstreamerMediaPlayer::playbackRate() const void QGstreamerMediaPlayer::setPlaybackRate(qreal rate) { - if (playerPipeline.setPlaybackRate(rate)) - playbackRateChanged(rate); + if (rate == m_rate) + return; + + m_rate = rate; + + playerPipeline.setPlaybackRate(rate); + playbackRateChanged(rate); } void QGstreamerMediaPlayer::setPosition(qint64 pos) { - qint64 currentPos = playerPipeline.position()/1e6; - if (pos == currentPos) + std::chrono::milliseconds posInMs{ pos }; + setPosition(posInMs); +} + +void QGstreamerMediaPlayer::setPosition(std::chrono::milliseconds pos) +{ + if (pos == playerPipeline.position()) return; playerPipeline.finishStateChange(); - playerPipeline.setPosition(pos*1e6); - qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << pos << playerPipeline.position()/1e6; + playerPipeline.setPosition(pos); + qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << pos << playerPipeline.positionInMs(); if (mediaStatus() == QMediaPlayer::EndOfMedia) mediaStatusChanged(QMediaPlayer::LoadedMedia); positionChanged(pos); @@ -198,116 +248,199 @@ void QGstreamerMediaPlayer::setPosition(qint64 pos) void QGstreamerMediaPlayer::play() { - if (state() == QMediaPlayer::PlayingState || m_url.isEmpty()) + QMediaPlayer::PlaybackState currentState = state(); + if (currentState == QMediaPlayer::PlayingState || !hasMedia()) return; - resetCurrentLoop(); - playerPipeline.setInStoppedState(false); + if (currentState != QMediaPlayer::PausedState) + resetCurrentLoop(); + + gstVideoOutput->setActive(true); if (mediaStatus() == QMediaPlayer::EndOfMedia) { - playerPipeline.setPosition(0); - updatePosition(); + playerPipeline.setPosition({}); + positionChanged(0); } qCDebug(qLcMediaPlayer) << "play()."; int ret = playerPipeline.setState(GST_STATE_PLAYING); if (m_requiresSeekOnPlay) { - // Flushing the pipeline is required to get track changes - // immediately, when they happen while paused. + // Flushing the pipeline is required to get track changes immediately, when they happen + // while paused. playerPipeline.flush(); m_requiresSeekOnPlay = false; + } else { + if (currentState == QMediaPlayer::StoppedState) { + // we get an assertion failure during instant playback rate changes + // https://gitlab.freedesktop.org/gstreamer/gstreamer/-/issues/3545 + constexpr bool performInstantRateChange = false; + playerPipeline.applyPlaybackRate(/*instantRateChange=*/performInstantRateChange); + } } if (ret == GST_STATE_CHANGE_FAILURE) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the playing state."; - if (mediaStatus() == QMediaPlayer::LoadedMedia) - mediaStatusChanged(QMediaPlayer::BufferedMedia); - emit stateChanged(QMediaPlayer::PlayingState); + positionUpdateTimer.start(100); + stateChanged(QMediaPlayer::PlayingState); } void QGstreamerMediaPlayer::pause() { - if (state() == QMediaPlayer::PausedState || m_url.isEmpty()) + if (state() == QMediaPlayer::PausedState || !hasMedia() + || m_resourceErrorState != ResourceErrorState::NoError) return; positionUpdateTimer.stop(); - if (playerPipeline.inStoppedState()) { - playerPipeline.setInStoppedState(false); - playerPipeline.flush(); - } - int ret = playerPipeline.setState(GST_STATE_PAUSED); + + gstVideoOutput->setActive(true); + int ret = playerPipeline.setStateSync(GST_STATE_PAUSED); if (ret == GST_STATE_CHANGE_FAILURE) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; if (mediaStatus() == QMediaPlayer::EndOfMedia) { - playerPipeline.setPosition(0); - mediaStatusChanged(QMediaPlayer::BufferedMedia); + playerPipeline.setPosition({}); + positionChanged(0); + } else { + updatePositionFromPipeline(); } - updatePosition(); - emit stateChanged(QMediaPlayer::PausedState); + stateChanged(QMediaPlayer::PausedState); + + if (m_bufferProgress > 0 || !canTrackProgress()) + mediaStatusChanged(QMediaPlayer::BufferedMedia); + else + mediaStatusChanged(QMediaPlayer::BufferingMedia); } void QGstreamerMediaPlayer::stop() { - if (state() == QMediaPlayer::StoppedState) + using namespace std::chrono_literals; + if (state() == QMediaPlayer::StoppedState) { + if (position() != 0) { + playerPipeline.setPosition({}); + positionChanged(0ms); + mediaStatusChanged(QMediaPlayer::LoadedMedia); + } return; + } stopOrEOS(false); } +const QGstPipeline &QGstreamerMediaPlayer::pipeline() const +{ + return playerPipeline; +} + void QGstreamerMediaPlayer::stopOrEOS(bool eos) { + using namespace std::chrono_literals; + positionUpdateTimer.stop(); - playerPipeline.setInStoppedState(true); + gstVideoOutput->setActive(false); bool ret = playerPipeline.setStateSync(GST_STATE_PAUSED); if (!ret) qCDebug(qLcMediaPlayer) << "Unable to set the pipeline to the stopped state."; - if (!eos) - playerPipeline.setPosition(0); - updatePosition(); - emit stateChanged(QMediaPlayer::StoppedState); - mediaStatusChanged(eos ? QMediaPlayer::EndOfMedia : QMediaPlayer::LoadedMedia); + if (!eos) { + playerPipeline.setPosition(0ms); + positionChanged(0ms); + } + stateChanged(QMediaPlayer::StoppedState); + if (eos) + mediaStatusChanged(QMediaPlayer::EndOfMedia); + else + mediaStatusChanged(QMediaPlayer::LoadedMedia); + m_initialBufferProgressSent = false; + bufferProgressChanged(0.f); } -bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) +void QGstreamerMediaPlayer::detectPipelineIsSeekable() { - if (message.isNull()) - return false; + std::optional<bool> canSeek = playerPipeline.canSeek(); + if (canSeek) { + qCDebug(qLcMediaPlayer) << "detectPipelineIsSeekable: pipeline is seekable:" << *canSeek; + seekableChanged(*canSeek); + } else { + qCWarning(qLcMediaPlayer) << "detectPipelineIsSeekable: query for seekable failed."; + seekableChanged(false); + } +} -// qCDebug(qLcMediaPlayer) << "received bus message from" << message.source().name() << message.type() << (message.type() == GST_MESSAGE_TAG); +QGstElement QGstreamerMediaPlayer::getSinkElementForTrackType(TrackType trackType) +{ + switch (trackType) { + case AudioStream: + return gstAudioOutput ? gstAudioOutput->gstElement() : QGstElement{}; + case VideoStream: + return gstVideoOutput ? gstVideoOutput->gstElement() : QGstElement{}; + case SubtitleStream: + return gstVideoOutput ? gstVideoOutput->gstSubtitleElement() : QGstElement{}; + break; + default: + Q_UNREACHABLE_RETURN(QGstElement{}); + } +} - GstMessage* gm = message.rawMessage(); +bool QGstreamerMediaPlayer::hasMedia() const +{ + return !m_url.isEmpty() || m_stream; +} + +bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) +{ + qCDebug(qLcMediaPlayer) << "received bus message:" << message; + + GstMessage* gm = message.message(); switch (message.type()) { case GST_MESSAGE_TAG: { // #### This isn't ideal. We shouldn't catch stream specific tags here, rather the global ones - GstTagList *tag_list; - gst_message_parse_tag(gm, &tag_list); - //qCDebug(qLcMediaPlayer) << "Got tags: " << message.source().name() << gst_tag_list_to_string(tag_list); - auto metaData = QGstreamerMetaData::fromGstTagList(tag_list); - for (auto k : metaData.keys()) - m_metaData.insert(k, metaData.value(k)); + QGstTagListHandle tagList; + gst_message_parse_tag(gm, &tagList); + + qCDebug(qLcMediaPlayer) << " Got tags: " << tagList.get(); + + QMediaMetaData originalMetaData = m_metaData; + extendMetaDataFromTagList(m_metaData, tagList); + if (originalMetaData != m_metaData) + metaDataChanged(); + + if (gstVideoOutput) { + QVariant rotation = m_metaData.value(QMediaMetaData::Orientation); + gstVideoOutput->setRotation(rotation.value<QtVideo::Rotation>()); + } break; } case GST_MESSAGE_DURATION_CHANGED: { - qint64 d = playerPipeline.duration()/1e6; - qCDebug(qLcMediaPlayer) << " duration changed message" << d; - if (d != m_duration) { - m_duration = d; - emit durationChanged(duration()); - } + if (!prerolling) + updateDurationFromPipeline(); + return false; } - case GST_MESSAGE_EOS: + case GST_MESSAGE_EOS: { + positionChanged(m_duration); if (doLoop()) { setPosition(0); break; } stopOrEOS(true); break; + } case GST_MESSAGE_BUFFERING: { - qCDebug(qLcMediaPlayer) << " buffering message"; int progress = 0; gst_message_parse_buffering(gm, &progress); - m_bufferProgress = progress; - mediaStatusChanged(m_bufferProgress == 100 ? QMediaPlayer::BufferedMedia : QMediaPlayer::BufferingMedia); - emit bufferProgressChanged(m_bufferProgress/100.); + + if (state() != QMediaPlayer::StoppedState && !prerolling) { + if (!m_initialBufferProgressSent) { + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + } + + if (m_bufferProgress > 0 && progress == 0) { + m_stalledMediaNotifier.start(stalledMediaDebouncePeriod); + } else if (progress >= 50) + // QTBUG-124517: rethink buffering + mediaStatusChanged(QMediaPlayer::BufferedMedia); + else + mediaStatusChanged(QMediaPlayer::BufferingMedia); + } + + updateBufferProgress(progress * 0.01); break; } case GST_MESSAGE_STATE_CHANGED: { @@ -319,116 +452,126 @@ bool QGstreamerMediaPlayer::processBusMessage(const QGstreamerMessage &message) GstState pending; gst_message_parse_state_changed(gm, &oldState, &newState, &pending); - qCDebug(qLcMediaPlayer) << " state changed message" << oldState << newState << pending; - -#ifdef DEBUG_PLAYBIN - static QStringList states = { - QStringLiteral("GST_STATE_VOID_PENDING"), QStringLiteral("GST_STATE_NULL"), - QStringLiteral("GST_STATE_READY"), QStringLiteral("GST_STATE_PAUSED"), - QStringLiteral("GST_STATE_PLAYING") }; - - qCDebug(qLcMediaPlayer) << QStringLiteral("state changed: old: %1 new: %2 pending: %3") \ - .arg(states[oldState]) \ - .arg(states[newState]) \ - .arg(states[pending]); -#endif + qCDebug(qLcMediaPlayer) << " state changed message from" + << QCompactGstMessageAdaptor(message); switch (newState) { case GST_STATE_VOID_PENDING: case GST_STATE_NULL: case GST_STATE_READY: break; - case GST_STATE_PAUSED: - { + case GST_STATE_PAUSED: { if (prerolling) { qCDebug(qLcMediaPlayer) << "Preroll done, setting status to Loaded"; + playerPipeline.dumpGraph("playerPipelinePrerollDone"); + prerolling = false; - GST_DEBUG_BIN_TO_DOT_FILE(playerPipeline.bin(), GST_DEBUG_GRAPH_SHOW_ALL, "playerPipeline"); - qint64 d = playerPipeline.duration()/1e6; - if (d != m_duration) { - m_duration = d; - qCDebug(qLcMediaPlayer) << " duration changed" << d; - emit durationChanged(duration()); - } + updateDurationFromPipeline(); + m_metaData.insert(QMediaMetaData::Duration, duration()); + if (!m_url.isEmpty()) + m_metaData.insert(QMediaMetaData::Url, m_url); parseStreamsAndMetadata(); + metaDataChanged(); - emit tracksChanged(); + tracksChanged(); mediaStatusChanged(QMediaPlayer::LoadedMedia); - GstQuery *query = gst_query_new_seeking(GST_FORMAT_TIME); - gboolean canSeek = false; - if (gst_element_query(playerPipeline.element(), query)) { - gst_query_parse_seeking(query, NULL, &canSeek, nullptr, nullptr); - qCDebug(qLcMediaPlayer) << " pipeline is seekable:" << canSeek; - } else { - qCDebug(qLcMediaPlayer) << " query for seekable failed."; + if (state() == QMediaPlayer::PlayingState) { + Q_ASSERT(!m_initialBufferProgressSent); + + bool immediatelySendBuffered = !canTrackProgress() || m_bufferProgress > 0; + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + if (immediatelySendBuffered) + mediaStatusChanged(QMediaPlayer::BufferedMedia); } - gst_query_unref(query); - seekableChanged(canSeek); } break; } - case GST_STATE_PLAYING: - mediaStatusChanged(QMediaPlayer::BufferedMedia); + case GST_STATE_PLAYING: { + if (!m_initialBufferProgressSent) { + bool immediatelySendBuffered = !canTrackProgress() || m_bufferProgress > 0; + mediaStatusChanged(QMediaPlayer::BufferingMedia); + m_initialBufferProgressSent = true; + if (immediatelySendBuffered) + mediaStatusChanged(QMediaPlayer::BufferedMedia); + } break; } + } break; } case GST_MESSAGE_ERROR: { - GError *err; - gchar *debug; + qCDebug(qLcMediaPlayer) << " error" << QCompactGstMessageAdaptor(message); + + QUniqueGErrorHandle err; + QUniqueGStringHandle debug; gst_message_parse_error(gm, &err, &debug); - if (err->domain == GST_STREAM_ERROR && err->code == GST_STREAM_ERROR_CODEC_NOT_FOUND) - emit error(QMediaPlayer::FormatError, tr("Cannot play stream of type: <unknown>")); - else - emit error(QMediaPlayer::ResourceError, QString::fromUtf8(err->message)); - playerPipeline.dumpGraph("error"); + GQuark errorDomain = err.get()->domain; + gint errorCode = err.get()->code; + + if (errorDomain == GST_STREAM_ERROR) { + if (errorCode == GST_STREAM_ERROR_CODEC_NOT_FOUND) + error(QMediaPlayer::FormatError, tr("Cannot play stream of type: <unknown>")); + else { + error(QMediaPlayer::FormatError, QString::fromUtf8(err.get()->message)); + } + } else if (errorDomain == GST_RESOURCE_ERROR) { + if (errorCode == GST_RESOURCE_ERROR_NOT_FOUND) { + if (m_resourceErrorState != ResourceErrorState::ErrorReported) { + // gstreamer seems to deliver multiple GST_RESOURCE_ERROR_NOT_FOUND events + error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); + m_resourceErrorState = ResourceErrorState::ErrorReported; + m_url.clear(); + m_stream = nullptr; + } + } else { + error(QMediaPlayer::ResourceError, QString::fromUtf8(err.get()->message)); + } + } else { + playerPipeline.dumpGraph("error"); + } mediaStatusChanged(QMediaPlayer::InvalidMedia); - g_error_free(err); - g_free(debug); break; } - case GST_MESSAGE_WARNING: { - GError *err; - gchar *debug; - gst_message_parse_warning (gm, &err, &debug); - qCWarning(qLcMediaPlayer) << "Warning:" << QString::fromUtf8(err->message); + + case GST_MESSAGE_WARNING: + qCWarning(qLcMediaPlayer) << "Warning:" << QCompactGstMessageAdaptor(message); playerPipeline.dumpGraph("warning"); - g_error_free (err); - g_free (debug); break; - } - case GST_MESSAGE_INFO: { - if (qLcMediaPlayer().isDebugEnabled()) { - GError *err; - gchar *debug; - gst_message_parse_info (gm, &err, &debug); - qCDebug(qLcMediaPlayer) << "Info:" << QString::fromUtf8(err->message); - g_error_free (err); - g_free (debug); - } + + case GST_MESSAGE_INFO: + if (qLcMediaPlayer().isDebugEnabled()) + qCDebug(qLcMediaPlayer) << "Info:" << QCompactGstMessageAdaptor(message); break; - } + case GST_MESSAGE_SEGMENT_START: { qCDebug(qLcMediaPlayer) << " segment start message, updating position"; - QGstStructure structure(gst_message_get_structure(gm)); + QGstStructureView structure(gst_message_get_structure(gm)); auto p = structure["position"].toInt64(); if (p) { - qint64 position = (*p)/1000000; - emit positionChanged(position); + std::chrono::milliseconds position{ + (*p) / 1000000, + }; + positionChanged(position); } break; } case GST_MESSAGE_ELEMENT: { - QGstStructure structure(gst_message_get_structure(gm)); + QGstStructureView structure(gst_message_get_structure(gm)); auto type = structure.name(); - if (type == "stream-topology") { - topology.free(); - topology = structure.copy(); - } + if (type == "stream-topology") + topology = structure.clone(); + + break; + } + + case GST_MESSAGE_ASYNC_DONE: { + if (playerPipeline.state() >= GST_STATE_PAUSED) + detectPipelineIsSeekable(); break; } @@ -447,7 +590,7 @@ bool QGstreamerMediaPlayer::processSyncMessage(const QGstreamerMessage &message) if (message.type() != GST_MESSAGE_NEED_CONTEXT) return false; const gchar *type = nullptr; - gst_message_parse_context_type (message.rawMessage(), &type); + gst_message_parse_context_type (message.message(), &type); if (strcmp(type, GST_GL_DISPLAY_CONTEXT_TYPE)) return false; if (!gstVideoOutput || !gstVideoOutput->gstreamerVideoSink()) @@ -455,7 +598,7 @@ bool QGstreamerMediaPlayer::processSyncMessage(const QGstreamerMessage &message) auto *context = gstVideoOutput->gstreamerVideoSink()->gstGlDisplayContext(); if (!context) return false; - gst_element_set_context(GST_ELEMENT(GST_MESSAGE_SRC(message.rawMessage())), context); + gst_element_set_context(GST_ELEMENT(GST_MESSAGE_SRC(message.message())), context); playerPipeline.dumpGraph("need_context"); return true; #else @@ -482,7 +625,7 @@ void QGstreamerMediaPlayer::decoderPadAdded(const QGstElement &src, const QGstPa auto caps = pad.currentCaps(); auto type = caps.at(0).name(); qCDebug(qLcMediaPlayer) << "Received new pad" << pad.name() << "from" << src.name() << "type" << type; - qCDebug(qLcMediaPlayer) << " " << caps.toString(); + qCDebug(qLcMediaPlayer) << " " << caps; TrackType streamType = NTrackTypes; if (type.startsWith("video/x-raw")) { @@ -508,19 +651,19 @@ void QGstreamerMediaPlayer::decoderPadAdded(const QGstElement &src, const QGstPa if (streamType == VideoStream) { connectOutput(ts); ts.setActiveInputPad(sinkPad); - emit videoAvailableChanged(true); + videoAvailableChanged(true); } else if (streamType == AudioStream) { connectOutput(ts); ts.setActiveInputPad(sinkPad); - emit audioAvailableChanged(true); + audioAvailableChanged(true); } } if (!prerolling) - emit tracksChanged(); + tracksChanged(); - decoderOutputMap.insert(pad.name(), sinkPad); + decoderOutputMap.emplace(pad, sinkPad); } void QGstreamerMediaPlayer::decoderPadRemoved(const QGstElement &src, const QGstPad &pad) @@ -529,9 +672,11 @@ void QGstreamerMediaPlayer::decoderPadRemoved(const QGstElement &src, const QGst return; qCDebug(qLcMediaPlayer) << "Removed pad" << pad.name() << "from" << src.name(); - auto track = decoderOutputMap.value(pad.name()); - if (track.isNull()) + + auto it = decoderOutputMap.find(pad); + if (it == decoderOutputMap.end()) return; + QGstPad track = it->second; auto ts = std::find_if(std::begin(trackSelectors), std::end(trackSelectors), [&](TrackSelector &ts){ return ts.selector == track.parent(); }); @@ -568,27 +713,12 @@ void QGstreamerMediaPlayer::connectOutput(TrackSelector &ts) if (ts.isConnected) return; - QGstElement e; - switch (ts.type) { - case AudioStream: - e = gstAudioOutput ? gstAudioOutput->gstElement() : QGstElement{}; - break; - case VideoStream: - e = gstVideoOutput ? gstVideoOutput->gstElement() : QGstElement{}; - break; - case SubtitleStream: - if (gstVideoOutput) - gstVideoOutput->linkSubtitleStream(ts.selector); - break; - default: - return; - } - - if (!e.isNull()) { + QGstElement e = getSinkElementForTrackType(ts.type); + if (e) { qCDebug(qLcMediaPlayer) << "connecting output for track type" << ts.type; playerPipeline.add(e); - ts.selector.link(e); - e.setState(GST_STATE_PAUSED); + qLinkGstElements(ts.selector, e); + e.syncStateWithParent(); } ts.isConnected = true; @@ -599,47 +729,133 @@ void QGstreamerMediaPlayer::removeOutput(TrackSelector &ts) if (!ts.isConnected) return; - QGstElement e; - switch (ts.type) { - case AudioStream: - e = gstAudioOutput ? gstAudioOutput->gstElement() : QGstElement{}; - break; - case VideoStream: - e = gstVideoOutput ? gstVideoOutput->gstElement() : QGstElement{}; - break; - case SubtitleStream: - if (gstVideoOutput) - gstVideoOutput->unlinkSubtitleStream(); - break; - default: - break; - } - - if (!e.isNull()) { + QGstElement e = getSinkElementForTrackType(ts.type); + if (e) { qCDebug(qLcMediaPlayer) << "removing output for track type" << ts.type; - playerPipeline.remove(e); - e.setStateSync(GST_STATE_NULL); + playerPipeline.stopAndRemoveElements(e); } ts.isConnected = false; } -void QGstreamerMediaPlayer::uridecodebinElementAddedCallback(GstElement */*uridecodebin*/, GstElement *child, QGstreamerMediaPlayer *that) +void QGstreamerMediaPlayer::removeDynamicPipelineElements() { - QGstElement c(child); + for (QGstElement *element : { &src, &decoder }) { + if (element->isNull()) + continue; + + element->setStateSync(GstState::GST_STATE_NULL); + playerPipeline.remove(*element); + *element = QGstElement{}; + } +} + +void QGstreamerMediaPlayer::uridecodebinElementAddedCallback(GstElement * /*uridecodebin*/, + GstElement *child, + QGstreamerMediaPlayer *) +{ + QGstElement c(child, QGstElement::NeedsRef); qCDebug(qLcMediaPlayer) << "New element added to uridecodebin:" << c.name(); - if (G_OBJECT_TYPE(child) == that->decodebinType) { + static const GType decodeBinType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("decodebin"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + if (c.type() == decodeBinType) { qCDebug(qLcMediaPlayer) << " -> setting post-stream-topology property"; c.set("post-stream-topology", true); } } +void QGstreamerMediaPlayer::sourceSetupCallback(GstElement *uridecodebin, GstElement *source, QGstreamerMediaPlayer *that) +{ + Q_UNUSED(uridecodebin) + Q_UNUSED(that) + + qCDebug(qLcMediaPlayer) << "Setting up source:" << g_type_name_from_instance((GTypeInstance*)source); + + if (std::string_view("GstRTSPSrc") == g_type_name_from_instance((GTypeInstance *)source)) { + QGstElement s(source, QGstElement::NeedsRef); + int latency{40}; + bool ok{false}; + int v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_LATENCY", &ok); + if (ok) + latency = v; + qCDebug(qLcMediaPlayer) << " -> setting source latency to:" << latency << "ms"; + s.set("latency", latency); + + bool drop{true}; + v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DROP_ON_LATENCY", &ok); + if (ok && v == 0) + drop = false; + qCDebug(qLcMediaPlayer) << " -> setting drop-on-latency to:" << drop; + s.set("drop-on-latency", drop); + + bool retrans{false}; + v = qEnvironmentVariableIntValue("QT_MEDIA_RTSP_DO_RETRANSMISSION", &ok); + if (ok && v != 0) + retrans = true; + qCDebug(qLcMediaPlayer) << " -> setting do-retransmission to:" << retrans; + s.set("do-retransmission", retrans); + } +} + +void QGstreamerMediaPlayer::unknownTypeCallback(GstElement *decodebin, GstPad *pad, GstCaps *caps, + QGstreamerMediaPlayer *self) +{ + Q_UNUSED(decodebin) + Q_UNUSED(pad) + Q_UNUSED(self) + qCDebug(qLcMediaPlayer) << "Unknown type:" << caps; + + QMetaObject::invokeMethod(self, [self] { + self->stop(); + }); +} + +static bool isQueue(const QGstElement &element) +{ + static const GType queueType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("queue"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + static const GType multiQueueType = [] { + QGstElementFactoryHandle factory = QGstElement::findFactory("multiqueue"); + return gst_element_factory_get_element_type(factory.get()); + }(); + + return element.type() == queueType || element.type() == multiQueueType; +} + +void QGstreamerMediaPlayer::decodebinElementAddedCallback(GstBin * /*decodebin*/, + GstBin * /*sub_bin*/, GstElement *child, + QGstreamerMediaPlayer *self) +{ + QGstElement c(child, QGstElement::NeedsRef); + if (isQueue(c)) + self->decodeBinQueues += 1; +} + +void QGstreamerMediaPlayer::decodebinElementRemovedCallback(GstBin * /*decodebin*/, + GstBin * /*sub_bin*/, GstElement *child, + QGstreamerMediaPlayer *self) +{ + QGstElement c(child, QGstElement::NeedsRef); + if (isQueue(c)) + self->decodeBinQueues -= 1; +} + void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) { + using namespace std::chrono_literals; + qCDebug(qLcMediaPlayer) << Q_FUNC_INFO << "setting location to" << content; prerolling = true; + m_requiresSeekOnPlay = true; + m_resourceErrorState = ResourceErrorState::NoError; bool ret = playerPipeline.setStateSync(GST_STATE_NULL); if (!ret) @@ -648,73 +864,104 @@ void QGstreamerMediaPlayer::setMedia(const QUrl &content, QIODevice *stream) m_url = content; m_stream = stream; - if (!src.isNull()) - playerPipeline.remove(src); - if (!decoder.isNull()) - playerPipeline.remove(decoder); - src = QGstElement(); - decoder = QGstElement(); + removeDynamicPipelineElements(); + disconnectDecoderHandlers(); removeAllOutputs(); seekableChanged(false); - playerPipeline.setInStoppedState(true); - if (m_duration != 0) { - m_duration = 0; - durationChanged(0); + if (m_duration != 0ms) { + m_duration = 0ms; + durationChanged(0ms); } stateChanged(QMediaPlayer::StoppedState); if (position() != 0) - positionChanged(0); - mediaStatusChanged(QMediaPlayer::NoMedia); + positionChanged(0ms); if (!m_metaData.isEmpty()) { m_metaData.clear(); metaDataChanged(); } - if (content.isEmpty()) + if (content.isEmpty() && !stream) { + mediaStatusChanged(QMediaPlayer::NoMedia); return; + } if (m_stream) { - if (!m_appSrc) - m_appSrc = new QGstAppSrc(this); + if (!m_appSrc) { + auto maybeAppSrc = QGstAppSource::create(this); + if (maybeAppSrc) { + m_appSrc = maybeAppSrc.value(); + } else { + error(QMediaPlayer::ResourceError, maybeAppSrc.error()); + return; + } + } src = m_appSrc->element(); - decoder = QGstElement("decodebin", "decoder"); + decoder = QGstElement::createFromFactory("decodebin", "decoder"); + if (!decoder) { + error(QMediaPlayer::ResourceError, qGstErrorMessageCannotFindElement("decodebin")); + return; + } decoder.set("post-stream-topology", true); + decoder.set("use-buffering", true); + unknownType = decoder.connect("unknown-type", GCallback(unknownTypeCallback), this); + elementAdded = decoder.connect("deep-element-added", + GCallback(decodebinElementAddedCallback), this); + elementRemoved = decoder.connect("deep-element-removed", + GCallback(decodebinElementAddedCallback), this); + playerPipeline.add(src, decoder); - src.link(decoder); + qLinkGstElements(src, decoder); m_appSrc->setup(m_stream); seekableChanged(!stream->isSequential()); } else { // use uridecodebin - decoder = QGstElement("uridecodebin", "uridecoder"); + decoder = QGstElement::createFromFactory("uridecodebin", "decoder"); + if (!decoder) { + error(QMediaPlayer::ResourceError, qGstErrorMessageCannotFindElement("uridecodebin")); + return; + } playerPipeline.add(decoder); - // can't set post-stream-topology to true, as uridecodebin doesn't have the property. Use a hack - decoder.connect("element-added", GCallback(QGstreamerMediaPlayer::uridecodebinElementAddedCallback), this); - decoder.set("uri", content.toEncoded().constData()); - if (m_bufferProgress != 0) { - m_bufferProgress = 0; - emit bufferProgressChanged(0.); + constexpr bool hasPostStreamTopology = GST_CHECK_VERSION(1, 22, 0); + if constexpr (hasPostStreamTopology) { + decoder.set("post-stream-topology", true); + } else { + // can't set post-stream-topology to true, as uridecodebin doesn't have the property. + // Use a hack + uridecodebinElementAdded = decoder.connect( + "element-added", GCallback(uridecodebinElementAddedCallback), this); } + + sourceSetup = decoder.connect("source-setup", GCallback(sourceSetupCallback), this); + unknownType = decoder.connect("unknown-type", GCallback(unknownTypeCallback), this); + + decoder.set("uri", content.toEncoded().constData()); + decoder.set("use-buffering", true); + + constexpr int mb = 1024 * 1024; + decoder.set("ring-buffer-max-size", 2 * mb); + + updateBufferProgress(0.f); + + elementAdded = decoder.connect("deep-element-added", + GCallback(decodebinElementAddedCallback), this); + elementRemoved = decoder.connect("deep-element-removed", + GCallback(decodebinElementAddedCallback), this); } - decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAdded>(this); - decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemoved>(this); + padAdded = decoder.onPadAdded<&QGstreamerMediaPlayer::decoderPadAdded>(this); + padRemoved = decoder.onPadRemoved<&QGstreamerMediaPlayer::decoderPadRemoved>(this); mediaStatusChanged(QMediaPlayer::LoadingMedia); - - if (state() == QMediaPlayer::PlayingState) { - int ret = playerPipeline.setState(GST_STATE_PLAYING); - if (ret == GST_STATE_CHANGE_FAILURE) - qCWarning(qLcMediaPlayer) << "Unable to set the pipeline to the playing state."; - } else { - int ret = playerPipeline.setState(GST_STATE_PAUSED); - if (!ret) - qCWarning(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; + if (!playerPipeline.setStateSync(GST_STATE_PAUSED)) { + qCWarning(qLcMediaPlayer) << "Unable to set the pipeline to the paused state."; + // Note: no further error handling: errors will be delivered via a GstMessage + return; } - playerPipeline.setPosition(0); - positionChanged(0); + playerPipeline.setPosition(0ms); + positionChanged(0ms); } void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) @@ -724,17 +971,14 @@ void QGstreamerMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) auto &ts = trackSelector(AudioStream); - playerPipeline.beginConfig(); - if (gstAudioOutput) { - removeOutput(ts); - gstAudioOutput->setPipeline({}); - } - gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); - if (gstAudioOutput) { - gstAudioOutput->setPipeline(playerPipeline); - connectOutput(ts); - } - playerPipeline.endConfig(); + playerPipeline.modifyPipelineWhileNotRunning([&] { + if (gstAudioOutput) + removeOutput(ts); + + gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); + if (gstAudioOutput) + connectOutput(ts); + }); } QMediaMetaData QGstreamerMediaPlayer::metaData() const @@ -747,9 +991,9 @@ void QGstreamerMediaPlayer::setVideoSink(QVideoSink *sink) gstVideoOutput->setVideoSink(sink); } -static QGstStructure endOfChain(const QGstStructure &s) +static QGstStructureView endOfChain(const QGstStructureView &s) { - QGstStructure e = s; + QGstStructureView e = s; while (1) { auto next = e["next"].toStructure(); if (!next.isNull()) @@ -763,31 +1007,26 @@ static QGstStructure endOfChain(const QGstStructure &s) void QGstreamerMediaPlayer::parseStreamsAndMetadata() { qCDebug(qLcMediaPlayer) << "============== parse topology ============"; - if (topology.isNull()) { + + if (!topology) { qCDebug(qLcMediaPlayer) << " null topology"; return; } - auto caps = topology["caps"].toCaps(); - auto structure = caps.at(0); - auto fileFormat = QGstreamerFormatInfo::fileFormatForCaps(structure); - qCDebug(qLcMediaPlayer) << caps.toString() << fileFormat; - m_metaData.insert(QMediaMetaData::FileFormat, QVariant::fromValue(fileFormat)); - m_metaData.insert(QMediaMetaData::Duration, duration()); - m_metaData.insert(QMediaMetaData::Url, m_url); - QGValue tags = topology["tags"]; - if (!tags.isNull()) { - GstTagList *tagList = nullptr; - gst_structure_get(topology.structure, "tags", GST_TYPE_TAG_LIST, &tagList, nullptr); - const auto metaData = QGstreamerMetaData::fromGstTagList(tagList); - for (auto k : metaData.keys()) - m_metaData.insert(k, metaData.value(k)); - } - - auto demux = endOfChain(topology); - auto next = demux["next"]; + + QGstStructureView topologyView{ topology }; + + QGstCaps caps = topologyView.caps(); + extendMetaDataFromCaps(m_metaData, caps); + + QGstTagListHandle tagList = QGstStructureView{ topology }.tags(); + if (tagList) + extendMetaDataFromTagList(m_metaData, tagList); + + QGstStructureView demux = endOfChain(topologyView); + QGValue next = demux["next"]; if (!next.isList()) { qCDebug(qLcMediaPlayer) << " no additional streams"; - emit metaDataChanged(); + metaDataChanged(); return; } @@ -795,38 +1034,28 @@ void QGstreamerMediaPlayer::parseStreamsAndMetadata() int size = next.listSize(); for (int i = 0; i < size; ++i) { auto val = next.at(i); - caps = val.toStructure()["caps"].toCaps(); - structure = caps.at(0); - if (structure.name().startsWith("audio/")) { - auto codec = QGstreamerFormatInfo::audioCodecForCaps(structure); - m_metaData.insert(QMediaMetaData::AudioCodec, QVariant::fromValue(codec)); - qCDebug(qLcMediaPlayer) << " audio" << caps.toString() << (int)codec; - } else if (structure.name().startsWith("video/")) { - auto codec = QGstreamerFormatInfo::videoCodecForCaps(structure); - m_metaData.insert(QMediaMetaData::VideoCodec, QVariant::fromValue(codec)); - qCDebug(qLcMediaPlayer) << " video" << caps.toString() << (int)codec; - auto framerate = structure["framerate"].getFraction(); - if (framerate) - m_metaData.insert(QMediaMetaData::VideoFrameRate, *framerate); - auto width = structure["width"].toInt(); - auto height = structure["height"].toInt(); - if (width && height) - m_metaData.insert(QMediaMetaData::Resolution, QSize(*width, *height)); + caps = val.toStructure().caps(); + + extendMetaDataFromCaps(m_metaData, caps); + + QGstStructureView structure = caps.at(0); + + if (structure.name().startsWith("video/")) { + QSize nativeSize = structure.nativeSize(); + gstVideoOutput->setNativeSize(nativeSize); } } auto sinkPad = trackSelector(VideoStream).activeInputPad(); - if (!sinkPad.isNull()) { - bool hasTags = g_object_class_find_property (G_OBJECT_GET_CLASS (sinkPad.object()), "tags") != NULL; - - GstTagList *tl = nullptr; - g_object_get(sinkPad.object(), "tags", &tl, nullptr); - qCDebug(qLcMediaPlayer) << " tags=" << hasTags << (tl ? gst_tag_list_to_string(tl) : "(null)"); + if (sinkPad) { + QGstTagListHandle tagList = sinkPad.tags(); + if (tagList) + qCDebug(qLcMediaPlayer) << " tags=" << tagList.get(); + else + qCDebug(qLcMediaPlayer) << " tags=(null)"; } - qCDebug(qLcMediaPlayer) << "============== end parse topology ============"; - emit metaDataChanged(); playerPipeline.dumpGraph("playback"); } @@ -838,13 +1067,11 @@ int QGstreamerMediaPlayer::trackCount(QPlatformMediaPlayer::TrackType type) QMediaMetaData QGstreamerMediaPlayer::trackMetaData(QPlatformMediaPlayer::TrackType type, int index) { auto track = trackSelector(type).inputPad(index); - if (track.isNull()) + if (!track) return {}; - GstTagList *tagList = nullptr; - g_object_get(track.object(), "tags", &tagList, nullptr); - - return tagList ? QGstreamerMetaData::fromGstTagList(tagList) : QMediaMetaData{}; + QGstTagListHandle tagList = track.tags(); + return taglistToMetaData(tagList); } int QGstreamerMediaPlayer::activeTrack(TrackType type) @@ -866,14 +1093,14 @@ void QGstreamerMediaPlayer::setActiveTrack(TrackType type, int index) if (type == QPlatformMediaPlayer::SubtitleStream) gstVideoOutput->flushSubtitles(); - playerPipeline.beginConfig(); - if (track.isNull()) { - removeOutput(ts); - } else { - ts.setActiveInputPad(track); - connectOutput(ts); - } - playerPipeline.endConfig(); + playerPipeline.modifyPipelineWhileNotRunning([&] { + if (track.isNull()) { + removeOutput(ts); + } else { + ts.setActiveInputPad(track); + connectOutput(ts); + } + }); // seek to force an immediate change of the stream if (playerPipeline.state() == GST_STATE_PLAYING) diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h index cfc576643..f634d32a1 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermediaplayer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMEDIAPLAYER_P_H #define QGSTREAMERMEDIAPLAYER_P_H @@ -54,9 +18,10 @@ #include <QtCore/qstack.h> #include <private/qplatformmediaplayer_p.h> #include <private/qtmultimediaglobal_p.h> +#include <private/qmultimediautils_p.h> #include <qurl.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> #include <QtCore/qtimer.h> @@ -66,23 +31,19 @@ QT_BEGIN_NAMESPACE class QNetworkAccessManager; class QGstreamerMessage; -class QGstAppSrc; +class QGstAppSource; class QGstreamerAudioOutput; class QGstreamerVideoOutput; -class Q_MULTIMEDIA_EXPORT QGstreamerMediaPlayer - : public QObject, - public QPlatformMediaPlayer, - public QGstreamerBusMessageFilter, - public QGstreamerSyncMessageFilter +class QGstreamerMediaPlayer : public QObject, + public QPlatformMediaPlayer, + public QGstreamerBusMessageFilter, + public QGstreamerSyncMessageFilter { - Q_OBJECT - public: - QGstreamerMediaPlayer(QMediaPlayer *parent = 0); + static QMaybe<QPlatformMediaPlayer *> create(QMediaPlayer *parent = nullptr); ~QGstreamerMediaPlayer(); - qint64 position() const override; qint64 duration() const override; float bufferProgress() const override; @@ -94,7 +55,7 @@ public: QUrl media() const override; const QIODevice *mediaStream() const override; - void setMedia(const QUrl&, QIODevice *) override; + void setMedia(const QUrl &, QIODevice *) override; bool streamPlaybackSupported() const override { return true; } @@ -110,26 +71,32 @@ public: void setActiveTrack(TrackType, int /*streamNumber*/) override; void setPosition(qint64 pos) override; + void setPosition(std::chrono::milliseconds pos); void play() override; void pause() override; void stop() override; + const QGstPipeline &pipeline() const; + bool processBusMessage(const QGstreamerMessage& message) override; bool processSyncMessage(const QGstreamerMessage& message) override; -public Q_SLOTS: - void updatePosition() { positionChanged(position()); } - private: - struct TrackSelector { - TrackSelector(TrackType, const char *name); + QGstreamerMediaPlayer(QGstreamerVideoOutput *videoOutput, QMediaPlayer *parent); + + struct TrackSelector + { + TrackSelector(TrackType, QGstElement selector); QGstPad createInputPad(); void removeInputPad(QGstPad pad); void removeAllInputPads(); QGstPad inputPad(int index); int activeInputIndex() const { return isConnected ? tracks.indexOf(activeInputPad()) : -1; } - QGstPad activeInputPad() const { return isConnected ? selector.getObject("active-pad") : QGstPad{}; } + QGstPad activeInputPad() const + { + return isConnected ? QGstPad{ selector.getObject("active-pad") } : QGstPad{}; + } void setActiveInputPad(QGstPad input) { selector.set("active-pad", input); } int trackCount() const { return tracks.count(); } @@ -142,31 +109,61 @@ private: friend class QGstreamerStreamsControl; void decoderPadAdded(const QGstElement &src, const QGstPad &pad); void decoderPadRemoved(const QGstElement &src, const QGstPad &pad); - static void uridecodebinElementAddedCallback(GstElement *uridecodebin, GstElement *child, QGstreamerMediaPlayer *that); + void disconnectDecoderHandlers(); + static void uridecodebinElementAddedCallback(GstElement *uridecodebin, GstElement *child, + QGstreamerMediaPlayer *that); + static void sourceSetupCallback(GstElement *uridecodebin, GstElement *source, + QGstreamerMediaPlayer *that); + static void unknownTypeCallback(GstElement *decodebin, GstPad *pad, GstCaps *caps, + QGstreamerMediaPlayer *self); + static void decodebinElementAddedCallback(GstBin *decodebin, GstBin *sub_bin, + GstElement *element, QGstreamerMediaPlayer *self); + static void decodebinElementRemovedCallback(GstBin *decodebin, GstBin *sub_bin, + GstElement *element, QGstreamerMediaPlayer *self); + void parseStreamsAndMetadata(); void connectOutput(TrackSelector &ts); void removeOutput(TrackSelector &ts); + void removeDynamicPipelineElements(); void removeAllOutputs(); void stopOrEOS(bool eos); + bool canTrackProgress() const { return decodeBinQueues > 0; } + void detectPipelineIsSeekable(); + bool hasMedia() const; + + std::chrono::nanoseconds pipelinePosition() const; + void updatePositionFromPipeline(); + void updateDurationFromPipeline(); + void updateBufferProgress(float); + + QGstElement getSinkElementForTrackType(TrackType); std::array<TrackSelector, NTrackTypes> trackSelectors; TrackSelector &trackSelector(TrackType type); QMediaMetaData m_metaData; - int m_bufferProgress = -1; QUrl m_url; QIODevice *m_stream = nullptr; + enum class ResourceErrorState : uint8_t { + NoError, + ErrorOccurred, + ErrorReported, + }; + bool prerolling = false; - bool m_requiresSeekOnPlay = false; - qint64 m_duration = 0; + bool m_requiresSeekOnPlay = true; + bool m_initialBufferProgressSent = false; + ResourceErrorState m_resourceErrorState = ResourceErrorState::NoError; + float m_rate = 1.f; + float m_bufferProgress = 0.f; + std::chrono::milliseconds m_duration{}; QTimer positionUpdateTimer; - QGstAppSrc *m_appSrc = nullptr; + QGstAppSource *m_appSrc = nullptr; - GType decodebinType; - QGstStructure topology; + QUniqueGstStructureHandle topology; // Gst elements QGstPipeline playerPipeline; @@ -178,7 +175,30 @@ private: // QGstElement streamSynchronizer; - QHash<QByteArray, QGstPad> decoderOutputMap; + struct QGstPadLess + { + bool operator()(const QGstPad &lhs, const QGstPad &rhs) const + { + return lhs.pad() < rhs.pad(); + } + }; + + std::map<QGstPad, QGstPad, QGstPadLess> decoderOutputMap; + + // decoder connections + QGObjectHandlerScopedConnection padAdded; + QGObjectHandlerScopedConnection padRemoved; + QGObjectHandlerScopedConnection sourceSetup; + QGObjectHandlerScopedConnection uridecodebinElementAdded; + QGObjectHandlerScopedConnection unknownType; + QGObjectHandlerScopedConnection elementAdded; + QGObjectHandlerScopedConnection elementRemoved; + + int decodeBinQueues = 0; + + void mediaStatusChanged(QMediaPlayer::MediaStatus status); + static constexpr auto stalledMediaDebouncePeriod = std::chrono::milliseconds{ 500 }; + QTimer m_stalledMediaNotifier; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermessage.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermessage.cpp deleted file mode 100644 index 02a5dd3bc..000000000 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermessage.cpp +++ /dev/null @@ -1,94 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <gst/gst.h> - -#include "qgstreamermessage_p.h" - -QT_BEGIN_NAMESPACE - -/*! - \class QGstreamerMessage - \internal -*/ - -QGstreamerMessage::QGstreamerMessage(GstMessage* message): - m_message(message) -{ - gst_message_ref(m_message); -} - -QGstreamerMessage::QGstreamerMessage(QGstreamerMessage const& m): - m_message(m.m_message) -{ - gst_message_ref(m_message); -} - -QGstreamerMessage::QGstreamerMessage(const QGstStructure &structure) -{ - gst_structure_get(structure.structure, "message", GST_TYPE_MESSAGE, &m_message, nullptr); -} - -QGstreamerMessage::~QGstreamerMessage() -{ - if (m_message != nullptr) - gst_message_unref(m_message); -} - -GstMessage* QGstreamerMessage::rawMessage() const -{ - return m_message; -} - -QGstreamerMessage& QGstreamerMessage::operator=(QGstreamerMessage const& rhs) -{ - if (rhs.m_message != m_message) { - if (rhs.m_message != nullptr) - gst_message_ref(rhs.m_message); - - if (m_message != nullptr) - gst_message_unref(m_message); - - m_message = rhs.m_message; - } - - return *this; -} - -QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h index 483f8f5e3..9836bd0cb 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermessage_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMESSAGE_P_H #define QGSTREAMERMESSAGE_P_H @@ -52,34 +16,36 @@ // #include <private/qtmultimediaglobal_p.h> -#include <qgst_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE // Required for QDoc workaround class QString; -class Q_MULTIMEDIA_EXPORT QGstreamerMessage +template <> +struct QGstPointerImpl::QGstRefcountingAdaptor<GstMessage> { -public: - QGstreamerMessage() = default; - QGstreamerMessage(QGstreamerMessage const& m); - explicit QGstreamerMessage(GstMessage* message); - explicit QGstreamerMessage(const QGstStructure &structure); - - ~QGstreamerMessage(); + static void ref(GstMessage *arg) noexcept { gst_message_ref(arg); } + static void unref(GstMessage *arg) noexcept { gst_message_unref(arg); } +}; - bool isNull() const { return !m_message; } - GstMessageType type() const { return GST_MESSAGE_TYPE(m_message); } - QGstObject source() const { return QGstObject(GST_MESSAGE_SRC(m_message), QGstObject::NeedsRef); } - QGstStructure structure() const { return QGstStructure(gst_message_get_structure(m_message)); } +class QGstreamerMessage : public QGstPointerImpl::QGstObjectWrapper<GstMessage> +{ + using BaseClass = QGstPointerImpl::QGstObjectWrapper<GstMessage>; - GstMessage* rawMessage() const; +public: + using BaseClass::BaseClass; + QGstreamerMessage(const QGstreamerMessage &) = default; + QGstreamerMessage(QGstreamerMessage &&) noexcept = default; + QGstreamerMessage &operator=(const QGstreamerMessage &) = default; + QGstreamerMessage &operator=(QGstreamerMessage &&) noexcept = default; - QGstreamerMessage& operator=(QGstreamerMessage const& rhs); + GstMessageType type() const { return GST_MESSAGE_TYPE(get()); } + QGstObject source() const { return QGstObject(GST_MESSAGE_SRC(get()), QGstObject::NeedsRef); } + QGstStructureView structure() const { return QGstStructureView(gst_message_get_structure(get())); } -private: - GstMessage* m_message = nullptr; + GstMessage *message() const { return get(); } }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp index 55f5ef155..9aa9406b9 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata.cpp @@ -1,308 +1,489 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamermetadata_p.h" -#include <QDebug> #include <QtMultimedia/qmediametadata.h> +#include <QtMultimedia/qtvideo.h> +#include <QtCore/qdebug.h> #include <QtCore/qdatetime.h> +#include <QtCore/qlocale.h> +#include <QtCore/qtimezone.h> +#include <QtGui/qimage.h> #include <gst/gstversion.h> -#include <qgstutils_p.h> -#include <qlocale.h> +#include <common/qgst_handle_types_p.h> +#include <common/qgstutils_p.h> +#include <qgstreamerformatinfo_p.h> QT_BEGIN_NAMESPACE -struct { +namespace { + +namespace MetadataLookupImpl { + +#ifdef __cpp_lib_constexpr_algorithms +# define constexpr_lookup constexpr +#else +# define constexpr_lookup /*constexpr*/ +#endif + +struct MetadataKeyValuePair +{ const char *tag; QMediaMetaData::Key key; -} gstTagToMetaDataKey[] = { - { GST_TAG_TITLE, QMediaMetaData::Title }, - { GST_TAG_COMMENT, QMediaMetaData::Comment }, - { GST_TAG_DESCRIPTION, QMediaMetaData::Description }, - { GST_TAG_GENRE, QMediaMetaData::Genre }, - { GST_TAG_DATE_TIME, QMediaMetaData::Date }, - { GST_TAG_DATE, QMediaMetaData::Date }, +}; + +constexpr const char *toTag(const char *t) +{ + return t; +} +constexpr const char *toTag(const MetadataKeyValuePair &kv) +{ + return kv.tag; +} + +constexpr QMediaMetaData::Key toKey(QMediaMetaData::Key k) +{ + return k; +} +constexpr QMediaMetaData::Key toKey(const MetadataKeyValuePair &kv) +{ + return kv.key; +} + +constexpr auto compareByKey = [](const auto &lhs, const auto &rhs) { + return toKey(lhs) < toKey(rhs); +}; - { GST_TAG_LANGUAGE_CODE, QMediaMetaData::Language }, +constexpr auto compareByTag = [](const auto &lhs, const auto &rhs) { + return std::strcmp(toTag(lhs), toTag(rhs)) < 0; +}; + +constexpr_lookup auto makeLookupTable() +{ + std::array<MetadataKeyValuePair, 22> lookupTable{ { + { GST_TAG_TITLE, QMediaMetaData::Title }, + { GST_TAG_COMMENT, QMediaMetaData::Comment }, + { GST_TAG_DESCRIPTION, QMediaMetaData::Description }, + { GST_TAG_GENRE, QMediaMetaData::Genre }, + { GST_TAG_DATE_TIME, QMediaMetaData::Date }, + { GST_TAG_DATE, QMediaMetaData::Date }, + + { GST_TAG_LANGUAGE_CODE, QMediaMetaData::Language }, + + { GST_TAG_ORGANIZATION, QMediaMetaData::Publisher }, + { GST_TAG_COPYRIGHT, QMediaMetaData::Copyright }, + + // Media + { GST_TAG_DURATION, QMediaMetaData::Duration }, + + // Audio + { GST_TAG_BITRATE, QMediaMetaData::AudioBitRate }, + { GST_TAG_AUDIO_CODEC, QMediaMetaData::AudioCodec }, + + // Music + { GST_TAG_ALBUM, QMediaMetaData::AlbumTitle }, + { GST_TAG_ALBUM_ARTIST, QMediaMetaData::AlbumArtist }, + { GST_TAG_ARTIST, QMediaMetaData::ContributingArtist }, + { GST_TAG_TRACK_NUMBER, QMediaMetaData::TrackNumber }, + + { GST_TAG_PREVIEW_IMAGE, QMediaMetaData::ThumbnailImage }, + { GST_TAG_IMAGE, QMediaMetaData::CoverArtImage }, + + // Image/Video + { "resolution", QMediaMetaData::Resolution }, + { GST_TAG_IMAGE_ORIENTATION, QMediaMetaData::Orientation }, + + // Video + { GST_TAG_VIDEO_CODEC, QMediaMetaData::VideoCodec }, + + // Movie + { GST_TAG_PERFORMER, QMediaMetaData::LeadPerformer }, + } }; + + std::sort(lookupTable.begin(), lookupTable.end(), + [](const MetadataKeyValuePair &lhs, const MetadataKeyValuePair &rhs) { + return std::string_view(lhs.tag) < std::string_view(rhs.tag); + }); + return lookupTable; +} - { GST_TAG_ORGANIZATION, QMediaMetaData::Publisher }, - { GST_TAG_COPYRIGHT, QMediaMetaData::Copyright }, +constexpr_lookup auto gstTagToMetaDataKey = makeLookupTable(); +constexpr_lookup auto metaDataKeyToGstTag = [] { + auto array = gstTagToMetaDataKey; + std::sort(array.begin(), array.end(), compareByKey); + return array; +}(); - // Media - { GST_TAG_DURATION, QMediaMetaData::Duration }, +} // namespace MetadataLookupImpl - // Audio - { GST_TAG_BITRATE, QMediaMetaData::AudioBitRate }, - { GST_TAG_AUDIO_CODEC, QMediaMetaData::AudioCodec }, +QMediaMetaData::Key tagToKey(const char *tag) +{ + if (tag == nullptr) + return QMediaMetaData::Key(-1); - // Music - { GST_TAG_ALBUM, QMediaMetaData::AlbumTitle }, - { GST_TAG_ALBUM_ARTIST, QMediaMetaData::AlbumArtist }, - { GST_TAG_ARTIST, QMediaMetaData::ContributingArtist }, - { GST_TAG_TRACK_NUMBER, QMediaMetaData::TrackNumber }, + using namespace MetadataLookupImpl; + auto foundIterator = std::lower_bound(gstTagToMetaDataKey.begin(), gstTagToMetaDataKey.end(), + tag, compareByTag); + if (std::strcmp(foundIterator->tag, tag) == 0) + return foundIterator->key; - { GST_TAG_PREVIEW_IMAGE, QMediaMetaData::ThumbnailImage }, - { GST_TAG_IMAGE, QMediaMetaData::CoverArtImage }, + return QMediaMetaData::Key(-1); +} - // Image/Video - { "resolution", QMediaMetaData::Resolution }, - { GST_TAG_IMAGE_ORIENTATION, QMediaMetaData::Orientation }, +const char *keyToTag(QMediaMetaData::Key key) +{ + using namespace MetadataLookupImpl; + auto foundIterator = std::lower_bound(metaDataKeyToGstTag.begin(), metaDataKeyToGstTag.end(), + key, compareByKey); + if (foundIterator->key == key) + return foundIterator->tag; - // Video - { GST_TAG_VIDEO_CODEC, QMediaMetaData::VideoCodec }, + return nullptr; +} - // Movie - { GST_TAG_PERFORMER, QMediaMetaData::LeadPerformer }, +#undef constexpr_lookup - { nullptr, QMediaMetaData::Title } -}; +QtVideo::Rotation parseRotationTag(const char *string) +{ + using namespace std::string_view_literals; + + if (string == "rotate-90"sv) + return QtVideo::Rotation::Clockwise90; + if (string == "rotate-180"sv) + return QtVideo::Rotation::Clockwise180; + if (string == "rotate-270"sv) + return QtVideo::Rotation::Clockwise270; + if (string == "rotate-0"sv) + return QtVideo::Rotation::None; + + qCritical() << "cannot parse orientation: {}" << string; + return QtVideo::Rotation::None; +} + +QDateTime parseDate(const GValue &val) +{ + Q_ASSERT(G_VALUE_TYPE(&val) == G_TYPE_DATE); + + const GDate *date = (const GDate *)g_value_get_boxed(&val); + if (!g_date_valid(date)) + return {}; + + int year = g_date_get_year(date); + int month = g_date_get_month(date); + int day = g_date_get_day(date); + return QDateTime(QDate(year, month, day), QTime()); +} -static QMediaMetaData::Key tagToKey(const char *tag) +QDateTime parseDateTime(const GValue &val) { - auto *map = gstTagToMetaDataKey; - while (map->tag) { - if (!strcmp(map->tag, tag)) - return map->key; - ++map; + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_DATE_TIME); + + const GstDateTime *dateTime = (const GstDateTime *)g_value_get_boxed(&val); + int year = gst_date_time_has_year(dateTime) ? gst_date_time_get_year(dateTime) : 0; + int month = gst_date_time_has_month(dateTime) ? gst_date_time_get_month(dateTime) : 0; + int day = gst_date_time_has_day(dateTime) ? gst_date_time_get_day(dateTime) : 0; + int hour = 0; + int minute = 0; + int second = 0; + float tz = 0; + if (gst_date_time_has_time(dateTime)) { + hour = gst_date_time_get_hour(dateTime); + minute = gst_date_time_get_minute(dateTime); + second = gst_date_time_get_second(dateTime); + tz = gst_date_time_get_time_zone_offset(dateTime); } - return QMediaMetaData::Key(-1); + return QDateTime{ + QDate(year, month, day), + QTime(hour, minute, second), + QTimeZone(tz * 60 * 60), + }; } -static const char *keyToTag(QMediaMetaData::Key key) +QImage parseImage(const GValue &val) { - auto *map = gstTagToMetaDataKey; - while (map->tag) { - if (map->key == key) - return map->tag; - ++map; + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_SAMPLE); + + GstSample *sample = (GstSample *)g_value_get_boxed(&val); + GstCaps *caps = gst_sample_get_caps(sample); + if (caps && !gst_caps_is_empty(caps)) { + GstStructure *structure = gst_caps_get_structure(caps, 0); + const gchar *name = gst_structure_get_name(structure); + if (QByteArray(name).startsWith("image/")) { + GstBuffer *buffer = gst_sample_get_buffer(sample); + if (buffer) { + GstMapInfo info; + gst_buffer_map(buffer, &info, GST_MAP_READ); + QImage image = QImage::fromData(info.data, info.size, name); + gst_buffer_unmap(buffer, &info); + return image; + } + } } - return nullptr; + + return {}; +} + +std::optional<double> parseFractionAsDouble(const GValue &val) +{ + Q_ASSERT(G_VALUE_TYPE(&val) == GST_TYPE_FRACTION); + + int nom = gst_value_get_fraction_numerator(&val); + int denom = gst_value_get_fraction_denominator(&val); + if (denom == 0) + return std::nullopt; + return double(nom) / double(denom); } -//internal -static void addTagToMap(const GstTagList *list, - const gchar *tag, - gpointer user_data) +constexpr std::string_view extendedComment{ GST_TAG_EXTENDED_COMMENT }; + +void addTagsFromExtendedComment(const GstTagList *list, const gchar *tag, QMediaMetaData &metadata) { + using namespace Qt::Literals; + assert(tag == extendedComment); + + int entryCount = gst_tag_list_get_tag_size(list, tag); + for (int i = 0; i != entryCount; ++i) { + const GValue *value = gst_tag_list_get_value_index(list, tag, i); + + const QLatin1StringView strValue{ g_value_get_string(value) }; + + auto equalIndex = strValue.indexOf(QLatin1StringView("=")); + if (equalIndex == -1) { + qDebug() << "Cannot parse GST_TAG_EXTENDED_COMMENT entry: " << value; + continue; + } + + const QLatin1StringView key = strValue.first(equalIndex); + const QLatin1StringView valueString = strValue.last(strValue.size() - equalIndex - 1); + + if (key == "DURATION"_L1) { + QUniqueGstDateTimeHandle duration{ + gst_date_time_new_from_iso8601_string(valueString.data()), + }; + + if (duration) { + using namespace std::chrono; + + auto chronoDuration = hours(gst_date_time_get_hour(duration.get())) + + minutes(gst_date_time_get_minute(duration.get())) + + seconds(gst_date_time_get_second(duration.get())) + + microseconds(gst_date_time_get_microsecond(duration.get())); + + metadata.insert(QMediaMetaData::Duration, + QVariant::fromValue(round<milliseconds>(chronoDuration).count())); + } + } + } +} + +void addTagToMetaData(const GstTagList *list, const gchar *tag, void *userdata) +{ + QMediaMetaData &metadata = *reinterpret_cast<QMediaMetaData *>(userdata); + QMediaMetaData::Key key = tagToKey(tag); - if (key == QMediaMetaData::Key(-1)) - return; + if (key == QMediaMetaData::Key(-1)) { + if (tag == extendedComment) + addTagsFromExtendedComment(list, tag, metadata); - auto *map = reinterpret_cast<QHash<QMediaMetaData::Key, QVariant>* >(user_data); + return; + } - GValue val; - val.g_type = 0; + GValue val{}; gst_tag_list_copy_value(&val, list, tag); + GType type = G_VALUE_TYPE(&val); - switch( G_VALUE_TYPE(&val) ) { - case G_TYPE_STRING: - { - const gchar *str_value = g_value_get_string(&val); - if (key == QMediaMetaData::Language) { - map->insert(key, QVariant::fromValue(QLocale::codeToLanguage(QString::fromUtf8(str_value), QLocale::ISO639Part2))); - break; - } - map->insert(key, QString::fromUtf8(str_value)); + if (auto entryCount = gst_tag_list_get_tag_size(list, tag) != 0; entryCount != 1) + qWarning() << "addTagToMetaData: invaled entry count for" << tag << "-" << entryCount; + + if (type == G_TYPE_STRING) { + const gchar *str_value = g_value_get_string(&val); + + switch (key) { + case QMediaMetaData::Language: { + metadata.insert(key, + QVariant::fromValue(QLocale::codeToLanguage( + QString::fromUtf8(str_value), QLocale::AnyLanguageCode))); break; } - case G_TYPE_INT: - map->insert(key, g_value_get_int(&val)); - break; - case G_TYPE_UINT: - map->insert(key, g_value_get_uint(&val)); - break; - case G_TYPE_LONG: - map->insert(key, qint64(g_value_get_long(&val))); - break; - case G_TYPE_BOOLEAN: - map->insert(key, g_value_get_boolean(&val)); - break; - case G_TYPE_CHAR: - map->insert(key, g_value_get_schar(&val)); - break; - case G_TYPE_DOUBLE: - map->insert(key, g_value_get_double(&val)); + case QMediaMetaData::Orientation: { + metadata.insert(key, QVariant::fromValue(parseRotationTag(str_value))); break; + } default: - // GST_TYPE_DATE is a function, not a constant, so pull it out of the switch - if (G_VALUE_TYPE(&val) == G_TYPE_DATE) { - const GDate *date = (const GDate *)g_value_get_boxed(&val); - if (g_date_valid(date)) { - int year = g_date_get_year(date); - int month = g_date_get_month(date); - int day = g_date_get_day(date); - // don't insert if we already have a datetime. - if (!map->contains(key)) - map->insert(key, QDateTime(QDate(year, month, day), QTime())); - } - } else if (G_VALUE_TYPE(&val) == GST_TYPE_DATE_TIME) { - const GstDateTime *dateTime = (const GstDateTime *)g_value_get_boxed(&val); - int year = gst_date_time_has_year(dateTime) ? gst_date_time_get_year(dateTime) : 0; - int month = gst_date_time_has_month(dateTime) ? gst_date_time_get_month(dateTime) : 0; - int day = gst_date_time_has_day(dateTime) ? gst_date_time_get_day(dateTime) : 0; - int hour = 0; - int minute = 0; - int second = 0; - float tz = 0; - if (gst_date_time_has_time(dateTime)) { - hour = gst_date_time_get_hour(dateTime); - minute = gst_date_time_get_minute(dateTime); - second = gst_date_time_get_second(dateTime); - tz = gst_date_time_get_time_zone_offset(dateTime); - } - QDateTime qDateTime(QDate(year, month, day), QTime(hour, minute, second), - Qt::OffsetFromUTC, tz * 60 * 60); - map->insert(key, qDateTime); - } else if (G_VALUE_TYPE(&val) == GST_TYPE_SAMPLE) { - GstSample *sample = (GstSample *)g_value_get_boxed(&val); - GstCaps* caps = gst_sample_get_caps(sample); - if (caps && !gst_caps_is_empty(caps)) { - GstStructure *structure = gst_caps_get_structure(caps, 0); - const gchar *name = gst_structure_get_name(structure); - if (QByteArray(name).startsWith("image/")) { - GstBuffer *buffer = gst_sample_get_buffer(sample); - if (buffer) { - GstMapInfo info; - gst_buffer_map(buffer, &info, GST_MAP_READ); - map->insert(key, QImage::fromData(info.data, info.size, name)); - gst_buffer_unmap(buffer, &info); - } - } - } - } else if (G_VALUE_TYPE(&val) == GST_TYPE_FRACTION) { - int nom = gst_value_get_fraction_numerator(&val); - int denom = gst_value_get_fraction_denominator(&val); - - if (denom > 0) { - map->insert(key, double(nom)/denom); - } - } + metadata.insert(key, QString::fromUtf8(str_value)); break; + }; + } else if (type == G_TYPE_INT) { + metadata.insert(key, g_value_get_int(&val)); + } else if (type == G_TYPE_UINT) { + metadata.insert(key, g_value_get_uint(&val)); + } else if (type == G_TYPE_LONG) { + metadata.insert(key, qint64(g_value_get_long(&val))); + } else if (type == G_TYPE_BOOLEAN) { + metadata.insert(key, g_value_get_boolean(&val)); + } else if (type == G_TYPE_CHAR) { + metadata.insert(key, g_value_get_schar(&val)); + } else if (type == G_TYPE_DOUBLE) { + metadata.insert(key, g_value_get_double(&val)); + } else if (type == G_TYPE_DATE) { + if (!metadata.keys().contains(key)) { + QDateTime date = parseDate(val); + if (date.isValid()) + metadata.insert(key, date); + } + } else if (type == GST_TYPE_DATE_TIME) { + metadata.insert(key, parseDateTime(val)); + } else if (type == GST_TYPE_SAMPLE) { + QImage image = parseImage(val); + if (!image.isNull()) + metadata.insert(key, image); + } else if (type == GST_TYPE_FRACTION) { + std::optional<double> fraction = parseFractionAsDouble(val); + + if (fraction) + metadata.insert(key, *fraction); } g_value_unset(&val); } +} // namespace -QGstreamerMetaData QGstreamerMetaData::fromGstTagList(const GstTagList *tags) +QMediaMetaData taglistToMetaData(const QGstTagListHandle &handle) { - QGstreamerMetaData m; - gst_tag_list_foreach(tags, addTagToMap, &m.data); + QMediaMetaData m; + extendMetaDataFromTagList(m, handle); return m; } - -void QGstreamerMetaData::setMetaData(GstElement *element) const +void extendMetaDataFromTagList(QMediaMetaData &metadata, const QGstTagListHandle &handle) { - if (!GST_IS_TAG_SETTER(element)) - return; + if (handle) + gst_tag_list_foreach(handle.get(), reinterpret_cast<GstTagForeachFunc>(&addTagToMetaData), + &metadata); +} - gst_tag_setter_reset_tags(GST_TAG_SETTER(element)); +static void applyMetaDataToTagSetter(const QMediaMetaData &metadata, GstTagSetter *element) +{ + gst_tag_setter_reset_tags(element); - for (auto it = data.cbegin(), end = data.cend(); it != end; ++it) { - const char *tagName = keyToTag(it.key()); + for (QMediaMetaData::Key key : metadata.keys()) { + const char *tagName = keyToTag(key); if (!tagName) continue; - const QVariant &tagValue = it.value(); + const QVariant &tagValue = metadata.value(key); + + auto setTag = [&](const auto &value) { + gst_tag_setter_add_tags(element, GST_TAG_MERGE_REPLACE, tagName, value, nullptr); + }; switch (tagValue.typeId()) { - case QMetaType::QString: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toString().toUtf8().constData(), - nullptr); - break; - case QMetaType::Int: - case QMetaType::LongLong: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toInt(), - nullptr); - break; - case QMetaType::Double: - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - tagValue.toDouble(), - nullptr); - break; - case QMetaType::QDate: - case QMetaType::QDateTime: { - QDateTime date = tagValue.toDateTime(); - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - gst_date_time_new(date.offsetFromUtc() / 60. / 60., - date.date().year(), date.date().month(), date.date().day(), - date.time().hour(), date.time().minute(), date.time().second()), - nullptr); - break; - } - default: { - if (tagValue.typeId() == qMetaTypeId<QLocale::Language>()) { - QByteArray language = QLocale::languageToCode(tagValue.value<QLocale::Language>(), QLocale::ISO639Part2).toUtf8(); - gst_tag_setter_add_tags(GST_TAG_SETTER(element), - GST_TAG_MERGE_REPLACE, - tagName, - language.constData(), - nullptr); - } - - break; + case QMetaType::QString: + setTag(tagValue.toString().toUtf8().constData()); + break; + case QMetaType::Int: + case QMetaType::LongLong: + setTag(tagValue.toInt()); + break; + case QMetaType::Double: + setTag(tagValue.toDouble()); + break; + case QMetaType::QDate: + case QMetaType::QDateTime: { + QDateTime date = tagValue.toDateTime(); + + QGstGstDateTimeHandle dateTime{ + gst_date_time_new(date.offsetFromUtc() / 60. / 60., date.date().year(), + date.date().month(), date.date().day(), date.time().hour(), + date.time().minute(), date.time().second()), + QGstGstDateTimeHandle::HasRef, + }; + + setTag(dateTime.get()); + break; + } + default: { + if (tagValue.typeId() == qMetaTypeId<QLocale::Language>()) { + QByteArray language = QLocale::languageToCode(tagValue.value<QLocale::Language>(), + QLocale::ISO639Part2) + .toUtf8(); + setTag(language.constData()); } + + break; + } } } } -void QGstreamerMetaData::setMetaData(GstBin *bin) const +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstElement &element) +{ + GstTagSetter *tagSetter = qGstSafeCast<GstTagSetter>(element.element()); + if (tagSetter) + applyMetaDataToTagSetter(metadata, tagSetter); + else + qWarning() << "applyMetaDataToTagSetter failed: element not a GstTagSetter" + << element.name(); +} + +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstBin &bin) { - GstIterator *elements = gst_bin_iterate_all_by_interface(bin, GST_TYPE_TAG_SETTER); - GValue item = G_VALUE_INIT; + GstIterator *elements = gst_bin_iterate_all_by_interface(bin.bin(), GST_TYPE_TAG_SETTER); + GValue item = {}; + while (gst_iterator_next(elements, &item) == GST_ITERATOR_OK) { - GstElement * const element = GST_ELEMENT(g_value_get_object(&item)); - setMetaData(element); + GstElement *element = static_cast<GstElement *>(g_value_get_object(&item)); + if (!element) + continue; + + GstTagSetter *tagSetter = qGstSafeCast<GstTagSetter>(element); + + if (tagSetter) + applyMetaDataToTagSetter(metadata, tagSetter); } + gst_iterator_free(elements); } +void extendMetaDataFromCaps(QMediaMetaData &metadata, const QGstCaps &caps) +{ + QGstStructureView structure = caps.at(0); + + QMediaFormat::FileFormat fileFormat = QGstreamerFormatInfo::fileFormatForCaps(structure); + if (fileFormat != QMediaFormat::FileFormat::UnspecifiedFormat) { + // Container caps + metadata.insert(QMediaMetaData::FileFormat, fileFormat); + return; + } + + QMediaFormat::AudioCodec audioCodec = QGstreamerFormatInfo::audioCodecForCaps(structure); + if (audioCodec != QMediaFormat::AudioCodec::Unspecified) { + // Audio stream caps + metadata.insert(QMediaMetaData::AudioCodec, QVariant::fromValue(audioCodec)); + return; + } + + QMediaFormat::VideoCodec videoCodec = QGstreamerFormatInfo::videoCodecForCaps(structure); + if (videoCodec != QMediaFormat::VideoCodec::Unspecified) { + // Video stream caps + metadata.insert(QMediaMetaData::VideoCodec, QVariant::fromValue(videoCodec)); + std::optional<float> framerate = structure["framerate"].getFraction(); + if (framerate) + metadata.insert(QMediaMetaData::VideoFrameRate, *framerate); + + QSize resolution = structure.resolution(); + if (resolution.isValid()) + metadata.insert(QMediaMetaData::Resolution, resolution); + } +} + +QMediaMetaData capsToMetaData(const QGstCaps &caps) +{ + QMediaMetaData metadata; + extendMetaDataFromCaps(metadata, caps); + return metadata; +} QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h index 53b29f548..f04a9aba9 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamermetadata_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMETADATA_H #define QGSTREAMERMETADATA_H @@ -52,21 +16,19 @@ // #include <qmediametadata.h> -#include <qvariant.h> -#include <gst/gst.h> +#include "qgst_p.h" QT_BEGIN_NAMESPACE -class QGstreamerMetaData : public QMediaMetaData -{ -public: - static QGstreamerMetaData fromGstTagList(const GstTagList *tags); - GstTagList *toGstTagList() const; +QMediaMetaData taglistToMetaData(const QGstTagListHandle &); +void extendMetaDataFromTagList(QMediaMetaData &, const QGstTagListHandle &); - void setMetaData(GstBin *bin) const; - void setMetaData(GstElement *element) const; -}; +QMediaMetaData capsToMetaData(const QGstCaps &); +void extendMetaDataFromCaps(QMediaMetaData &, const QGstCaps &); + +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstBin &); +void applyMetaDataToTagSetter(const QMediaMetaData &metadata, const QGstElement &); QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp index d064f593e..40ff5cd85 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput.cpp @@ -1,194 +1,174 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qgstreamervideooutput_p.h> -#include <qgstreamervideosink_p.h> -#include <qgstsubtitlesink_p.h> -#include <qvideosink.h> +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/qvideosink.h> #include <QtCore/qloggingcategory.h> -#include <qthread.h> +#include <QtCore/qthread.h> + +#include <common/qgstreamervideooutput_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgstsubtitlesink_p.h> -Q_LOGGING_CATEGORY(qLcMediaVideoOutput, "qt.multimedia.videooutput") +static Q_LOGGING_CATEGORY(qLcMediaVideoOutput, "qt.multimedia.videooutput") QT_BEGIN_NAMESPACE +static QGstElement makeVideoConvertScale(const char *name) +{ + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + if (factory) // videoconvertscale is only available in gstreamer 1.20 + return QGstElement::createFromFactory(factory, name); + + return QGstBin::createFromPipelineDescription("videoconvert ! videoscale", name, + /*ghostUnlinkedPads=*/true); +} + +QMaybe<QGstreamerVideoOutput *> QGstreamerVideoOutput::create(QObject *parent) +{ + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + + static std::optional<QString> elementCheck = []() -> std::optional<QString> { + std::optional<QString> error = qGstErrorMessageIfElementsNotAvailable("fakesink", "queue"); + if (error) + return error; + + QGstElementFactoryHandle factory = QGstElement::findFactory("videoconvertscale"); + if (factory) + return std::nullopt; + + return qGstErrorMessageIfElementsNotAvailable("videoconvert", "videoscale"); + }(); + + if (elementCheck) + return *elementCheck; + + return new QGstreamerVideoOutput(parent); +} + QGstreamerVideoOutput::QGstreamerVideoOutput(QObject *parent) : QObject(parent), - gstVideoOutput("videoOutput") + m_outputBin{ + QGstBin::create("videoOutput"), + }, + m_videoQueue{ + QGstElement::createFromFactory("queue", "videoQueue"), + }, + m_videoConvertScale{ + makeVideoConvertScale("videoConvertScale"), + }, + m_videoSink{ + QGstElement::createFromFactory("fakesink", "fakeVideoSink"), + } { - videoQueue = QGstElement("queue", "videoQueue"); - videoConvert = QGstElement("videoconvert", "videoConvert"); - videoSink = QGstElement("fakesink", "fakeVideoSink"); - videoSink.set("sync", true); - gstVideoOutput.add(videoQueue, videoConvert, videoSink); - if (!videoQueue.link(videoConvert, videoSink)) - qCDebug(qLcMediaVideoOutput) << ">>>>>> linking failed"; - - gstVideoOutput.addGhostPad(videoQueue, "sink"); + m_videoSink.set("sync", true); + m_videoSink.set("async", false); // no asynchronous state changes + + m_outputBin.add(m_videoQueue, m_videoConvertScale, m_videoSink); + qLinkGstElements(m_videoQueue, m_videoConvertScale, m_videoSink); + + m_subtitleSink = QGstSubtitleSink::createSink(this); + + m_outputBin.addGhostPad(m_videoQueue, "sink"); } QGstreamerVideoOutput::~QGstreamerVideoOutput() { - gstVideoOutput.setStateSync(GST_STATE_NULL); + QObject::disconnect(m_subtitleConnection); + m_outputBin.setStateSync(GST_STATE_NULL); } void QGstreamerVideoOutput::setVideoSink(QVideoSink *sink) { auto *gstVideoSink = sink ? static_cast<QGstreamerVideoSink *>(sink->platformVideoSink()) : nullptr; - if (gstVideoSink == m_videoSink) + if (gstVideoSink == m_platformVideoSink) return; - if (m_videoSink) - m_videoSink->setPipeline({}); - - m_videoSink = gstVideoSink; - if (m_videoSink) - m_videoSink->setPipeline(gstPipeline); - - QGstElement gstSink; - if (m_videoSink) { - gstSink = m_videoSink->gstSink(); - isFakeSink = false; + m_platformVideoSink = gstVideoSink; + if (m_platformVideoSink) { + m_platformVideoSink->setActive(m_isActive); + if (m_nativeSize.isValid()) + m_platformVideoSink->setNativeSize(m_nativeSize); + } + QGstElement videoSink; + if (m_platformVideoSink) { + videoSink = m_platformVideoSink->gstSink(); } else { - gstSink = QGstElement("fakesink", "fakevideosink"); - gstSink.set("sync", true); - isFakeSink = true; + videoSink = QGstElement::createFromFactory("fakesink", "fakevideosink"); + Q_ASSERT(videoSink); + videoSink.set("sync", true); + videoSink.set("async", false); // no asynchronous state changes } - if (videoSink == gstSink) - return; - - gstPipeline.beginConfig(); - if (!videoSink.isNull()) { - gstVideoOutput.remove(videoSink); - videoSink.setStateSync(GST_STATE_NULL); + QObject::disconnect(m_subtitleConnection); + if (sink) { + m_subtitleConnection = QObject::connect(this, &QGstreamerVideoOutput::subtitleChanged, sink, + [sink](const QString &subtitle) { + sink->setSubtitleText(subtitle); + }); + sink->setSubtitleText(m_lastSubtitleString); } - videoSink = gstSink; - gstVideoOutput.add(videoSink); - videoConvert.link(videoSink); - GstEvent *event = gst_event_new_reconfigure(); - gst_element_send_event(videoSink.element(), event); - videoSink.syncStateWithParent(); + if (m_videoSink == videoSink) + return; + + m_pipeline.modifyPipelineWhileNotRunning([&] { + if (!m_videoSink.isNull()) + m_outputBin.stopAndRemoveElements(m_videoSink); - doLinkSubtitleStream(); + m_videoSink = videoSink; + m_outputBin.add(m_videoSink); - gstPipeline.endConfig(); + qLinkGstElements(m_videoConvertScale, m_videoSink); - qCDebug(qLcMediaVideoOutput) << "sinkChanged" << gstSink.name(); + GstEvent *event = gst_event_new_reconfigure(); + gst_element_send_event(m_videoSink.element(), event); + m_videoSink.syncStateWithParent(); + }); - GST_DEBUG_BIN_TO_DOT_FILE(gstPipeline.bin(), - GstDebugGraphDetails(/*GST_DEBUG_GRAPH_SHOW_ALL |*/ GST_DEBUG_GRAPH_SHOW_MEDIA_TYPE | - GST_DEBUG_GRAPH_SHOW_NON_DEFAULT_PARAMS | GST_DEBUG_GRAPH_SHOW_STATES), - videoSink.name()); + qCDebug(qLcMediaVideoOutput) << "sinkChanged" << videoSink.name(); + m_pipeline.dumpGraph(m_videoSink.name().constData()); } void QGstreamerVideoOutput::setPipeline(const QGstPipeline &pipeline) { - gstPipeline = pipeline; - if (m_videoSink) - m_videoSink->setPipeline(gstPipeline); + m_pipeline = pipeline; } -void QGstreamerVideoOutput::linkSubtitleStream(QGstElement src) +void QGstreamerVideoOutput::setActive(bool isActive) { - qCDebug(qLcMediaVideoOutput) << "link subtitle stream" << src.isNull(); - if (src == subtitleSrc) + if (m_isActive == isActive) return; - gstPipeline.beginConfig(); - subtitleSrc = src; - doLinkSubtitleStream(); - gstPipeline.endConfig(); + m_isActive = isActive; + if (m_platformVideoSink) + m_platformVideoSink->setActive(isActive); } -void QGstreamerVideoOutput::unlinkSubtitleStream() +void QGstreamerVideoOutput::updateNativeSize() { - if (subtitleSrc.isNull()) + if (!m_platformVideoSink) return; - qCDebug(qLcMediaVideoOutput) << "unlink subtitle stream"; - subtitleSrc = {}; - if (!subtitleSink.isNull()) { - gstPipeline.beginConfig(); - gstPipeline.remove(subtitleSink); - gstPipeline.endConfig(); - subtitleSink.setStateSync(GST_STATE_NULL); - subtitleSink = {}; - } - if (m_videoSink) - m_videoSink->setSubtitleText({}); -} -void QGstreamerVideoOutput::doLinkSubtitleStream() -{ - if (!subtitleSink.isNull()) { - gstPipeline.remove(subtitleSink); - subtitleSink.setStateSync(GST_STATE_NULL); - subtitleSink = {}; - } - if (!m_videoSink || subtitleSrc.isNull()) - return; - if (subtitleSink.isNull()) { - subtitleSink = m_videoSink->subtitleSink(); - gstPipeline.add(subtitleSink); - } - if (!subtitleSrc.link(subtitleSink)) - qCDebug(qLcMediaVideoOutput) << "link subtitle stream failed"; + m_platformVideoSink->setNativeSize(qRotatedFrameSize(m_nativeSize, m_rotation)); } void QGstreamerVideoOutput::setIsPreview() { // configures the queue to be fast and lightweight for camera preview // also avoids blocking the queue in case we have an encodebin attached to the tee as well - videoQueue.set("leaky", 2 /*downstream*/); - videoQueue.set("silent", true); - videoQueue.set("max-size-buffers", uint(1)); - videoQueue.set("max-size-bytes", uint(0)); - videoQueue.set("max-size-time", quint64(0)); + m_videoQueue.set("leaky", 2 /*downstream*/); + m_videoQueue.set("silent", true); + m_videoQueue.set("max-size-buffers", uint(1)); + m_videoQueue.set("max-size-bytes", uint(0)); + m_videoQueue.set("max-size-time", quint64(0)); } void QGstreamerVideoOutput::flushSubtitles() { - if (!subtitleSink.isNull()) { - auto pad = subtitleSink.staticPad("sink"); + if (!m_subtitleSink.isNull()) { + auto pad = m_subtitleSink.staticPad("sink"); auto *event = gst_event_new_flush_start(); pad.sendEvent(event); event = gst_event_new_flush_stop(false); @@ -196,4 +176,28 @@ void QGstreamerVideoOutput::flushSubtitles() } } +void QGstreamerVideoOutput::setNativeSize(QSize sz) +{ + m_nativeSize = sz; + updateNativeSize(); +} + +void QGstreamerVideoOutput::setRotation(QtVideo::Rotation rot) +{ + m_rotation = rot; + updateNativeSize(); +} + +void QGstreamerVideoOutput::updateSubtitle(QString string) +{ + // GStreamer thread + + QMetaObject::invokeMethod(this, [this, string = std::move(string)]() mutable { + m_lastSubtitleString = string; + Q_EMIT subtitleChanged(std::move(string)); + }); +} + QT_END_NAMESPACE + +#include "moc_qgstreamervideooutput_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h index bf6bc5497..a460745f4 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooutput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERVIDEOOUTPUT_P_H #define QGSTREAMERVIDEOOUTPUT_P_H @@ -53,53 +17,70 @@ #include <QtCore/qobject.h> #include <private/qtmultimediaglobal_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> +#include <private/qmultimediautils_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgstsubtitlesink_p.h> #include <qwaitcondition.h> #include <qmutex.h> #include <qpointer.h> -#include <qgstreamervideosink_p.h> QT_BEGIN_NAMESPACE class QVideoSink; -class Q_MULTIMEDIA_EXPORT QGstreamerVideoOutput : public QObject +class QGstreamerVideoOutput : public QObject, QAbstractSubtitleObserver { Q_OBJECT public: - QGstreamerVideoOutput(QObject *parent = 0); + static QMaybe<QGstreamerVideoOutput *> create(QObject *parent = nullptr); ~QGstreamerVideoOutput(); void setVideoSink(QVideoSink *sink); - QGstreamerVideoSink *gstreamerVideoSink() const { return m_videoSink; } + QGstreamerVideoSink *gstreamerVideoSink() const { return m_platformVideoSink; } void setPipeline(const QGstPipeline &pipeline); - QGstElement gstElement() const { return gstVideoOutput; } - void linkSubtitleStream(QGstElement subtitleSrc); - void unlinkSubtitleStream(); + QGstElement gstElement() const { return m_outputBin; } + QGstElement gstSubtitleElement() const { return m_subtitleSink; } + + void setActive(bool); void setIsPreview(); void flushSubtitles(); + void setNativeSize(QSize); + void setRotation(QtVideo::Rotation); + + void updateSubtitle(QString) override; + +signals: + void subtitleChanged(QString); + private: - void doLinkSubtitleStream(); + explicit QGstreamerVideoOutput(QObject *parent); + + void updateNativeSize(); - QPointer<QGstreamerVideoSink> m_videoSink; - bool isFakeSink = true; + QPointer<QGstreamerVideoSink> m_platformVideoSink; // Gst elements - QGstPipeline gstPipeline; + QGstPipeline m_pipeline; + + QGstBin m_outputBin; + QGstElement m_videoQueue; + QGstElement m_videoConvertScale; + QGstElement m_videoSink; - QGstBin gstVideoOutput; - QGstElement videoQueue; - QGstElement videoConvert; - QGstElement videoSink; + QGstElement m_subtitleSink; + QMetaObject::Connection m_subtitleConnection; + QString m_lastSubtitleString; - QGstElement subtitleSrc; - QGstElement subtitleSink; + bool m_isActive{ false }; + QSize m_nativeSize; + QtVideo::Rotation m_rotation{}; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp index f83f1d518..6ca23006b 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay.cpp @@ -1,70 +1,34 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamervideooverlay_p.h" #include <QtGui/qguiapplication.h> -#include "qgstutils_p.h" -#include "qgst_p.h" -#include "qgstreamermessage_p.h" -#include "qgstreamervideosink_p.h" +#include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include <gst/video/videooverlay.h> +#include <common/qglist_helper_p.h> +#include <common/qgst_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgstutils_p.h> -#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <gst/video/videooverlay.h> QT_BEGIN_NAMESPACE struct ElementMap { - const char *qtPlatform; - const char *gstreamerElement; + QStringView qtPlatform; + const char *gstreamerElement = nullptr; }; // Ordered by descending priority -static constexpr ElementMap elementMap[] = -{ - { "xcb", "xvimagesink" }, - { "xcb", "ximagesink" }, +static constexpr ElementMap elementMap[] = { + { u"xcb", "xvimagesink" }, + { u"xcb", "ximagesink" }, // wayland - { "wayland", "waylandsink" } + { u"wayland", "waylandsink" }, }; static bool qt_gst_element_is_functioning(QGstElement element) @@ -80,13 +44,14 @@ static bool qt_gst_element_is_functioning(QGstElement element) static QGstElement findBestVideoSink() { + using namespace Qt::StringLiterals; QString platform = QGuiApplication::platformName(); // First, try some known video sinks, depending on the Qt platform plugin in use. - for (auto i : elementMap) { - if (platform != QLatin1String(i.qtPlatform)) + for (const auto &i : elementMap) { + if (platform != i.qtPlatform) continue; - QGstElement choice(i.gstreamerElement, i.gstreamerElement); + QGstElement choice = QGstElement::createFromFactory(i.gstreamerElement, i.gstreamerElement); if (choice.isNull()) continue; @@ -96,20 +61,18 @@ static QGstElement findBestVideoSink() // We need a native window ID to use the GstVideoOverlay interface. // Bail out if the Qt platform plugin in use cannot provide a sensible WId. - if (platform != QLatin1String("xcb") && platform != QLatin1String("wayland")) + if (platform != QStringView{ u"xcb" } && platform != QStringView{ u"wayland" }) return {}; QGstElement choice; // If none of the known video sinks are available, try to find one that implements the // GstVideoOverlay interface and has autoplugging rank. GList *list = qt_gst_video_sinks(); - for (GList *item = list; item != nullptr; item = item->next) { - GstElementFactory *f = GST_ELEMENT_FACTORY(item->data); - + for (GstElementFactory *f : QGstUtils::GListRangeAdaptor<GstElementFactory *>(list)) { if (!gst_element_factory_has_interface(f, "GstVideoOverlay")) continue; - choice = QGstElement(gst_element_factory_create(f, nullptr)); + choice = QGstElement::createFromFactory(f, nullptr); if (choice.isNull()) continue; @@ -132,7 +95,7 @@ QGstreamerVideoOverlay::QGstreamerVideoOverlay(QGstreamerVideoSink *parent, cons { QGstElement sink; if (!elementName.isEmpty()) - sink = QGstElement(elementName.constData(), nullptr); + sink = QGstElement::createFromFactory(elementName.constData()); else sink = findBestVideoSink(); @@ -157,7 +120,7 @@ void QGstreamerVideoOverlay::setVideoSink(QGstElement sink) if (sink.isNull()) return; - m_videoSink = sink; + m_videoSink = std::move(sink); QGstPad pad = m_videoSink.staticPad("sink"); addProbeToPad(pad.pad()); @@ -220,7 +183,7 @@ void QGstreamerVideoOverlay::applyRenderRect() void QGstreamerVideoOverlay::probeCaps(GstCaps *caps) { - QSize size = QGstCaps(caps).at(0).resolution(); + QSize size = QGstCaps(caps, QGstCaps::NeedsRef).at(0).resolution(); if (size != m_nativeVideoSize) { m_nativeVideoSize = size; m_gstreamerVideoSink->setNativeSize(m_nativeVideoSize); @@ -244,10 +207,12 @@ void QGstreamerVideoOverlay::setFullScreen(bool fullscreen) bool QGstreamerVideoOverlay::processSyncMessage(const QGstreamerMessage &message) { - if (!gst_is_video_overlay_prepare_window_handle_message(message.rawMessage())) + if (!gst_is_video_overlay_prepare_window_handle_message(message.message())) return false; gst_video_overlay_set_window_handle(GST_VIDEO_OVERLAY(m_videoSink.object()), m_windowId); return true; } QT_END_NAMESPACE + +#include "moc_qgstreamervideooverlay_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h index c58b54d3c..588e8b5e4 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideooverlay_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERVIDEOOVERLAY_P_H #define QGSTREAMERVIDEOOVERLAY_P_H @@ -51,22 +15,22 @@ // We mean it. // -#include <qgstpipeline_p.h> -#include <qgstreamerbufferprobe_p.h> -#include <qgst_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamerbufferprobe_p.h> +#include <common/qgst_p.h> #include <QtGui/qwindowdefs.h> QT_BEGIN_NAMESPACE class QGstreamerVideoSink; -class Q_MULTIMEDIA_EXPORT QGstreamerVideoOverlay - : public QObject - , public QGstreamerSyncMessageFilter - , private QGstreamerBufferProbe +class QGstreamerVideoOverlay : public QObject, + public QGstreamerSyncMessageFilter, + private QGstreamerBufferProbe { Q_OBJECT public: - explicit QGstreamerVideoOverlay(QGstreamerVideoSink *parent = 0, const QByteArray &elementName = QByteArray()); + explicit QGstreamerVideoOverlay(QGstreamerVideoSink *parent = nullptr, + const QByteArray &elementName = QByteArray()); virtual ~QGstreamerVideoOverlay(); QGstElement videoSink() const; diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp index dc439f71e..05d1fe0ed 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink.cpp @@ -1,124 +1,146 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamervideosink_p.h" -#include "qgstvideorenderersink_p.h" -#include "qgstsubtitlesink_p.h" -#include <qgstutils_p.h> -#include <QtGui/private/qrhi_p.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <common/qgstreamervideosink_p.h> +#include <common/qgstvideorenderersink_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstutils_p.h> +#include <rhi/qrhi.h> + +#include <QtCore/qdebug.h> +#include <QtCore/qloggingcategory.h> #if QT_CONFIG(gstreamer_gl) -#include <QtGui/private/qrhigles2_p.h> -#include <QGuiApplication> -#include <QtGui/qopenglcontext.h> -#include <QWindow> -#include <qpa/qplatformnativeinterface.h> -#include <gst/gl/gstglconfig.h> +# include <QtGui/QGuiApplication> +# include <QtGui/qopenglcontext.h> +# include <QtGui/QWindow> +# include <QtGui/qpa/qplatformnativeinterface.h> +# include <gst/gl/gstglconfig.h> -#if GST_GL_HAVE_WINDOW_X11 && __has_include("X11/Xlib-xcb.h") +# if GST_GL_HAVE_WINDOW_X11 && __has_include("X11/Xlib-xcb.h") # include <gst/gl/x11/gstgldisplay_x11.h> -#endif -#if GST_GL_HAVE_PLATFORM_EGL +# endif +# if GST_GL_HAVE_PLATFORM_EGL # include <gst/gl/egl/gstgldisplay_egl.h> # include <EGL/egl.h> # include <EGL/eglext.h> -#endif -#if GST_GL_HAVE_WINDOW_WAYLAND && __has_include("wayland-client.h") +# endif +# if GST_GL_HAVE_WINDOW_WAYLAND && __has_include("wayland-client.h") # include <gst/gl/wayland/gstgldisplay_wayland.h> -#endif +# endif #endif // #if QT_CONFIG(gstreamer_gl) -#include <QtCore/qdebug.h> - -#include <QtCore/qloggingcategory.h> - QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcMediaVideoSink, "qt.multimedia.videosink") +static Q_LOGGING_CATEGORY(qLcGstVideoSink, "qt.multimedia.gstvideosink"); QGstreamerVideoSink::QGstreamerVideoSink(QVideoSink *parent) - : QPlatformVideoSink(parent) + : QPlatformVideoSink{ + parent, + }, + m_sinkBin{ + QGstBin::create("videoSinkBin"), + } { - sinkBin = QGstBin("videoSinkBin"); - // This is a hack for some iMX platforms. Thos require the use of a special video + // This is a hack for some iMX and NVidia platforms. These require the use of a special video // conversion element in the pipeline before the video sink, as they unfortunately - // output some proprietary format from the decoder even though it's marked as + // output some proprietary format from the decoder even though it's sometimes marked as // a regular supported video/x-raw format. // // To fix this, simply insert the element into the pipeline if it's available. Otherwise // we simply use an identity element. - gstQueue = QGstElement("queue"); - auto imxVideoConvert = QGstElement("imxvideoconvert_g2d"); - if (!imxVideoConvert.isNull()) - gstPreprocess = imxVideoConvert; - else - gstPreprocess = QGstElement("identity"); - sinkBin.add(gstQueue, gstPreprocess); - gstQueue.link(gstPreprocess); - sinkBin.addGhostPad(gstQueue, "sink"); - - gstSubtitleSink = GST_ELEMENT(QGstSubtitleSink::createSink(this)); + QGstElementFactoryHandle factory; + + // QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT allows users to override the + // conversion element. Ideally we construct the element programatically, though. + QByteArray preprocessOverride = qgetenv("QT_GSTREAMER_OVERRIDE_VIDEO_CONVERSION_ELEMENT"); + if (!preprocessOverride.isEmpty()) { + qCDebug(qLcGstVideoSink) << "requesting conversion element from environment:" + << preprocessOverride; + + m_gstPreprocess = QGstBin::createFromPipelineDescription(preprocessOverride, nullptr, + /*ghostUnlinkedPads=*/true); + if (!m_gstPreprocess) + qCWarning(qLcGstVideoSink) << "Cannot create conversion element:" << preprocessOverride; + } + + if (!m_gstPreprocess) { + // This is a hack for some iMX and NVidia platforms. These require the use of a special + // video conversion element in the pipeline before the video sink, as they unfortunately + // output some proprietary format from the decoder even though it's sometimes marked as + // a regular supported video/x-raw format. + static constexpr auto decodersToTest = { + "imxvideoconvert_g2d", + "nvvidconv", + }; + + for (const char *decoder : decodersToTest) { + factory = QGstElement::findFactory(decoder); + if (factory) + break; + } + + if (factory) { + qCDebug(qLcGstVideoSink) + << "instantiating conversion element:" + << g_type_name(gst_element_factory_get_element_type(factory.get())); + + m_gstPreprocess = QGstElement::createFromFactory(factory, "preprocess"); + } + } + + bool disablePixelAspectRatio = + qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO"); + if (disablePixelAspectRatio) { + // Enabling the pixel aspect ratio may expose a gstreamer bug on cameras that don't expose a + // pixel-aspect-ratio via `VIDIOC_CROPCAP`. This can cause the caps negotiation to fail. + // Using the QT_GSTREAMER_DISABLE_PIXEL_ASPECT_RATIO environment variable, one can disable + // pixel-aspect-ratio handling + // + // compare: https://gitlab.freedesktop.org/gstreamer/gstreamer/-/merge_requests/6242 + m_gstCapsFilter = + QGstElement::createFromFactory("identity", "nullPixelAspectRatioCapsFilter"); + } else { + m_gstCapsFilter = + QGstElement::createFromFactory("capsfilter", "pixelAspectRatioCapsFilter"); + QGstCaps capsFilterCaps{ + gst_caps_new_simple("video/x-raw", "pixel-aspect-ratio", GST_TYPE_FRACTION, 1, 1, NULL), + QGstCaps::HasRef, + }; + g_object_set(m_gstCapsFilter.element(), "caps", capsFilterCaps.caps(), NULL); + } + + if (m_gstPreprocess) { + m_sinkBin.add(m_gstPreprocess, m_gstCapsFilter); + qLinkGstElements(m_gstPreprocess, m_gstCapsFilter); + m_sinkBin.addGhostPad(m_gstPreprocess, "sink"); + } else { + m_sinkBin.add(m_gstCapsFilter); + m_sinkBin.addGhostPad(m_gstCapsFilter, "sink"); + } } QGstreamerVideoSink::~QGstreamerVideoSink() { - unrefGstContexts(); + emit aboutToBeDestroyed(); - setPipeline(QGstPipeline()); + unrefGstContexts(); } QGstElement QGstreamerVideoSink::gstSink() { updateSinkElement(); - return sinkBin; + return m_sinkBin; } -void QGstreamerVideoSink::setPipeline(QGstPipeline pipeline) +void QGstreamerVideoSink::setActive(bool isActive) { - gstPipeline = pipeline; -} + if (m_isActive == isActive) + return; + m_isActive = isActive; -bool QGstreamerVideoSink::inStoppedState() const -{ - if (gstPipeline.isNull()) - return true; - return gstPipeline.inStoppedState(); + if (m_gstQtSink) + m_gstQtSink.setActive(isActive); } void QGstreamerVideoSink::setRhi(QRhi *rhi) @@ -130,59 +152,65 @@ void QGstreamerVideoSink::setRhi(QRhi *rhi) m_rhi = rhi; updateGstContexts(); - if (!gstQtSink.isNull()) { - // force creation of a new sink with proper caps + if (!m_gstQtSink.isNull()) { + // force creation of a new sink with proper caps. createQtSink(); updateSinkElement(); + + QGstPipeline pipeline = m_sinkBin.getPipeline(); + if (pipeline) + pipeline.flush(); // the caps may change, so we need to flush the pipeline. } } void QGstreamerVideoSink::createQtSink() { - gstQtSink = QGstElement(reinterpret_cast<GstElement *>(QGstVideoRendererSink::createSink(this))); + if (m_gstQtSink) + m_gstQtSink.setStateSync(GST_STATE_NULL); + + m_gstQtSink = QGstVideoRendererSink::createSink(this); + if (m_gstQtSink) + m_gstQtSink.setActive(m_isActive); } void QGstreamerVideoSink::updateSinkElement() { QGstElement newSink; - if (gstQtSink.isNull()) + if (m_gstQtSink.isNull()) createQtSink(); - newSink = gstQtSink; - if (newSink == gstVideoSink) + newSink = m_gstQtSink; + + if (newSink == m_gstVideoSink) return; - gstPipeline.beginConfig(); + QGstPipeline::modifyPipelineWhileNotRunning(m_sinkBin.getPipeline(), [&] { + if (!m_gstVideoSink.isNull()) + m_sinkBin.stopAndRemoveElements(m_gstVideoSink); - if (!gstVideoSink.isNull()) { - gstVideoSink.setStateSync(GST_STATE_NULL); - sinkBin.remove(gstVideoSink); - } + newSink.set("async", false); // no asynchronous state changes - gstVideoSink = newSink; - sinkBin.add(gstVideoSink); - if (!gstPreprocess.link(gstVideoSink)) - qCDebug(qLcMediaVideoSink) << "couldn't link preprocess and sink"; - gstVideoSink.setState(GST_STATE_PAUSED); + m_gstVideoSink = newSink; + m_sinkBin.add(m_gstVideoSink); + qLinkGstElements(m_gstCapsFilter, m_gstVideoSink); + m_gstVideoSink.syncStateWithParent(); + }); - gstPipeline.endConfig(); - gstPipeline.dumpGraph("updateVideoSink"); + m_sinkBin.dumpPipelineGraph("updateVideoSink"); } void QGstreamerVideoSink::unrefGstContexts() { - if (m_gstGlDisplayContext) - gst_context_unref(m_gstGlDisplayContext); - m_gstGlDisplayContext = nullptr; - if (m_gstGlLocalContext) - gst_context_unref(m_gstGlLocalContext); - m_gstGlLocalContext = nullptr; + m_gstGlDisplayContext.close(); + m_gstGlLocalContext.close(); m_eglDisplay = nullptr; m_eglImageTargetTexture2D = nullptr; } void QGstreamerVideoSink::updateGstContexts() { + using namespace Qt::Literals; + unrefGstContexts(); #if QT_CONFIG(gstreamer_gl) @@ -195,34 +223,38 @@ void QGstreamerVideoSink::updateGstContexts() const QString platform = QGuiApplication::platformName(); QPlatformNativeInterface *pni = QGuiApplication::platformNativeInterface(); - m_eglDisplay = pni->nativeResourceForIntegration("egldisplay"); + m_eglDisplay = pni->nativeResourceForIntegration("egldisplay"_ba); // qDebug() << "platform is" << platform << m_eglDisplay; - GstGLDisplay *gstGlDisplay = nullptr; - const char *contextName = "eglcontext"; + QGstGLDisplayHandle gstGlDisplay; + + QByteArray contextName = "eglcontext"_ba; GstGLPlatform glPlatform = GST_GL_PLATFORM_EGL; // use the egl display if we have one if (m_eglDisplay) { #if GST_GL_HAVE_PLATFORM_EGL - gstGlDisplay = (GstGLDisplay *)gst_gl_display_egl_new_with_egl_display(m_eglDisplay); + gstGlDisplay.reset( + GST_GL_DISPLAY_CAST(gst_gl_display_egl_new_with_egl_display(m_eglDisplay))); m_eglImageTargetTexture2D = eglGetProcAddress("glEGLImageTargetTexture2DOES"); #endif } else { - auto display = pni->nativeResourceForIntegration("display"); + auto display = pni->nativeResourceForIntegration("display"_ba); if (display) { #if GST_GL_HAVE_WINDOW_X11 && __has_include("X11/Xlib-xcb.h") if (platform == QLatin1String("xcb")) { - contextName = "glxcontext"; + contextName = "glxcontext"_ba; glPlatform = GST_GL_PLATFORM_GLX; - gstGlDisplay = (GstGLDisplay *)gst_gl_display_x11_new_with_display((Display *)display); + gstGlDisplay.reset(GST_GL_DISPLAY_CAST( + gst_gl_display_x11_new_with_display(reinterpret_cast<Display *>(display)))); } #endif #if GST_GL_HAVE_WINDOW_WAYLAND && __has_include("wayland-client.h") if (platform.startsWith(QLatin1String("wayland"))) { Q_ASSERT(!gstGlDisplay); - gstGlDisplay = (GstGLDisplay *)gst_gl_display_wayland_new_with_display((struct wl_display *)display); + gstGlDisplay.reset(GST_GL_DISPLAY_CAST(gst_gl_display_wayland_new_with_display( + reinterpret_cast<struct wl_display *>(display)))); } #endif } @@ -238,33 +270,43 @@ void QGstreamerVideoSink::updateGstContexts() qWarning() << "Could not find resource for" << contextName; GstGLAPI glApi = QOpenGLContext::openGLModuleType() == QOpenGLContext::LibGL ? GST_GL_API_OPENGL : GST_GL_API_GLES2; - GstGLContext *appContext = gst_gl_context_new_wrapped(gstGlDisplay, (guintptr)nativeContext, glPlatform, glApi); + QGstGLContextHandle appContext{ + gst_gl_context_new_wrapped(gstGlDisplay.get(), guintptr(nativeContext), glPlatform, glApi), + }; if (!appContext) qWarning() << "Could not create wrappped context for platform:" << glPlatform; - GstGLContext *displayContext = nullptr; - GError *error = nullptr; - gst_gl_display_create_context(gstGlDisplay, appContext, &displayContext, &error); + gst_gl_context_activate(appContext.get(), true); + + QUniqueGErrorHandle error; + gst_gl_context_fill_info(appContext.get(), &error); if (error) { - qWarning() << "Could not create display context:" << error->message; - g_clear_error(&error); + qWarning() << "Could not fill context info:" << error; + error = {}; } - if (appContext) - gst_object_unref(appContext); + QGstGLContextHandle displayContext; + gst_gl_display_create_context(gstGlDisplay.get(), appContext.get(), &displayContext, &error); + if (error) + qWarning() << "Could not create display context:" << error; + + appContext.close(); - m_gstGlDisplayContext = gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, false); - gst_context_set_gl_display(m_gstGlDisplayContext, gstGlDisplay); - gst_object_unref(gstGlDisplay); + m_gstGlDisplayContext.reset(gst_context_new(GST_GL_DISPLAY_CONTEXT_TYPE, false)); + gst_context_set_gl_display(m_gstGlDisplayContext.get(), gstGlDisplay.get()); - m_gstGlLocalContext = gst_context_new("gst.gl.local_context", false); - GstStructure *structure = gst_context_writable_structure(m_gstGlLocalContext); - gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, displayContext, nullptr); - gst_object_unref(displayContext); + m_gstGlLocalContext.reset(gst_context_new("gst.gl.local_context", false)); + GstStructure *structure = gst_context_writable_structure(m_gstGlLocalContext.get()); + gst_structure_set(structure, "context", GST_TYPE_GL_CONTEXT, displayContext.get(), nullptr); + displayContext.close(); - if (!gstPipeline.isNull()) - gst_element_set_context(gstPipeline.element(), m_gstGlLocalContext); + QGstPipeline pipeline = m_sinkBin.getPipeline(); + + if (pipeline) + gst_element_set_context(pipeline.element(), m_gstGlLocalContext.get()); #endif // #if QT_CONFIG(gstreamer_gl) } QT_END_NAMESPACE + +#include "moc_qgstreamervideosink_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h index 216dd61ee..b892ac752 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstreamervideosink_p.h @@ -1,44 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef QGSTREAMERVIDEOWINDOW_H -#define QGSTREAMERVIDEOWINDOW_H +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QGSTREAMERVIDEOSINK_H +#define QGSTREAMERVIDEOSINK_H // // W A R N I N G @@ -51,44 +15,37 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <private/qplatformvideosink_p.h> - -#include <qgstpipeline_p.h> -#include <qgstreamervideooverlay_p.h> -#include <QtGui/qcolor.h> -#include <qvideosink.h> +#include <QtMultimedia/qvideosink.h> +#include <QtMultimedia/private/qplatformvideosink_p.h> -#if QT_CONFIG(gstreamer_gl) -#include <gst/gl/gl.h> -#endif +#include <common/qgstvideorenderersink_p.h> +#include <common/qgstpipeline_p.h> QT_BEGIN_NAMESPACE -class QGstreamerVideoRenderer; -class QVideoWindow; -class Q_MULTIMEDIA_EXPORT QGstreamerVideoSink - : public QPlatformVideoSink +class QGstreamerVideoSink : public QPlatformVideoSink { Q_OBJECT + public: - explicit QGstreamerVideoSink(QVideoSink *parent = 0); + explicit QGstreamerVideoSink(QVideoSink *parent = nullptr); ~QGstreamerVideoSink(); void setRhi(QRhi *rhi) override; QRhi *rhi() const { return m_rhi; } QGstElement gstSink(); - QGstElement subtitleSink() const { return gstSubtitleSink; } - void setPipeline(QGstPipeline pipeline); - bool inStoppedState() const; - - GstContext *gstGlDisplayContext() const { return m_gstGlDisplayContext; } - GstContext *gstGlLocalContext() const { return m_gstGlLocalContext; } + GstContext *gstGlDisplayContext() const { return m_gstGlDisplayContext.get(); } + GstContext *gstGlLocalContext() const { return m_gstGlLocalContext.get(); } Qt::HANDLE eglDisplay() const { return m_eglDisplay; } QFunctionPointer eglImageTargetTexture2D() const { return m_eglImageTargetTexture2D; } + void setActive(bool); + +Q_SIGNALS: + void aboutToBeDestroyed(); + private: void createQtSink(); void updateSinkElement(); @@ -96,20 +53,20 @@ private: void unrefGstContexts(); void updateGstContexts(); - QGstPipeline gstPipeline; - QGstBin sinkBin; - QGstElement gstQueue; - QGstElement gstPreprocess; - QGstElement gstVideoSink; - QGstElement gstQtSink; - QGstElement gstSubtitleSink; + QGstBin m_sinkBin; + QGstElement m_gstPreprocess; + QGstElement m_gstCapsFilter; + QGstElement m_gstVideoSink; + QGstVideoRendererSinkElement m_gstQtSink; QRhi *m_rhi = nullptr; + bool m_isActive = true; Qt::HANDLE m_eglDisplay = nullptr; QFunctionPointer m_eglImageTargetTexture2D = nullptr; - GstContext *m_gstGlLocalContext = nullptr; - GstContext *m_gstGlDisplayContext = nullptr; + + QGstContextHandle m_gstGlLocalContext; + QGstContextHandle m_gstGlDisplayContext; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp index d9b76d57f..58b5c3f53 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink.cpp @@ -1,94 +1,64 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <QDebug> -#include <QThread> -#include <QEvent> - -#include "qgstreamervideosink_p.h" +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + #include "qgstsubtitlesink_p.h" -#include "qgstutils_p.h" +#include "qgst_debug_p.h" + +#include <QtCore/qdebug.h> QT_BEGIN_NAMESPACE -static GstBaseSinkClass *sink_parent_class; -static thread_local QGstreamerVideoSink *current_sink; +namespace { +GstBaseSinkClass *gst_sink_parent_class; +thread_local QAbstractSubtitleObserver *gst_current_observer; + +class QGstSubtitleSinkClass +{ +public: + GstBaseSinkClass parent_class; +}; + +} // namespace #define ST_SINK(s) QGstSubtitleSink *sink(reinterpret_cast<QGstSubtitleSink *>(s)) -QGstSubtitleSink *QGstSubtitleSink::createSink(QGstreamerVideoSink *sink) +QGstElement QGstSubtitleSink::createSink(QAbstractSubtitleObserver *observer) { - current_sink = sink; + gst_current_observer = observer; QGstSubtitleSink *gstSink = reinterpret_cast<QGstSubtitleSink *>( g_object_new(QGstSubtitleSink::get_type(), nullptr)); g_object_set(gstSink, "async", false, nullptr); - return gstSink; + return QGstElement{ + qGstCheckedCast<GstElement>(gstSink), + QGstElement::NeedsRef, + }; } GType QGstSubtitleSink::get_type() { - static GType type = 0; - - if (type == 0) { - static const GTypeInfo info = - { - sizeof(QGstSubtitleSinkClass), // class_size - base_init, // base_init - nullptr, // base_finalize - class_init, // class_init - nullptr, // class_finalize - nullptr, // class_data - sizeof(QGstSubtitleSink), // instance_size - 0, // n_preallocs - instance_init, // instance_init - nullptr // value_table - }; - - type = g_type_register_static( + // clang-format off + static constexpr GTypeInfo info = + { + sizeof(QGstSubtitleSinkClass), // class_size + base_init, // base_init + nullptr, // base_finalize + class_init, // class_init + nullptr, // class_finalize + nullptr, // class_data + sizeof(QGstSubtitleSink), // instance_size + 0, // n_preallocs + instance_init, // instance_init + nullptr // value_table + }; + // clang-format on + + static const GType type = []() { + const auto result = g_type_register_static( GST_TYPE_BASE_SINK, "QGstSubtitleSink", &info, GTypeFlags(0)); - - // Register the sink type to be used in custom piplines. - // When surface is ready the sink can be used. - gst_element_register(nullptr, "qtsubtitlesink", GST_RANK_PRIMARY, type); - } + return result; + }(); return type; } @@ -97,7 +67,7 @@ void QGstSubtitleSink::class_init(gpointer g_class, gpointer class_data) { Q_UNUSED(class_data); - sink_parent_class = reinterpret_cast<GstBaseSinkClass *>(g_type_class_peek_parent(g_class)); + gst_sink_parent_class = reinterpret_cast<GstBaseSinkClass *>(g_type_class_peek_parent(g_class)); GstBaseSinkClass *base_sink_class = reinterpret_cast<GstBaseSinkClass *>(g_class); base_sink_class->render = QGstSubtitleSink::render; @@ -120,57 +90,56 @@ void QGstSubtitleSink::class_init(gpointer g_class, gpointer class_data) void QGstSubtitleSink::base_init(gpointer g_class) { - static GstStaticPadTemplate sink_pad_template = GST_STATIC_PAD_TEMPLATE( - "sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS("ANY")); + static GstStaticPadTemplate sink_pad_template = + GST_STATIC_PAD_TEMPLATE("sink", GST_PAD_SINK, GST_PAD_ALWAYS, GST_STATIC_CAPS("ANY")); gst_element_class_add_pad_template( GST_ELEMENT_CLASS(g_class), gst_static_pad_template_get(&sink_pad_template)); } -void QGstSubtitleSink::instance_init(GTypeInstance *instance, gpointer g_class) +void QGstSubtitleSink::instance_init(GTypeInstance *instance, gpointer /*g_class*/) { - Q_UNUSED(g_class); ST_SINK(instance); - Q_ASSERT(current_sink); - sink->sink = current_sink; - current_sink = nullptr; + Q_ASSERT(gst_current_observer); + sink->observer = gst_current_observer; + gst_current_observer = nullptr; } void QGstSubtitleSink::finalize(GObject *object) { // Chain up - G_OBJECT_CLASS(sink_parent_class)->finalize(object); + G_OBJECT_CLASS(gst_sink_parent_class)->finalize(object); } GstStateChangeReturn QGstSubtitleSink::change_state(GstElement *element, GstStateChange transition) { - return GST_ELEMENT_CLASS(sink_parent_class)->change_state(element, transition); + return GST_ELEMENT_CLASS(gst_sink_parent_class)->change_state(element, transition); } GstCaps *QGstSubtitleSink::get_caps(GstBaseSink *base, GstCaps *filter) { - return sink_parent_class->get_caps(base, filter); + return gst_sink_parent_class->get_caps(base, filter); } gboolean QGstSubtitleSink::set_caps(GstBaseSink *base, GstCaps *caps) { - qDebug() << "set_caps:" << QGstCaps(caps).toString(); - return sink_parent_class->set_caps(base, caps); + qDebug() << "set_caps:" << caps; + return gst_sink_parent_class->set_caps(base, caps); } gboolean QGstSubtitleSink::propose_allocation(GstBaseSink *base, GstQuery *query) { - return sink_parent_class->propose_allocation(base, query); + return gst_sink_parent_class->propose_allocation(base, query); } GstFlowReturn QGstSubtitleSink::wait_event(GstBaseSink *base, GstEvent *event) { - GstFlowReturn retval = sink_parent_class->wait_event(base, event); + GstFlowReturn retval = gst_sink_parent_class->wait_event(base, event); ST_SINK(base); if (event->type == GST_EVENT_GAP) { -// qDebug() << "gap, clearing subtitle"; - sink->sink->setSubtitleText(QString()); + // qDebug() << "gap, clearing subtitle"; + sink->observer->updateSubtitle(QString()); } return retval; } @@ -182,10 +151,10 @@ GstFlowReturn QGstSubtitleSink::render(GstBaseSink *base, GstBuffer *buffer) GstMapInfo info; QString subtitle; if (gst_memory_map(mem, &info, GST_MAP_READ)) - subtitle = QString::fromUtf8(info.data); + subtitle = QString::fromUtf8(reinterpret_cast<const char *>(info.data)); gst_memory_unmap(mem, &info); // qDebug() << "render" << buffer << subtitle; - sink->sink->setSubtitleText(subtitle); + sink->observer->updateSubtitle(subtitle); return GST_FLOW_OK; } diff --git a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h index 179b02a50..1970ac48b 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstsubtitlesink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTSUBTITLESINK_P_H #define QGSTSUBTITLESINK_P_H @@ -53,24 +17,25 @@ #include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include <QtCore/qlist.h> -#include <QtCore/qmutex.h> -#include <QtCore/qqueue.h> -#include <QtCore/qpointer.h> -#include <QtCore/qwaitcondition.h> -#include <qgst_p.h> +#include <QtCore/qstring.h> +#include <common/qgst_p.h> #include <gst/base/gstbasesink.h> QT_BEGIN_NAMESPACE -class QGstreamerVideoSink; +class QAbstractSubtitleObserver +{ +public: + virtual ~QAbstractSubtitleObserver() = default; + virtual void updateSubtitle(QString) = 0; +}; -class Q_MULTIMEDIA_EXPORT QGstSubtitleSink +class QGstSubtitleSink { public: - GstBaseSink parent; + GstBaseSink parent{}; - static QGstSubtitleSink *createSink(QGstreamerVideoSink *sink); + static QGstElement createSink(QAbstractSubtitleObserver *observer); private: static GType get_type(); @@ -91,14 +56,7 @@ private: static GstFlowReturn render(GstBaseSink *sink, GstBuffer *buffer); private: - QGstreamerVideoSink *sink = nullptr; -}; - - -class QGstSubtitleSinkClass -{ -public: - GstBaseSinkClass parent_class; + QAbstractSubtitleObserver *observer = nullptr; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstutils.cpp b/src/plugins/multimedia/gstreamer/common/qgstutils.cpp index 21173a5cd..8ec2bde3c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstutils.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstutils.cpp @@ -1,70 +1,18 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -#include <QtMultimedia/private/qtmultimediaglobal_p.h> -#include "qgstutils_p.h" +#include <common/qgstutils_p.h> +#include <common/qgst_p.h> -#include <QtCore/qdatetime.h> -#include <QtCore/qdir.h> -#include <QtCore/qbytearray.h> -#include <QtCore/qvariant.h> -#include <QtCore/qregularexpression.h> -#include <QtCore/qsize.h> -#include <QtCore/qset.h> -#include <QtCore/qstringlist.h> -#include <QtGui/qimage.h> -#include <qaudioformat.h> -#include <QtCore/qelapsedtimer.h> -#include <QtMultimedia/qvideoframeformat.h> -#include <private/qmultimediautils_p.h> +#include <QtMultimedia/qaudioformat.h> -#include <gst/audio/audio.h> -#include <gst/video/video.h> - -template<typename T, int N> constexpr int lengthOf(const T (&)[N]) { return N; } +#include <chrono> QT_BEGIN_NAMESPACE - namespace { -static const char *audioSampleFormatNames[QAudioFormat::NSampleFormats] = { +const char *audioSampleFormatNames[QAudioFormat::NSampleFormats] = { nullptr, #if Q_BYTE_ORDER == Q_LITTLE_ENDIAN "U8", @@ -79,7 +27,7 @@ static const char *audioSampleFormatNames[QAudioFormat::NSampleFormats] = { #endif }; -static QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) +QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) { if (fmt) { for (int i = 1; i < QAudioFormat::NSampleFormats; ++i) { @@ -91,7 +39,7 @@ static QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) return QAudioFormat::Unknown; } -} +} // namespace /* Returns audio format for a sample \a sample. @@ -99,16 +47,16 @@ static QAudioFormat::SampleFormat gstSampleFormatToSampleFormat(const char *fmt) */ QAudioFormat QGstUtils::audioFormatForSample(GstSample *sample) { - QGstCaps caps = gst_sample_get_caps(sample); + auto caps = QGstCaps(gst_sample_get_caps(sample), QGstCaps::NeedsRef); if (caps.isNull()) - return QAudioFormat(); + return {}; return audioFormatForCaps(caps); } -QAudioFormat QGstUtils::audioFormatForCaps(QGstCaps caps) +QAudioFormat QGstUtils::audioFormatForCaps(const QGstCaps &caps) { QAudioFormat format; - QGstStructure s = caps.at(0); + QGstStructureView s = caps.at(0); if (s.name() != "audio/x-raw") return format; @@ -132,19 +80,21 @@ QAudioFormat QGstUtils::audioFormatForCaps(QGstCaps caps) \note Caller must unreference GstCaps. */ -QGstMutableCaps QGstUtils::capsForAudioFormat(const QAudioFormat &format) +QGstCaps QGstUtils::capsForAudioFormat(const QAudioFormat &format) { if (!format.isValid()) return {}; auto sampleFormat = format.sampleFormat(); - return gst_caps_new_simple( + auto caps = gst_caps_new_simple( "audio/x-raw", "format" , G_TYPE_STRING, audioSampleFormatNames[sampleFormat], "rate" , G_TYPE_INT , format.sampleRate(), "channels", G_TYPE_INT , format.channelCount(), "layout" , G_TYPE_STRING, "interleaved", nullptr); + + return QGstCaps(caps, QGstCaps::HasRef); } QList<QAudioFormat::SampleFormat> QGValue::getSampleFormats() const @@ -155,346 +105,37 @@ QList<QAudioFormat::SampleFormat> QGValue::getSampleFormats() const QList<QAudioFormat::SampleFormat> formats; guint nFormats = gst_value_list_get_size(value); for (guint f = 0; f < nFormats; ++f) { - QGValue v = gst_value_list_get_value(value, f); + QGValue v = QGValue{ gst_value_list_get_value(value, f) }; auto *name = v.toString(); QAudioFormat::SampleFormat fmt = gstSampleFormatToSampleFormat(name); if (fmt == QAudioFormat::Unknown) - continue;; + continue; formats.append(fmt); } return formats; } -namespace { - -struct VideoFormat -{ - QVideoFrameFormat::PixelFormat pixelFormat; - GstVideoFormat gstFormat; -}; - -static const VideoFormat qt_videoFormatLookup[] = -{ - { QVideoFrameFormat::Format_YUV420P, GST_VIDEO_FORMAT_I420 }, - { QVideoFrameFormat::Format_YUV422P, GST_VIDEO_FORMAT_Y42B }, - { QVideoFrameFormat::Format_YV12 , GST_VIDEO_FORMAT_YV12 }, - { QVideoFrameFormat::Format_UYVY , GST_VIDEO_FORMAT_UYVY }, - { QVideoFrameFormat::Format_YUYV , GST_VIDEO_FORMAT_YUY2 }, - { QVideoFrameFormat::Format_NV12 , GST_VIDEO_FORMAT_NV12 }, - { QVideoFrameFormat::Format_NV21 , GST_VIDEO_FORMAT_NV21 }, - { QVideoFrameFormat::Format_AYUV , GST_VIDEO_FORMAT_AYUV }, - { QVideoFrameFormat::Format_Y8 , GST_VIDEO_FORMAT_GRAY8 }, - { QVideoFrameFormat::Format_XRGB8888 , GST_VIDEO_FORMAT_xRGB }, - { QVideoFrameFormat::Format_XBGR8888 , GST_VIDEO_FORMAT_xBGR }, - { QVideoFrameFormat::Format_RGBX8888 , GST_VIDEO_FORMAT_RGBx }, - { QVideoFrameFormat::Format_BGRX8888 , GST_VIDEO_FORMAT_BGRx }, - { QVideoFrameFormat::Format_ARGB8888, GST_VIDEO_FORMAT_ARGB }, - { QVideoFrameFormat::Format_ABGR8888, GST_VIDEO_FORMAT_ABGR }, - { QVideoFrameFormat::Format_RGBA8888, GST_VIDEO_FORMAT_RGBA }, - { QVideoFrameFormat::Format_BGRA8888, GST_VIDEO_FORMAT_BGRA }, -#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN - { QVideoFrameFormat::Format_Y16 , GST_VIDEO_FORMAT_GRAY16_LE }, - { QVideoFrameFormat::Format_P010 , GST_VIDEO_FORMAT_P010_10LE }, -#else - { QVideoFrameFormat::Format_Y16 , GST_VIDEO_FORMAT_GRAY16_BE }, - { QVideoFrameFormat::Format_P010 , GST_VIDEO_FORMAT_P010_10BE }, -#endif -}; - -static int indexOfVideoFormat(QVideoFrameFormat::PixelFormat format) -{ - for (int i = 0; i < lengthOf(qt_videoFormatLookup); ++i) - if (qt_videoFormatLookup[i].pixelFormat == format) - return i; - - return -1; -} - -static int indexOfVideoFormat(GstVideoFormat format) -{ - for (int i = 0; i < lengthOf(qt_videoFormatLookup); ++i) - if (qt_videoFormatLookup[i].gstFormat == format) - return i; - - return -1; -} - -} - -QVideoFrameFormat QGstCaps::formatForCaps(GstVideoInfo *info) const -{ - GstVideoInfo vidInfo; - GstVideoInfo *infoPtr = info ? info : &vidInfo; - - if (gst_video_info_from_caps(infoPtr, caps)) { - int index = indexOfVideoFormat(infoPtr->finfo->format); - - if (index != -1) { - QVideoFrameFormat format( - QSize(infoPtr->width, infoPtr->height), - qt_videoFormatLookup[index].pixelFormat); - - if (infoPtr->fps_d > 0) - format.setFrameRate(qreal(infoPtr->fps_n) / infoPtr->fps_d); - - QVideoFrameFormat::ColorRange range = QVideoFrameFormat::ColorRange_Unknown; - switch (infoPtr->colorimetry.range) { - case GST_VIDEO_COLOR_RANGE_UNKNOWN: - break; - case GST_VIDEO_COLOR_RANGE_0_255: - range = QVideoFrameFormat::ColorRange_Full; - break; - case GST_VIDEO_COLOR_RANGE_16_235: - range = QVideoFrameFormat::ColorRange_Video; - break; - } - format.setColorRange(range); - - QVideoFrameFormat::ColorSpace colorSpace = QVideoFrameFormat::ColorSpace_Undefined; - switch (infoPtr->colorimetry.matrix) { - case GST_VIDEO_COLOR_MATRIX_UNKNOWN: - case GST_VIDEO_COLOR_MATRIX_RGB: - case GST_VIDEO_COLOR_MATRIX_FCC: - break; - case GST_VIDEO_COLOR_MATRIX_BT709: - colorSpace = QVideoFrameFormat::ColorSpace_BT709; - break; - case GST_VIDEO_COLOR_MATRIX_BT601: - colorSpace = QVideoFrameFormat::ColorSpace_BT601; - break; - case GST_VIDEO_COLOR_MATRIX_SMPTE240M: - colorSpace = QVideoFrameFormat::ColorSpace_AdobeRgb; - break; - case GST_VIDEO_COLOR_MATRIX_BT2020: - colorSpace = QVideoFrameFormat::ColorSpace_BT2020; - break; - } - format.setColorSpace(colorSpace); - - QVideoFrameFormat::ColorTransfer transfer = QVideoFrameFormat::ColorTransfer_Unknown; - switch (infoPtr->colorimetry.transfer) { - case GST_VIDEO_TRANSFER_UNKNOWN: - break; - case GST_VIDEO_TRANSFER_GAMMA10: - transfer = QVideoFrameFormat::ColorTransfer_Linear; - break; - case GST_VIDEO_TRANSFER_GAMMA22: - case GST_VIDEO_TRANSFER_SMPTE240M: - case GST_VIDEO_TRANSFER_SRGB: - case GST_VIDEO_TRANSFER_ADOBERGB: - transfer = QVideoFrameFormat::ColorTransfer_Gamma22; - break; - case GST_VIDEO_TRANSFER_GAMMA18: - case GST_VIDEO_TRANSFER_GAMMA20: - // not quite, but best fit - case GST_VIDEO_TRANSFER_BT709: - case GST_VIDEO_TRANSFER_BT2020_12: - transfer = QVideoFrameFormat::ColorTransfer_BT709; - break; - case GST_VIDEO_TRANSFER_GAMMA28: - transfer = QVideoFrameFormat::ColorTransfer_Gamma28; - break; - case GST_VIDEO_TRANSFER_LOG100: - case GST_VIDEO_TRANSFER_LOG316: - break; -#if GST_CHECK_VERSION(1, 18, 0) - case GST_VIDEO_TRANSFER_SMPTE2084: - transfer = QVideoFrameFormat::ColorTransfer_ST2084; - break; - case GST_VIDEO_TRANSFER_ARIB_STD_B67: - transfer = QVideoFrameFormat::ColorTransfer_STD_B67; - break; - case GST_VIDEO_TRANSFER_BT2020_10: - transfer = QVideoFrameFormat::ColorTransfer_BT709; - break; - case GST_VIDEO_TRANSFER_BT601: - transfer = QVideoFrameFormat::ColorTransfer_BT601; - break; -#endif - } - format.setColorTransfer(transfer); - - return format; - } - } - return QVideoFrameFormat(); -} - -void QGstMutableCaps::addPixelFormats(const QList<QVideoFrameFormat::PixelFormat> &formats, const char *modifier) -{ - GValue list = {}; - g_value_init(&list, GST_TYPE_LIST); - - for (QVideoFrameFormat::PixelFormat format : formats) { - int index = indexOfVideoFormat(format); - if (index == -1) - continue; - GValue item = {}; - - g_value_init(&item, G_TYPE_STRING); - g_value_set_string(&item, gst_video_format_to_string(qt_videoFormatLookup[index].gstFormat)); - gst_value_list_append_value(&list, &item); - g_value_unset(&item); - } - QGValue v(&list); - auto *structure = gst_structure_new("video/x-raw", - "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, INT_MAX, 1, - "width" , GST_TYPE_INT_RANGE, 1, INT_MAX, - "height" , GST_TYPE_INT_RANGE, 1, INT_MAX, - nullptr); - gst_structure_set_value(structure, "format", &list); - gst_caps_append_structure(caps, structure); - g_value_unset(&list); - - if (modifier) - gst_caps_set_features(caps, size() - 1, gst_caps_features_from_string(modifier)); -} - -QGstMutableCaps QGstMutableCaps::fromCameraFormat(const QCameraFormat &format) +void QGstUtils::setFrameTimeStampsFromBuffer(QVideoFrame *frame, GstBuffer *buffer) { - QGstMutableCaps caps; - caps.create(); - - QSize size = format.resolution(); - GstStructure *structure = nullptr; -// int num = 0; -// int den = 1; -// if (format.maxFrameRate() > 0) -// qt_real_to_fraction(1. / format.maxFrameRate(), &num, &den); -// qDebug() << "fromCameraFormat" << format.maxFrameRate() << num << den; - - if (format.pixelFormat() == QVideoFrameFormat::Format_Jpeg) { - structure = gst_structure_new("image/jpeg", - "width" , G_TYPE_INT, size.width(), - "height" , G_TYPE_INT, size.height(), -// "framerate", GST_TYPE_FRACTION, den, num, - nullptr); - } else { - int index = indexOfVideoFormat(format.pixelFormat()); - if (index < 0) - return QGstMutableCaps(); - auto gstFormat = qt_videoFormatLookup[index].gstFormat; - structure = gst_structure_new("video/x-raw", - "format" , G_TYPE_STRING, gst_video_format_to_string(gstFormat), - "width" , G_TYPE_INT, size.width(), - "height" , G_TYPE_INT, size.height(), -// "framerate", GST_TYPE_FRACTION, den, num, - nullptr); - } - gst_caps_append_structure(caps.caps, structure); - return caps; -} + using namespace std::chrono; + using namespace std::chrono_literals; -void QGstUtils::setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer) -{ // GStreamer uses nanoseconds, Qt uses microseconds - qint64 startTime = GST_BUFFER_TIMESTAMP(buffer); - if (startTime >= 0) { - frame->setStartTime(startTime/G_GINT64_CONSTANT (1000)); - - qint64 duration = GST_BUFFER_DURATION(buffer); - if (duration >= 0) - frame->setEndTime((startTime + duration)/G_GINT64_CONSTANT (1000)); - } -} - -QSize QGstStructure::resolution() const -{ - QSize size; + nanoseconds startTime{ GST_BUFFER_TIMESTAMP(buffer) }; + if (startTime >= 0ns) { + frame->setStartTime(floor<microseconds>(startTime).count()); - int w, h; - if (structure && - gst_structure_get_int(structure, "width", &w) && - gst_structure_get_int(structure, "height", &h)) { - size.rwidth() = w; - size.rheight() = h; + nanoseconds duration{ GST_BUFFER_DURATION(buffer) }; + if (duration >= 0ns) + frame->setEndTime(floor<microseconds>(startTime + duration).count()); } - - return size; -} - -QVideoFrameFormat::PixelFormat QGstStructure::pixelFormat() const -{ - QVideoFrameFormat::PixelFormat pixelFormat = QVideoFrameFormat::Format_Invalid; - - if (!structure) - return pixelFormat; - - if (gst_structure_has_name(structure, "video/x-raw")) { - const gchar *s = gst_structure_get_string(structure, "format"); - if (s) { - GstVideoFormat format = gst_video_format_from_string(s); - int index = indexOfVideoFormat(format); - - if (index != -1) - pixelFormat = qt_videoFormatLookup[index].pixelFormat; - } - } else if (gst_structure_has_name(structure, "image/jpeg")) { - pixelFormat = QVideoFrameFormat::Format_Jpeg; - } - - return pixelFormat; -} - -QGRange<float> QGstStructure::frameRateRange() const -{ - float minRate = 0.; - float maxRate = 0.; - - if (!structure) - return {0.f, 0.f}; - - auto extractFraction = [] (const GValue *v) -> float { - return (float)gst_value_get_fraction_numerator(v)/(float)gst_value_get_fraction_denominator(v); - }; - auto extractFrameRate = [&] (const GValue *v) { - auto insert = [&] (float min, float max) { - if (max > maxRate) - maxRate = max; - if (min < minRate) - minRate = min; - }; - - if (GST_VALUE_HOLDS_FRACTION(v)) { - float rate = extractFraction(v); - insert(rate, rate); - } else if (GST_VALUE_HOLDS_FRACTION_RANGE(v)) { - auto *min = gst_value_get_fraction_range_max(v); - auto *max = gst_value_get_fraction_range_max(v); - insert(extractFraction(min), extractFraction(max)); - } - }; - - const GValue *gstFrameRates = gst_structure_get_value(structure, "framerate"); - if (gstFrameRates) { - if (GST_VALUE_HOLDS_LIST(gstFrameRates)) { - guint nFrameRates = gst_value_list_get_size(gstFrameRates); - for (guint f = 0; f < nFrameRates; ++f) { - extractFrameRate(gst_value_list_get_value(gstFrameRates, f)); - } - } else { - extractFrameRate(gstFrameRates); - } - } else { - const GValue *min = gst_structure_get_value(structure, "min-framerate"); - const GValue *max = gst_structure_get_value(structure, "max-framerate"); - if (min && max) { - minRate = extractFraction(min); - maxRate = extractFraction(max); - } - } - - return {minRate, maxRate}; } GList *qt_gst_video_sinks() { - GList *list = nullptr; - - list = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_SINK | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, + return gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_SINK + | GST_ELEMENT_FACTORY_TYPE_MEDIA_VIDEO, GST_RANK_MARGINAL); - - return list; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstutils_p.h b/src/plugins/multimedia/gstreamer/common/qgstutils_p.h index 9782486a7..c65fcf090 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstutils_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstutils_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTUTILS_P_H #define QGSTUTILS_P_H @@ -51,34 +15,26 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <QtCore/qlist.h> -#include <QtCore/qmap.h> -#include <QtCore/qset.h> -#include <qgst_p.h> -#include <gst/video/video.h> -#include <qaudioformat.h> -#include <qcamera.h> -#include <qvideoframe.h> -#include <QDebug> +#include <gst/gstsample.h> +#include <gst/gstbuffer.h> + +#include <QtCore/qglobal.h> QT_BEGIN_NAMESPACE -class QSize; -class QVariant; -class QByteArray; -class QImage; -class QVideoFrameFormat; +class QAudioFormat; +class QGstCaps; +class QVideoFrame; namespace QGstUtils { - Q_MULTIMEDIA_EXPORT QAudioFormat audioFormatForSample(GstSample *sample); - QAudioFormat audioFormatForCaps(QGstCaps caps); - Q_MULTIMEDIA_EXPORT QGstMutableCaps capsForAudioFormat(const QAudioFormat &format); +QAudioFormat audioFormatForSample(GstSample *sample); +QAudioFormat audioFormatForCaps(const QGstCaps &caps); +QGstCaps capsForAudioFormat(const QAudioFormat &format); - void setFrameTimeStamps(QVideoFrame *frame, GstBuffer *buffer); -} +void setFrameTimeStampsFromBuffer(QVideoFrame *frame, GstBuffer *buffer); +} // namespace QGstUtils -Q_MULTIMEDIA_EXPORT GList *qt_gst_video_sinks(); +GList *qt_gst_video_sinks(); QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp index 0d22999b2..df3fb3d69 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstvideobuffer_p.h" #include "qgstreamervideosink_p.h" @@ -48,24 +12,24 @@ #include <gst/video/gstvideometa.h> #include <gst/pbutils/gstpluginsbaseversion.h> -#include "qgstutils_p.h" +#include <common/qgstutils_p.h> #if QT_CONFIG(gstreamer_gl) -#include <QtGui/private/qrhi_p.h> -#include <QtGui/private/qrhigles2_p.h> -#include <QtGui/qopenglcontext.h> -#include <QtGui/qopenglfunctions.h> -#include <QtGui/qopengl.h> - -#include <gst/gl/gstglconfig.h> -#include <gst/gl/gstglmemory.h> -#include <gst/gl/gstglsyncmeta.h> -#if QT_CONFIG(linux_dmabuf) -#include <gst/allocators/gstdmabuf.h> -#endif +# include <QtGui/rhi/qrhi.h> +# include <QtGui/qopenglcontext.h> +# include <QtGui/qopenglfunctions.h> +# include <QtGui/qopengl.h> + +# include <gst/gl/gstglconfig.h> +# include <gst/gl/gstglmemory.h> +# include <gst/gl/gstglsyncmeta.h> -#include <EGL/egl.h> -#include <EGL/eglext.h> +# include <EGL/egl.h> +# include <EGL/eglext.h> + +# if QT_CONFIG(linux_dmabuf) +# include <gst/allocators/gstdmabuf.h> +# endif #endif QT_BEGIN_NAMESPACE @@ -87,79 +51,60 @@ QT_BEGIN_NAMESPACE #define DRM_FORMAT_GR1616 fourcc_code('G', 'R', '3', '2') /* [31:0] G:R 16:16 little endian */ #define DRM_FORMAT_BGRA1010102 fourcc_code('B', 'A', '3', '0') /* [31:0] B:G:R:A 10:10:10:2 little endian */ -QGstVideoBuffer::QGstVideoBuffer(GstBuffer *buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, - const QVideoFrameFormat &frameFormat, +QGstVideoBuffer::QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, + QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, QGstCaps::MemoryFormat format) - : QAbstractVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory) ? - QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, sink ? sink->rhi() : nullptr) - , memoryFormat(format) - , m_frameFormat(frameFormat) - , m_rhi(sink ? sink->rhi() : nullptr) - , m_videoInfo(info) - , m_buffer(buffer) + : QHwVideoBuffer((sink && sink->rhi() && format != QGstCaps::CpuMemory) + ? QVideoFrame::RhiTextureHandle + : QVideoFrame::NoHandle, + sink ? sink->rhi() : nullptr), + memoryFormat(format), + m_frameFormat(frameFormat), + m_rhi(sink ? sink->rhi() : nullptr), + m_videoInfo(info), + m_buffer(std::move(buffer)) { - gst_buffer_ref(m_buffer); if (sink) { eglDisplay = sink->eglDisplay(); eglImageTargetTexture2D = sink->eglImageTargetTexture2D(); } -} - -QGstVideoBuffer::~QGstVideoBuffer() -{ - unmap(); - gst_buffer_unref(m_buffer); - if (m_syncBuffer) - gst_buffer_unref(m_syncBuffer); - - if (m_ownTextures && glContext) { - int planes = 0; - for (planes = 0; planes < 3; ++planes) { - if (m_textures[planes] == 0) - break; - } -#if QT_CONFIG(gstreamer_gl) - if (m_rhi) { - m_rhi->makeThreadLocalNativeContextCurrent(); - QOpenGLFunctions functions(glContext); - functions.glDeleteTextures(planes, m_textures); - } +#if !QT_CONFIG(gstreamer_gl) + Q_UNUSED(memoryFormat); #endif - } } - -QVideoFrame::MapMode QGstVideoBuffer::mapMode() const +QGstVideoBuffer::~QGstVideoBuffer() { - return m_mode; + Q_ASSERT(m_mode == QtVideo::MapMode::NotMapped); } -QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode) +QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QtVideo::MapMode mode) { - const GstMapFlags flags = GstMapFlags(((mode & QVideoFrame::ReadOnly) ? GST_MAP_READ : 0) - | ((mode & QVideoFrame::WriteOnly) ? GST_MAP_WRITE : 0)); + const GstMapFlags flags = GstMapFlags( + ((mode & QtVideo::MapMode::ReadOnly ) == QtVideo::MapMode::NotMapped ? 0 : GST_MAP_READ) + | ((mode & QtVideo::MapMode::WriteOnly) == QtVideo::MapMode::NotMapped ? 0 : GST_MAP_WRITE)); MapData mapData; - if (mode == QVideoFrame::NotMapped || m_mode != QVideoFrame::NotMapped) + if (mode == QtVideo::MapMode::NotMapped || m_mode != QtVideo::MapMode::NotMapped) return mapData; if (m_videoInfo.finfo->n_planes == 0) { // Encoded - if (gst_buffer_map(m_buffer, &m_frame.map[0], flags)) { - mapData.nPlanes = 1; + if (gst_buffer_map(m_buffer.get(), &m_frame.map[0], flags)) { + mapData.planeCount = 1; mapData.bytesPerLine[0] = -1; - mapData.size[0] = m_frame.map[0].size; + mapData.dataSize[0] = m_frame.map[0].size; mapData.data[0] = static_cast<uchar *>(m_frame.map[0].data); m_mode = mode; } - } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer, flags)) { - mapData.nPlanes = GST_VIDEO_FRAME_N_PLANES(&m_frame); + } else if (gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer.get(), flags)) { + mapData.planeCount = GST_VIDEO_FRAME_N_PLANES(&m_frame); for (guint i = 0; i < GST_VIDEO_FRAME_N_PLANES(&m_frame); ++i) { mapData.bytesPerLine[i] = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i); mapData.data[i] = static_cast<uchar *>(GST_VIDEO_FRAME_PLANE_DATA(&m_frame, i)); - mapData.size[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); + mapData.dataSize[i] = mapData.bytesPerLine[i]*GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); } m_mode = mode; @@ -169,13 +114,13 @@ QAbstractVideoBuffer::MapData QGstVideoBuffer::map(QVideoFrame::MapMode mode) void QGstVideoBuffer::unmap() { - if (m_mode != QVideoFrame::NotMapped) { + if (m_mode != QtVideo::MapMode::NotMapped) { if (m_videoInfo.finfo->n_planes == 0) - gst_buffer_unmap(m_buffer, &m_frame.map[0]); + gst_buffer_unmap(m_buffer.get(), &m_frame.map[0]); else gst_video_frame_unmap(&m_frame); } - m_mode = QVideoFrame::NotMapped; + m_mode = QtVideo::MapMode::NotMapped; } #if QT_CONFIG(gstreamer_gl) && QT_CONFIG(linux_dmabuf) @@ -264,122 +209,185 @@ fourccFromVideoInfo(const GstVideoInfo * info, int plane) } #endif -void QGstVideoBuffer::mapTextures() +#if QT_CONFIG(gstreamer_gl) +struct GlTextures { - if (!m_rhi) - return; + uint count = 0; + bool owned = false; + std::array<guint32, QVideoTextureHelper::TextureDescription::maxPlanes> names{}; +}; -#if QT_CONFIG(gstreamer_gl) - if (memoryFormat == QGstCaps::GLTexture) { - auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(m_buffer, 0)); - Q_ASSERT(mem); - if (!gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) { - qWarning() << "Could not map GL textures"; - } else { - auto *sync_meta = gst_buffer_get_gl_sync_meta(m_buffer); - - if (!sync_meta) { - m_syncBuffer = gst_buffer_new(); - sync_meta = gst_buffer_add_gl_sync_meta(mem->context, m_syncBuffer); - } - gst_gl_sync_meta_set_sync_point (sync_meta, mem->context); - gst_gl_sync_meta_wait (sync_meta, mem->context); - - int nPlanes = m_frame.info.finfo->n_planes; - for (int i = 0; i < nPlanes; ++i) { - m_textures[i] = *(guint32 *)m_frame.data[i]; - } - gst_video_frame_unmap(&m_frame); +class QGstQVideoFrameTextures : public QVideoFrameTextures +{ +public: + QGstQVideoFrameTextures(QRhi *rhi, QSize size, QVideoFrameFormat::PixelFormat format, GlTextures &textures) + : m_rhi(rhi) + , m_glTextures(textures) + { + auto desc = QVideoTextureHelper::textureDescription(format); + for (uint i = 0; i < textures.count; ++i) { + QSize planeSize(desc->widthForPlane(size.width(), int(i)), + desc->heightForPlane(size.height(), int(i))); + m_textures[i].reset(rhi->newTexture(desc->textureFormat[i], planeSize, 1, {})); + m_textures[i]->createFrom({textures.names[i], 0}); } } + + ~QGstQVideoFrameTextures() + { + m_rhi->makeThreadLocalNativeContextCurrent(); + auto ctx = QOpenGLContext::currentContext(); + if (m_glTextures.owned && ctx) + ctx->functions()->glDeleteTextures(int(m_glTextures.count), m_glTextures.names.data()); + } + + QRhiTexture *texture(uint plane) const override + { + return plane < m_glTextures.count ? m_textures[plane].get() : nullptr; + } + +private: + QRhi *m_rhi = nullptr; + GlTextures m_glTextures; + std::unique_ptr<QRhiTexture> m_textures[QVideoTextureHelper::TextureDescription::maxPlanes]; +}; + +static GlTextures mapFromGlTexture(const QGstBufferHandle &bufferHandle, GstVideoFrame &frame, + GstVideoInfo &videoInfo) +{ + GstBuffer *buffer = bufferHandle.get(); + auto *mem = GST_GL_BASE_MEMORY_CAST(gst_buffer_peek_memory(buffer, 0)); + if (!mem) + return {}; + + if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ|GST_MAP_GL))) { + qWarning() << "Could not map GL textures"; + return {}; + } + + auto *sync_meta = gst_buffer_get_gl_sync_meta(buffer); + GstBuffer *sync_buffer = nullptr; + if (!sync_meta) { + sync_buffer = gst_buffer_new(); + sync_meta = gst_buffer_add_gl_sync_meta(mem->context, sync_buffer); + } + gst_gl_sync_meta_set_sync_point (sync_meta, mem->context); + gst_gl_sync_meta_wait (sync_meta, mem->context); + if (sync_buffer) + gst_buffer_unref(sync_buffer); + + GlTextures textures; + textures.count = frame.info.finfo->n_planes; + + for (uint i = 0; i < textures.count; ++i) + textures.names[i] = *(guint32 *)frame.data[i]; + + gst_video_frame_unmap(&frame); + + return textures; +} + #if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) - else if (memoryFormat == QGstCaps::DMABuf) { - if (m_textures[0]) - return; - Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(m_buffer, 0))); - Q_ASSERT(eglDisplay); - Q_ASSERT(eglImageTargetTexture2D); - - auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(m_rhi->nativeHandles()); - glContext = nativeHandles->context; - if (!glContext) { - qWarning() << "no GL context"; - return; - } +static GlTextures mapFromDmaBuffer(QRhi *rhi, const QGstBufferHandle &bufferHandle, + GstVideoFrame &frame, GstVideoInfo &videoInfo, + Qt::HANDLE eglDisplay, QFunctionPointer eglImageTargetTexture2D) +{ + GstBuffer *buffer = bufferHandle.get(); - if (!gst_video_frame_map(&m_frame, &m_videoInfo, m_buffer, GstMapFlags(GST_MAP_READ))) { - qDebug() << "Couldn't map DMA video frame"; - return; - } + Q_ASSERT(gst_is_dmabuf_memory(gst_buffer_peek_memory(buffer, 0))); + Q_ASSERT(eglDisplay); + Q_ASSERT(eglImageTargetTexture2D); - int nPlanes = GST_VIDEO_FRAME_N_PLANES(&m_frame); -// int width = GST_VIDEO_FRAME_WIDTH(&m_frame); -// int height = GST_VIDEO_FRAME_HEIGHT(&m_frame); - Q_ASSERT(GST_VIDEO_FRAME_N_PLANES(&m_frame) == gst_buffer_n_memory(m_buffer)); - - QOpenGLFunctions functions(glContext); - functions.glGenTextures(nPlanes, m_textures); - m_ownTextures = true; - -// qDebug() << Qt::hex << "glGenTextures: glerror" << glGetError() << "egl error" << eglGetError(); -// qDebug() << "converting DMA buffer nPlanes=" << nPlanes << m_textures[0] << m_textures[1] << m_textures[2]; - - for (int i = 0; i < nPlanes; ++i) { - auto offset = GST_VIDEO_FRAME_PLANE_OFFSET(&m_frame, i); - auto stride = GST_VIDEO_FRAME_PLANE_STRIDE(&m_frame, i); - int planeWidth = GST_VIDEO_FRAME_COMP_WIDTH(&m_frame, i); - int planeHeight = GST_VIDEO_FRAME_COMP_HEIGHT(&m_frame, i); - auto mem = gst_buffer_peek_memory(m_buffer, i); - int fd = gst_dmabuf_memory_get_fd(mem); - -// qDebug() << " plane" << i << "size" << width << height << "stride" << stride << "offset" << offset << "fd=" << fd; - // ### do we need to open/close the fd? - // ### can we convert several planes at once? - // Get the correct DRM_FORMATs from the texture format in the description - EGLAttrib const attribute_list[] = { - EGL_WIDTH, planeWidth, - EGL_HEIGHT, planeHeight, - EGL_LINUX_DRM_FOURCC_EXT, fourccFromVideoInfo(&m_videoInfo, i), - EGL_DMA_BUF_PLANE0_FD_EXT, fd, - EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLAttrib)offset, - EGL_DMA_BUF_PLANE0_PITCH_EXT, stride, - EGL_NONE - }; - EGLImage image = eglCreateImage(eglDisplay, - EGL_NO_CONTEXT, - EGL_LINUX_DMA_BUF_EXT, - nullptr, - attribute_list); - if (image == EGL_NO_IMAGE_KHR) { - qWarning() << "could not create EGL image for plane" << i << Qt::hex << eglGetError(); - } -// qDebug() << Qt::hex << "eglCreateImage: glerror" << glGetError() << "egl error" << eglGetError(); - functions.glBindTexture(GL_TEXTURE_2D, m_textures[i]); -// qDebug() << Qt::hex << "bind texture: glerror" << glGetError() << "egl error" << eglGetError(); - auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D; - EGLImageTargetTexture2D(GL_TEXTURE_2D, image); -// qDebug() << Qt::hex << "glerror" << glGetError() << "egl error" << eglGetError(); - eglDestroyImage(eglDisplay, image); + auto *nativeHandles = static_cast<const QRhiGles2NativeHandles *>(rhi->nativeHandles()); + auto glContext = nativeHandles->context; + if (!glContext) { + qWarning() << "no GL context"; + return {}; + } + + if (!gst_video_frame_map(&frame, &videoInfo, buffer, GstMapFlags(GST_MAP_READ))) { + qDebug() << "Couldn't map DMA video frame"; + return {}; + } + + GlTextures textures = {}; + textures.owned = true; + textures.count = GST_VIDEO_FRAME_N_PLANES(&frame); + // int width = GST_VIDEO_FRAME_WIDTH(&frame); + // int height = GST_VIDEO_FRAME_HEIGHT(&frame); + Q_ASSERT(GST_VIDEO_FRAME_N_PLANES(&frame) == gst_buffer_n_memory(buffer)); + + QOpenGLFunctions functions(glContext); + functions.glGenTextures(int(textures.count), textures.names.data()); + + // qDebug() << Qt::hex << "glGenTextures: glerror" << glGetError() << "egl error" << eglGetError(); + // qDebug() << "converting DMA buffer nPlanes=" << nPlanes << m_textures[0] << m_textures[1] << m_textures[2]; + + for (int i = 0; i < int(textures.count); ++i) { + auto offset = GST_VIDEO_FRAME_PLANE_OFFSET(&frame, i); + auto stride = GST_VIDEO_FRAME_PLANE_STRIDE(&frame, i); + int planeWidth = GST_VIDEO_FRAME_COMP_WIDTH(&frame, i); + int planeHeight = GST_VIDEO_FRAME_COMP_HEIGHT(&frame, i); + auto mem = gst_buffer_peek_memory(buffer, i); + int fd = gst_dmabuf_memory_get_fd(mem); + + // qDebug() << " plane" << i << "size" << width << height << "stride" << stride << "offset" << offset << "fd=" << fd; + // ### do we need to open/close the fd? + // ### can we convert several planes at once? + // Get the correct DRM_FORMATs from the texture format in the description + EGLAttrib const attribute_list[] = { + EGL_WIDTH, planeWidth, + EGL_HEIGHT, planeHeight, + EGL_LINUX_DRM_FOURCC_EXT, fourccFromVideoInfo(&videoInfo, i), + EGL_DMA_BUF_PLANE0_FD_EXT, fd, + EGL_DMA_BUF_PLANE0_OFFSET_EXT, (EGLAttrib)offset, + EGL_DMA_BUF_PLANE0_PITCH_EXT, stride, + EGL_NONE + }; + EGLImage image = eglCreateImage(eglDisplay, + EGL_NO_CONTEXT, + EGL_LINUX_DMA_BUF_EXT, + nullptr, + attribute_list); + if (image == EGL_NO_IMAGE_KHR) { + qWarning() << "could not create EGL image for plane" << i << Qt::hex << eglGetError(); } - gst_video_frame_unmap(&m_frame); + // qDebug() << Qt::hex << "eglCreateImage: glerror" << glGetError() << "egl error" << eglGetError(); + functions.glBindTexture(GL_TEXTURE_2D, textures.names[i]); + // qDebug() << Qt::hex << "bind texture: glerror" << glGetError() << "egl error" << eglGetError(); + auto EGLImageTargetTexture2D = (PFNGLEGLIMAGETARGETTEXTURE2DOESPROC)eglImageTargetTexture2D; + EGLImageTargetTexture2D(GL_TEXTURE_2D, image); + // qDebug() << Qt::hex << "glerror" << glGetError() << "egl error" << eglGetError(); + eglDestroyImage(eglDisplay, image); } + gst_video_frame_unmap(&frame); + + return textures; +} #endif #endif - m_texturesUploaded = true; -} -std::unique_ptr<QRhiTexture> QGstVideoBuffer::texture(int plane) const +std::unique_ptr<QVideoFrameTextures> QGstVideoBuffer::mapTextures(QRhi *rhi) { - auto desc = QVideoTextureHelper::textureDescription(m_frameFormat.pixelFormat()); - if (!m_rhi || !desc || plane >= desc->nplanes) + if (!rhi) return {}; - QSize size(desc->widthForPlane(m_videoInfo.width, plane), desc->heightForPlane(m_videoInfo.height, plane)); - std::unique_ptr<QRhiTexture> tex(m_rhi->newTexture(desc->textureFormat[plane], size, 1, {})); - if (tex) { - if (!tex->createFrom({m_textures[plane], 0})) - return {}; - } - return tex; + +#if QT_CONFIG(gstreamer_gl) + GlTextures textures = {}; + if (memoryFormat == QGstCaps::GLTexture) + textures = mapFromGlTexture(m_buffer, m_frame, m_videoInfo); + +# if GST_GL_HAVE_PLATFORM_EGL && QT_CONFIG(linux_dmabuf) + else if (memoryFormat == QGstCaps::DMABuf) + textures = mapFromDmaBuffer(m_rhi, m_buffer, m_frame, m_videoInfo, eglDisplay, + eglImageTargetTexture2D); + +# endif + if (textures.count > 0) + return std::make_unique<QGstQVideoFrameTextures>(rhi, QSize{m_videoInfo.width, m_videoInfo.height}, + m_frameFormat.pixelFormat(), textures); +#endif + return {}; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h index 09dc8b992..573a4662c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstvideobuffer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTVIDEOBUFFER_P_H #define QGSTVIDEOBUFFER_P_H @@ -51,11 +15,10 @@ // We mean it. // -#include <private/qtmultimediaglobal_p.h> -#include <private/qabstractvideobuffer_p.h> +#include <private/qhwvideobuffer_p.h> #include <QtCore/qvariant.h> -#include <qgst_p.h> +#include <common/qgst_p.h> #include <gst/video/video.h> QT_BEGIN_NAMESPACE @@ -63,40 +26,28 @@ class QVideoFrameFormat; class QGstreamerVideoSink; class QOpenGLContext; -class Q_MULTIMEDIA_EXPORT QGstVideoBuffer : public QAbstractVideoBuffer +class QGstVideoBuffer final : public QHwVideoBuffer { public: - - QGstVideoBuffer(GstBuffer *buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, + QGstVideoBuffer(QGstBufferHandle buffer, const GstVideoInfo &info, QGstreamerVideoSink *sink, const QVideoFrameFormat &frameFormat, QGstCaps::MemoryFormat format); - QGstVideoBuffer(GstBuffer *buffer, const QVideoFrameFormat &format, const GstVideoInfo &info) - : QGstVideoBuffer(buffer, info, nullptr, format, QGstCaps::CpuMemory) - {} ~QGstVideoBuffer(); - GstBuffer *buffer() const { return m_buffer; } - QVideoFrame::MapMode mapMode() const override; - - MapData map(QVideoFrame::MapMode mode) override; + MapData map(QtVideo::MapMode mode) override; void unmap() override; - void mapTextures() override; - std::unique_ptr<QRhiTexture> texture(int plane) const override; + std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *) override; + private: - QGstCaps::MemoryFormat memoryFormat = QGstCaps::CpuMemory; - QVideoFrameFormat m_frameFormat; + const QGstCaps::MemoryFormat memoryFormat = QGstCaps::CpuMemory; + const QVideoFrameFormat m_frameFormat; QRhi *m_rhi = nullptr; mutable GstVideoInfo m_videoInfo; - mutable GstVideoFrame m_frame; - GstBuffer *m_buffer = nullptr; - GstBuffer *m_syncBuffer = nullptr; - QVideoFrame::MapMode m_mode = QVideoFrame::NotMapped; - QOpenGLContext *glContext = nullptr; + mutable GstVideoFrame m_frame{}; + const QGstBufferHandle m_buffer; + QtVideo::MapMode m_mode = QtVideo::MapMode::NotMapped; Qt::HANDLE eglDisplay = nullptr; QFunctionPointer eglImageTargetTexture2D = nullptr; - uint m_textures[3] = {}; - bool m_texturesUploaded = false; - bool m_ownTextures = false; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp index 1a8107889..a2580973c 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp +++ b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink.cpp @@ -1,64 +1,32 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qvideoframe.h> -#include <qvideosink.h> -#include <QDebug> -#include <QMap> -#include <QThread> -#include <QEvent> -#include <QCoreApplication> - -#include <private/qfactoryloader_p.h> -#include "qgstvideobuffer_p.h" -#include "qgstreamervideosink_p.h" +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstvideorenderersink_p.h" +#include <QtMultimedia/qvideoframe.h> +#include <QtMultimedia/qvideosink.h> +#include <QtCore/private/qfactoryloader_p.h> +#include <QtCore/private/quniquehandle_p.h> +#include <QtCore/qcoreapplication.h> +#include <QtCore/qdebug.h> +#include <QtCore/qdebug.h> +#include <QtCore/qloggingcategory.h> +#include <QtCore/qmap.h> +#include <QtCore/qthread.h> +#include <QtGui/qevent.h> + +#include <common/qgstvideobuffer_p.h> +#include <common/qgstreamervideosink_p.h> +#include <common/qgst_debug_p.h> +#include <common/qgstutils_p.h> + +#include <private/qvideoframe_p.h> + #include <gst/video/video.h> #include <gst/video/gstvideometa.h> -#include <qloggingcategory.h> -#include <qdebug.h> -#include "qgstutils_p.h" -#include <QtGui/private/qrhi_p.h> +#include <rhi/qrhi.h> #if QT_CONFIG(gstreamer_gl) #include <gst/gl/gl.h> #endif // #if QT_CONFIG(gstreamer_gl) @@ -68,29 +36,27 @@ #include <gst/allocators/gstdmabuf.h> #endif -//#define DEBUG_VIDEO_SURFACE_SINK - -Q_LOGGING_CATEGORY(qLcGstVideoRenderer, "qt.multimedia.gstvideorenderer") +static Q_LOGGING_CATEGORY(qLcGstVideoRenderer, "qt.multimedia.gstvideorenderer") QT_BEGIN_NAMESPACE QGstVideoRenderer::QGstVideoRenderer(QGstreamerVideoSink *sink) - : m_sink(sink) + : m_sink(sink), m_surfaceCaps(createSurfaceCaps(sink)) { - createSurfaceCaps(); + QObject::connect( + sink, &QGstreamerVideoSink::aboutToBeDestroyed, this, + [this] { + QMutexLocker locker(&m_sinkMutex); + m_sink = nullptr; + }, + Qt::DirectConnection); } -QGstVideoRenderer::~QGstVideoRenderer() -{ -} +QGstVideoRenderer::~QGstVideoRenderer() = default; -void QGstVideoRenderer::createSurfaceCaps() +QGstCaps QGstVideoRenderer::createSurfaceCaps([[maybe_unused]] QGstreamerVideoSink *sink) { - QRhi *rhi = m_sink->rhi(); - Q_UNUSED(rhi); - - QGstMutableCaps caps; - caps.create(); + QGstCaps caps = QGstCaps::create(); // All the formats that both we and gstreamer support auto formats = QList<QVideoFrameFormat::PixelFormat>() @@ -115,10 +81,11 @@ void QGstVideoRenderer::createSurfaceCaps() << QVideoFrameFormat::Format_Y16 ; #if QT_CONFIG(gstreamer_gl) + QRhi *rhi = sink->rhi(); if (rhi && rhi->backend() == QRhi::OpenGLES2) { caps.addPixelFormats(formats, GST_CAPS_FEATURE_MEMORY_GL_MEMORY); #if QT_CONFIG(linux_dmabuf) - if (m_sink->eglDisplay() && m_sink->eglImageTargetTexture2D()) { + if (sink->eglDisplay() && sink->eglImageTargetTexture2D()) { // We currently do not handle planar DMA buffers, as it's somewhat unclear how to // convert the planar EGLImage into something we can use from OpenGL auto singlePlaneFormats = QList<QVideoFrameFormat::PixelFormat>() @@ -142,105 +109,106 @@ void QGstVideoRenderer::createSurfaceCaps() } #endif caps.addPixelFormats(formats); - - m_surfaceCaps = caps; + return caps; } -QGstMutableCaps QGstVideoRenderer::caps() +const QGstCaps &QGstVideoRenderer::caps() { - QMutexLocker locker(&m_mutex); - return m_surfaceCaps; } -bool QGstVideoRenderer::start(GstCaps *caps) +bool QGstVideoRenderer::start(const QGstCaps& caps) { - qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::start" << QGstCaps(caps).toString(); - QMutexLocker locker(&m_mutex); + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::start" << caps; - m_frameMirrored = false; - m_frameRotationAngle = QVideoFrame::Rotation0; - - if (m_active) { - m_flush = true; - m_stop = true; - } - - m_startCaps = QGstMutableCaps(caps, QGstMutableCaps::NeedsRef); - - /* - Waiting for start() to be invoked in the main thread may block - if gstreamer blocks the main thread until this call is finished. - This situation is rare and usually caused by setState(Null) - while pipeline is being prerolled. - - The proper solution to this involves controlling gstreamer pipeline from - other thread than video surface. - - Currently start() fails if wait() timed out. - */ - if (!waitForAsyncEvent(&locker, &m_setupCondition, 1000) && !m_startCaps.isNull()) { - qWarning() << "Failed to start video surface due to main thread blocked."; - m_startCaps = {}; + { + m_frameRotationAngle = QtVideo::Rotation::None; + auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo(); + if (optionalFormatAndVideoInfo) { + std::tie(m_format, m_videoInfo) = std::move(*optionalFormatAndVideoInfo); + } else { + m_format = {}; + m_videoInfo = {}; + } + m_memoryFormat = caps.memoryFormat(); } - return m_active; + return true; } void QGstVideoRenderer::stop() { - QMutexLocker locker(&m_mutex); - - if (!m_active) - return; - - m_flush = true; - m_stop = true; + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::stop"; - m_startCaps = {}; - - waitForAsyncEvent(&locker, &m_setupCondition, 500); + QMetaObject::invokeMethod(this, [this] { + m_currentState.buffer = {}; + m_currentPipelineFrame = {}; + updateCurrentVideoFrame(m_currentVideoFrame); + }); } void QGstVideoRenderer::unlock() { - QMutexLocker locker(&m_mutex); - - m_setupCondition.wakeAll(); - m_renderCondition.wakeAll(); -} - -bool QGstVideoRenderer::proposeAllocation(GstQuery *query) -{ - Q_UNUSED(query); - QMutexLocker locker(&m_mutex); - return m_active; + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::unlock"; } -void QGstVideoRenderer::flush() +bool QGstVideoRenderer::proposeAllocation(GstQuery *) { - QMutexLocker locker(&m_mutex); - - m_flush = true; - m_renderBuffer = nullptr; - m_renderCondition.wakeAll(); - - notify(); + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::proposeAllocation"; + return true; } GstFlowReturn QGstVideoRenderer::render(GstBuffer *buffer) { - QMutexLocker locker(&m_mutex); qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::render"; - m_renderReturn = GST_FLOW_OK; - m_renderBuffer = buffer; + GstVideoCropMeta *meta = gst_buffer_get_video_crop_meta(buffer); + if (meta) { + QRect vp(meta->x, meta->y, meta->width, meta->height); + if (m_format.viewport() != vp) { + qCDebug(qLcGstVideoRenderer) + << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x" + << meta->width << " | " << meta->x << "x" << meta->y << "]"; + // Update viewport if data is not the same + m_format.setViewport(vp); + } + } - waitForAsyncEvent(&locker, &m_renderCondition, 300); + RenderBufferState state{ + .buffer = QGstBufferHandle{ buffer, QGstBufferHandle::NeedsRef }, + .format = m_format, + .memoryFormat = m_memoryFormat, + .mirrored = m_frameMirrored, + .rotationAngle = m_frameRotationAngle, + }; + + qCDebug(qLcGstVideoRenderer) << " sending video frame"; + + QMetaObject::invokeMethod(this, [this, state = std::move(state)]() mutable { + if (state == m_currentState) + // same buffer received twice + return; + + auto videoBuffer = std::make_unique<QGstVideoBuffer>(state.buffer, m_videoInfo, m_sink, + state.format, state.memoryFormat); + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(videoBuffer), state.format); + QGstUtils::setFrameTimeStampsFromBuffer(&frame, state.buffer.get()); + frame.setMirrored(state.mirrored); + frame.setRotation(state.rotationAngle); + + m_currentPipelineFrame = std::move(frame); + m_currentState = std::move(state); + + if (!m_isActive) { + qCDebug(qLcGstVideoRenderer) << " showing empty video frame"; + updateCurrentVideoFrame({}); + return; + } - m_renderBuffer = nullptr; + updateCurrentVideoFrame(m_currentPipelineFrame); + }); - return m_renderReturn; + return GST_FLOW_OK; } bool QGstVideoRenderer::query(GstQuery *query) @@ -253,6 +221,10 @@ bool QGstVideoRenderer::query(GstQuery *query) if (strcmp(type, "gst.gl.local_context") != 0) return false; + QMutexLocker locker(&m_sinkMutex); + if (!m_sink) + return false; + auto *gstGlContext = m_sink->gstGlLocalContext(); if (!gstGlContext) return false; @@ -269,15 +241,47 @@ bool QGstVideoRenderer::query(GstQuery *query) void QGstVideoRenderer::gstEvent(GstEvent *event) { - if (GST_EVENT_TYPE(event) != GST_EVENT_TAG) + switch (GST_EVENT_TYPE(event)) { + case GST_EVENT_TAG: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: Tag"; + return gstEventHandleTag(event); + case GST_EVENT_EOS: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: EOS"; + return gstEventHandleEOS(event); + + default: + qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::gstEvent: unhandled event - " << event; + return; + } +} + +void QGstVideoRenderer::setActive(bool isActive) +{ + if (isActive == m_isActive) return; + m_isActive = isActive; + if (isActive) + updateCurrentVideoFrame(m_currentPipelineFrame); + else + updateCurrentVideoFrame({}); +} + +void QGstVideoRenderer::updateCurrentVideoFrame(QVideoFrame frame) +{ + m_currentVideoFrame = std::move(frame); + if (m_sink) + m_sink->setVideoFrame(m_currentVideoFrame); +} + +void QGstVideoRenderer::gstEventHandleTag(GstEvent *event) +{ GstTagList *taglist = nullptr; gst_event_parse_tag(event, &taglist); if (!taglist) return; - gchar *value = nullptr; + QGString value; if (!gst_tag_list_get_string(taglist, GST_TAG_IMAGE_ORIENTATION, &value)) return; @@ -289,203 +293,79 @@ void QGstVideoRenderer::gstEvent(GstEvent *event) bool mirrored = false; int rotationAngle = 0; - if (!strncmp(rotate, value, rotateLen)) { - rotationAngle = atoi(value + rotateLen); - } else if (!strncmp(flipRotate, value, flipRotateLen)) { + if (!strncmp(rotate, value.get(), rotateLen)) { + rotationAngle = atoi(value.get() + rotateLen); + } else if (!strncmp(flipRotate, value.get(), flipRotateLen)) { // To flip by horizontal axis is the same as to mirror by vertical axis // and rotate by 180 degrees. mirrored = true; - rotationAngle = (180 + atoi(value + flipRotateLen)) % 360; + rotationAngle = (180 + atoi(value.get() + flipRotateLen)) % 360; } - QMutexLocker locker(&m_mutex); m_frameMirrored = mirrored; switch (rotationAngle) { - case 0: m_frameRotationAngle = QVideoFrame::Rotation0; break; - case 90: m_frameRotationAngle = QVideoFrame::Rotation90; break; - case 180: m_frameRotationAngle = QVideoFrame::Rotation180; break; - case 270: m_frameRotationAngle = QVideoFrame::Rotation270; break; - default: m_frameRotationAngle = QVideoFrame::Rotation0; + case 0: + m_frameRotationAngle = QtVideo::Rotation::None; + break; + case 90: + m_frameRotationAngle = QtVideo::Rotation::Clockwise90; + break; + case 180: + m_frameRotationAngle = QtVideo::Rotation::Clockwise180; + break; + case 270: + m_frameRotationAngle = QtVideo::Rotation::Clockwise270; + break; + default: + m_frameRotationAngle = QtVideo::Rotation::None; } } -bool QGstVideoRenderer::event(QEvent *event) +void QGstVideoRenderer::gstEventHandleEOS(GstEvent *) { - if (event->type() == QEvent::UpdateRequest) { - QMutexLocker locker(&m_mutex); - - if (m_notified) { - while (handleEvent(&locker)) {} - m_notified = false; - } - return true; - } - - return QObject::event(event); + stop(); } -bool QGstVideoRenderer::handleEvent(QMutexLocker<QMutex> *locker) -{ - if (m_flush) { - m_flush = false; - if (m_active) { - locker->unlock(); - - if (m_sink && !m_flushed) - m_sink->setVideoFrame(QVideoFrame()); - m_flushed = true; - } - } else if (m_stop) { - m_stop = false; - - if (m_active) { - m_active = false; - m_flushed = true; - } - } else if (!m_startCaps.isNull()) { - Q_ASSERT(!m_active); - - auto startCaps = m_startCaps; - m_startCaps = nullptr; - - if (m_sink) { - locker->unlock(); - - m_flushed = true; - m_format = startCaps.formatForCaps(&m_videoInfo); - memoryFormat = startCaps.memoryFormat(); - - locker->relock(); - m_active = m_format.isValid(); - } else if (m_active) { - m_active = false; - m_flushed = true; - } - - } else if (m_renderBuffer) { - GstBuffer *buffer = m_renderBuffer; - m_renderBuffer = nullptr; - m_renderReturn = GST_FLOW_ERROR; - - qCDebug(qLcGstVideoRenderer) << "QGstVideoRenderer::handleEvent(renderBuffer)" << m_active << m_sink; - if (m_active && m_sink) { - gst_buffer_ref(buffer); - - locker->unlock(); - - m_flushed = false; - - auto meta = gst_buffer_get_video_crop_meta (buffer); - if (meta) { - QRect vp(meta->x, meta->y, meta->width, meta->height); - if (m_format.viewport() != vp) { - qCDebug(qLcGstVideoRenderer) << Q_FUNC_INFO << " Update viewport on Metadata: [" << meta->height << "x" << meta->width << " | " << meta->x << "x" << meta->y << "]"; - // Update viewport if data is not the same - m_format.setViewport(vp); - } - } - - if (m_sink->inStoppedState()) { - qCDebug(qLcGstVideoRenderer) << " sending empty video frame"; - m_sink->setVideoFrame(QVideoFrame()); - } else { - QGstVideoBuffer *videoBuffer = new QGstVideoBuffer(buffer, m_videoInfo, m_sink, m_format, memoryFormat); - QVideoFrame frame(videoBuffer, m_format); - QGstUtils::setFrameTimeStamps(&frame, buffer); - frame.setMirrored(m_frameMirrored); - frame.setRotationAngle(m_frameRotationAngle); - - qCDebug(qLcGstVideoRenderer) << " sending video frame"; - m_sink->setVideoFrame(frame); - } - - gst_buffer_unref(buffer); - - locker->relock(); - - m_renderReturn = GST_FLOW_OK; - } - - m_renderCondition.wakeAll(); - } else { - m_setupCondition.wakeAll(); - - return false; - } - return true; -} - -void QGstVideoRenderer::notify() -{ - if (!m_notified) { - m_notified = true; - QCoreApplication::postEvent(this, new QEvent(QEvent::UpdateRequest)); - } -} - -bool QGstVideoRenderer::waitForAsyncEvent( - QMutexLocker<QMutex> *locker, QWaitCondition *condition, unsigned long time) -{ - if (QThread::currentThread() == thread()) { - while (handleEvent(locker)) {} - m_notified = false; - - return true; - } - - notify(); - - return condition->wait(&m_mutex, time); -} - -static GstVideoSinkClass *sink_parent_class; -static thread_local QGstreamerVideoSink *current_sink; +static GstVideoSinkClass *gvrs_sink_parent_class; +static thread_local QGstreamerVideoSink *gvrs_current_sink; #define VO_SINK(s) QGstVideoRendererSink *sink(reinterpret_cast<QGstVideoRendererSink *>(s)) -QGstVideoRendererSink *QGstVideoRendererSink::createSink(QGstreamerVideoSink *sink) +QGstVideoRendererSinkElement QGstVideoRendererSink::createSink(QGstreamerVideoSink *sink) { setSink(sink); QGstVideoRendererSink *gstSink = reinterpret_cast<QGstVideoRendererSink *>( g_object_new(QGstVideoRendererSink::get_type(), nullptr)); - g_signal_connect(G_OBJECT(gstSink), "notify::show-preroll-frame", G_CALLBACK(handleShowPrerollChange), gstSink); - - return gstSink; + return QGstVideoRendererSinkElement{ + gstSink, + QGstElement::NeedsRef, + }; } void QGstVideoRendererSink::setSink(QGstreamerVideoSink *sink) { - current_sink = sink; - get_type(); + gvrs_current_sink = sink; } GType QGstVideoRendererSink::get_type() { - static GType type = 0; - - if (type == 0) { - static const GTypeInfo info = - { - sizeof(QGstVideoRendererSinkClass), // class_size - base_init, // base_init - nullptr, // base_finalize - class_init, // class_init - nullptr, // class_finalize - nullptr, // class_data - sizeof(QGstVideoRendererSink), // instance_size - 0, // n_preallocs - instance_init, // instance_init - nullptr // value_table - }; - - type = g_type_register_static( - GST_TYPE_VIDEO_SINK, "QGstVideoRendererSink", &info, GTypeFlags(0)); - - // Register the sink type to be used in custom piplines. - // When surface is ready the sink can be used. - gst_element_register(nullptr, "qtvideosink", GST_RANK_PRIMARY, type); - } + static const GTypeInfo info = + { + sizeof(QGstVideoRendererSinkClass), // class_size + base_init, // base_init + nullptr, // base_finalize + class_init, // class_init + nullptr, // class_finalize + nullptr, // class_data + sizeof(QGstVideoRendererSink), // instance_size + 0, // n_preallocs + instance_init, // instance_init + nullptr // value_table + }; + + static const GType type = g_type_register_static(GST_TYPE_VIDEO_SINK, "QGstVideoRendererSink", + &info, GTypeFlags(0)); return type; } @@ -494,7 +374,7 @@ void QGstVideoRendererSink::class_init(gpointer g_class, gpointer class_data) { Q_UNUSED(class_data); - sink_parent_class = reinterpret_cast<GstVideoSinkClass *>(g_type_class_peek_parent(g_class)); + gvrs_sink_parent_class = reinterpret_cast<GstVideoSinkClass *>(g_type_class_peek_parent(g_class)); GstVideoSinkClass *video_sink_class = reinterpret_cast<GstVideoSinkClass *>(g_class); video_sink_class->show_frame = QGstVideoRendererSink::show_frame; @@ -538,11 +418,11 @@ void QGstVideoRendererSink::instance_init(GTypeInstance *instance, gpointer g_cl Q_UNUSED(g_class); VO_SINK(instance); - Q_ASSERT(current_sink); + Q_ASSERT(gvrs_current_sink); - sink->renderer = new QGstVideoRenderer(current_sink); - sink->renderer->moveToThread(current_sink->thread()); - current_sink = nullptr; + sink->renderer = new QGstVideoRenderer(gvrs_current_sink); + sink->renderer->moveToThread(gvrs_current_sink->thread()); + gvrs_current_sink = nullptr; } void QGstVideoRendererSink::finalize(GObject *object) @@ -552,74 +432,39 @@ void QGstVideoRendererSink::finalize(GObject *object) delete sink->renderer; // Chain up - G_OBJECT_CLASS(sink_parent_class)->finalize(object); -} - -void QGstVideoRendererSink::handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d) -{ - Q_UNUSED(o); - Q_UNUSED(p); - QGstVideoRendererSink *sink = reinterpret_cast<QGstVideoRendererSink *>(d); - - gboolean showPrerollFrame = true; // "show-preroll-frame" property is true by default - g_object_get(G_OBJECT(sink), "show-preroll-frame", &showPrerollFrame, nullptr); - - if (!showPrerollFrame) { - GstState state = GST_STATE_VOID_PENDING; - GstClockTime timeout = 10000000; // 10 ms - gst_element_get_state(GST_ELEMENT(sink), &state, nullptr, timeout); - // show-preroll-frame being set to 'false' while in GST_STATE_PAUSED means - // the QMediaPlayer was stopped from the paused state. - // We need to flush the current frame. - if (state == GST_STATE_PAUSED) - sink->renderer->flush(); - } + G_OBJECT_CLASS(gvrs_sink_parent_class)->finalize(object); } GstStateChangeReturn QGstVideoRendererSink::change_state( GstElement *element, GstStateChange transition) { - QGstVideoRendererSink *sink = reinterpret_cast<QGstVideoRendererSink *>(element); - - gboolean showPrerollFrame = true; // "show-preroll-frame" property is true by default - g_object_get(G_OBJECT(element), "show-preroll-frame", &showPrerollFrame, nullptr); - - // If show-preroll-frame is 'false' when transitioning from GST_STATE_PLAYING to - // GST_STATE_PAUSED, it means the QMediaPlayer was stopped. - // We need to flush the current frame. - if (transition == GST_STATE_CHANGE_PLAYING_TO_PAUSED && !showPrerollFrame) - sink->renderer->flush(); - - return GST_ELEMENT_CLASS(sink_parent_class)->change_state(element, transition); + return GST_ELEMENT_CLASS(gvrs_sink_parent_class)->change_state(element, transition); } GstCaps *QGstVideoRendererSink::get_caps(GstBaseSink *base, GstCaps *filter) { VO_SINK(base); - QGstMutableCaps caps = sink->renderer->caps(); + QGstCaps caps = sink->renderer->caps(); if (filter) - caps = gst_caps_intersect(caps.get(), filter); + caps = QGstCaps(gst_caps_intersect(caps.caps(), filter), QGstCaps::HasRef); - gst_caps_ref(caps.get()); - return caps.get(); + return caps.release(); } -gboolean QGstVideoRendererSink::set_caps(GstBaseSink *base, GstCaps *caps) +gboolean QGstVideoRendererSink::set_caps(GstBaseSink *base, GstCaps *gcaps) { VO_SINK(base); + auto caps = QGstCaps(gcaps, QGstCaps::NeedsRef); - qCDebug(qLcGstVideoRenderer) << "set_caps:" << QGstCaps(caps).toString(); + qCDebug(qLcGstVideoRenderer) << "set_caps:" << caps; - if (!caps) { + if (caps.isNull()) { sink->renderer->stop(); - return TRUE; - } else if (sink->renderer->start(caps)) { - return TRUE; - } else { - return FALSE; } + + return sink->renderer->start(caps); } gboolean QGstVideoRendererSink::propose_allocation(GstBaseSink *base, GstQuery *query) @@ -654,14 +499,33 @@ gboolean QGstVideoRendererSink::query(GstBaseSink *base, GstQuery *query) if (sink->renderer->query(query)) return TRUE; - return GST_BASE_SINK_CLASS(sink_parent_class)->query(base, query); + return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->query(base, query); } gboolean QGstVideoRendererSink::event(GstBaseSink *base, GstEvent * event) { VO_SINK(base); sink->renderer->gstEvent(event); - return GST_BASE_SINK_CLASS(sink_parent_class)->event(base, event); + return GST_BASE_SINK_CLASS(gvrs_sink_parent_class)->event(base, event); +} + +QGstVideoRendererSinkElement::QGstVideoRendererSinkElement(QGstVideoRendererSink *element, + RefMode mode) + : QGstBaseSink{ + qGstCheckedCast<GstBaseSink>(element), + mode, + } +{ +} + +void QGstVideoRendererSinkElement::setActive(bool isActive) +{ + qGstVideoRendererSink()->renderer->setActive(isActive); +} + +QGstVideoRendererSink *QGstVideoRendererSinkElement::qGstVideoRendererSink() const +{ + return reinterpret_cast<QGstVideoRendererSink *>(element()); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h index 18530f62e..60cd25db0 100644 --- a/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h +++ b/src/plugins/multimedia/gstreamer/common/qgstvideorenderersink_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Jolla Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Jolla Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTVIDEORENDERERSINK_P_H #define QGSTVIDEORENDERERSINK_P_H @@ -51,7 +15,11 @@ // We mean it. // +#include <QtMultimedia/qvideoframeformat.h> +#include <QtMultimedia/qvideoframe.h> #include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtCore/qmutex.h> + #include <gst/video/gstvideosink.h> #include <gst/video/video.h> @@ -62,79 +30,85 @@ #include <QtCore/qwaitcondition.h> #include <qvideoframeformat.h> #include <qvideoframe.h> -#include <qgstvideobuffer_p.h> -#include <qgst_p.h> +#include <common/qgstvideobuffer_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE -class QVideoSink; class QGstVideoRenderer : public QObject { - Q_OBJECT public: - QGstVideoRenderer(QGstreamerVideoSink *sink); + explicit QGstVideoRenderer(QGstreamerVideoSink *); ~QGstVideoRenderer(); - QGstMutableCaps caps(); + const QGstCaps &caps(); - bool start(GstCaps *caps); + bool start(const QGstCaps &); void stop(); void unlock(); - bool proposeAllocation(GstQuery *query); - - void flush(); - - GstFlowReturn render(GstBuffer *buffer); + bool proposeAllocation(GstQuery *); + GstFlowReturn render(GstBuffer *); + bool query(GstQuery *); + void gstEvent(GstEvent *); - bool event(QEvent *event) override; - bool query(GstQuery *query); - void gstEvent(GstEvent *event); - -private slots: - bool handleEvent(QMutexLocker<QMutex> *locker); + void setActive(bool); private: - void notify(); - bool waitForAsyncEvent(QMutexLocker<QMutex> *locker, QWaitCondition *condition, unsigned long time); - void createSurfaceCaps(); - - QPointer<QGstreamerVideoSink> m_sink; + void updateCurrentVideoFrame(QVideoFrame); - QMutex m_mutex; - QWaitCondition m_setupCondition; - QWaitCondition m_renderCondition; - - // --- accessed from multiple threads, need to hold mutex to access - GstFlowReturn m_renderReturn = GST_FLOW_OK; - bool m_active = false; - - QGstMutableCaps m_surfaceCaps; + void notify(); + static QGstCaps createSurfaceCaps(QGstreamerVideoSink *); - QGstMutableCaps m_startCaps; - GstBuffer *m_renderBuffer = nullptr; + void gstEventHandleTag(GstEvent *); + void gstEventHandleEOS(GstEvent *); - bool m_notified = false; - bool m_stop = false; - bool m_flush = false; - bool m_frameMirrored = false; - QVideoFrame::RotationAngle m_frameRotationAngle = QVideoFrame::Rotation0; + QMutex m_sinkMutex; + QGstreamerVideoSink *m_sink = nullptr; // written only from qt thread. so only readers on + // worker threads need to acquire the lock - // --- only accessed from one thread + // --- only accessed from gstreamer thread + const QGstCaps m_surfaceCaps; QVideoFrameFormat m_format; - GstVideoInfo m_videoInfo; - bool m_flushed = true; - QGstCaps::MemoryFormat memoryFormat = QGstCaps::CpuMemory; + GstVideoInfo m_videoInfo{}; + QGstCaps::MemoryFormat m_memoryFormat = QGstCaps::CpuMemory; + bool m_frameMirrored = false; + QtVideo::Rotation m_frameRotationAngle = QtVideo::Rotation::None; + + // --- only accessed from qt thread + QVideoFrame m_currentPipelineFrame; + QVideoFrame m_currentVideoFrame; + bool m_isActive{ false }; + + struct RenderBufferState + { + QGstBufferHandle buffer; + QVideoFrameFormat format; + QGstCaps::MemoryFormat memoryFormat; + bool mirrored; + QtVideo::Rotation rotationAngle; + + bool operator==(const RenderBufferState &rhs) const + { + return std::tie(buffer, format, memoryFormat, mirrored, rotationAngle) + == std::tie(rhs.buffer, rhs.format, rhs.memoryFormat, rhs.mirrored, + rhs.rotationAngle); + } + }; + RenderBufferState m_currentState; }; -class Q_MULTIMEDIA_EXPORT QGstVideoRendererSink +class QGstVideoRendererSinkElement; + +class QGstVideoRendererSink { public: - GstVideoSink parent; + GstVideoSink parent{}; - static QGstVideoRendererSink *createSink(QGstreamerVideoSink *surface); - static void setSink(QGstreamerVideoSink *surface); + static QGstVideoRendererSinkElement createSink(QGstreamerVideoSink *surface); private: + static void setSink(QGstreamerVideoSink *surface); + static GType get_type(); static void class_init(gpointer g_class, gpointer class_data); static void base_init(gpointer g_class); @@ -142,8 +116,6 @@ private: static void finalize(GObject *object); - static void handleShowPrerollChange(GObject *o, GParamSpec *p, gpointer d); - static GstStateChangeReturn change_state(GstElement *element, GstStateChange transition); static GstCaps *get_caps(GstBaseSink *sink, GstCaps *filter); @@ -157,19 +129,36 @@ private: static GstFlowReturn show_frame(GstVideoSink *sink, GstBuffer *buffer); static gboolean query(GstBaseSink *element, GstQuery *query); - static gboolean event(GstBaseSink *element, GstEvent * event); + static gboolean event(GstBaseSink *element, GstEvent *event); + + friend class QGstVideoRendererSinkElement; -private: QGstVideoRenderer *renderer = nullptr; }; - class QGstVideoRendererSinkClass { public: GstVideoSinkClass parent_class; }; +class QGstVideoRendererSinkElement : public QGstBaseSink +{ +public: + using QGstBaseSink::QGstBaseSink; + + explicit QGstVideoRendererSinkElement(QGstVideoRendererSink *, RefMode); + + QGstVideoRendererSinkElement(const QGstVideoRendererSinkElement &) = default; + QGstVideoRendererSinkElement(QGstVideoRendererSinkElement &&) noexcept = default; + QGstVideoRendererSinkElement &operator=(const QGstVideoRendererSinkElement &) = default; + QGstVideoRendererSinkElement &operator=(QGstVideoRendererSinkElement &&) noexcept = default; + + void setActive(bool); + + QGstVideoRendererSink *qGstVideoRendererSink() const; +}; + QT_END_NAMESPACE #endif diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp index 24f4301ee..c54e8b74b 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera.cpp @@ -1,78 +1,63 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include <qcameradevice.h> - -#include "qgstreamercamera_p.h" -#include "qgstreamerimagecapture_p.h" +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <mediacapture/qgstreamercamera_p.h> + +#include <QtMultimedia/qcameradevice.h> +#include <QtMultimedia/qmediacapturesession.h> +#include <QtMultimedia/private/qcameradevice_p.h> +#include <QtCore/qdebug.h> + +#include <common/qgst_debug_p.h> #include <qgstreamervideodevices_p.h> #include <qgstreamerintegration_p.h> -#include <qmediacapturesession.h> #if QT_CONFIG(linux_v4l) #include <linux/videodev2.h> #include <private/qcore_unix_p.h> #endif -#include <QtCore/qdebug.h> + +QT_BEGIN_NAMESPACE + +QMaybe<QPlatformCamera *> QGstreamerCamera::create(QCamera *camera) +{ + static const auto error = qGstErrorMessageIfElementsNotAvailable( + "videotestsrc", "capsfilter", "videoconvert", "videoscale", "identity"); + if (error) + return *error; + + return new QGstreamerCamera(camera); +} QGstreamerCamera::QGstreamerCamera(QCamera *camera) - : QPlatformCamera(camera) -{ - gstCamera = QGstElement("videotestsrc"); - gstCapsFilter = QGstElement("capsfilter", "videoCapsFilter"); - gstDecode = QGstElement("identity"); - gstVideoConvert = QGstElement("videoconvert", "videoConvert"); - gstVideoScale = QGstElement("videoscale", "videoScale"); - gstCameraBin = QGstBin("camerabin"); + : QGstreamerCameraBase(camera), + gstCameraBin{ + QGstBin::create("camerabin"), + }, + gstCamera{ + QGstElement::createFromFactory("videotestsrc"), + }, + gstCapsFilter{ + QGstElement::createFromFactory("capsfilter", "videoCapsFilter"), + }, + gstDecode{ + QGstElement::createFromFactory("identity"), + }, + gstVideoConvert{ + QGstElement::createFromFactory("videoconvert", "videoConvert"), + }, + gstVideoScale{ + QGstElement::createFromFactory("videoscale", "videoScale"), + } +{ gstCameraBin.add(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert, gstVideoScale); - gstCamera.link(gstCapsFilter, gstDecode, gstVideoConvert, gstVideoScale); + qLinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert, gstVideoScale); gstCameraBin.addGhostPad(gstVideoScale, "src"); } QGstreamerCamera::~QGstreamerCamera() { -#if QT_CONFIG(linux_v4l) - if (v4l2FileDescriptor >= 0) - qt_safe_close(v4l2FileDescriptor); - v4l2FileDescriptor = -1; -#endif gstCameraBin.setStateSync(GST_STATE_NULL); } @@ -95,6 +80,8 @@ void QGstreamerCamera::setActive(bool active) void QGstreamerCamera::setCamera(const QCameraDevice &camera) { + using namespace Qt::Literals; + if (m_cameraDevice == camera) return; @@ -102,48 +89,51 @@ void QGstreamerCamera::setCamera(const QCameraDevice &camera) QGstElement gstNewCamera; if (camera.isNull()) { - gstNewCamera = QGstElement("videotestsrc"); + gstNewCamera = QGstElement::createFromFactory("videotestsrc"); } else { auto *integration = static_cast<QGstreamerIntegration *>(QGstreamerIntegration::instance()); - auto *device = integration->videoDevice(camera.id()); - gstNewCamera = gst_device_create_element(device, "camerasrc"); - QGstStructure properties = gst_device_get_properties(device); - if (properties.name() == "v4l2deviceprovider") - m_v4l2Device = QString::fromUtf8(properties["device.path"].toString()); - } + GstDevice *device = integration->videoDevice(camera.id()); - QCameraFormat f = findBestCameraFormat(camera); - auto caps = QGstMutableCaps::fromCameraFormat(f); - auto gstNewDecode = QGstElement(f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); + if (!device) { + updateError(QCamera::Error::CameraError, + u"Failed to create GstDevice for camera: "_s + + QString::fromUtf8(camera.id())); + return; + } - gstCamera.unlink(gstCapsFilter); - gstCapsFilter.unlink(gstDecode); - gstDecode.unlink(gstVideoConvert); + gstNewCamera = QGstElement::createFromDevice(device, "camerasrc"); + QUniqueGstStructureHandle properties{ + gst_device_get_properties(device), + }; - gstCameraBin.remove(gstCamera); - gstCameraBin.remove(gstDecode); + if (properties) { + QGstStructureView propertiesView{ properties }; + if (propertiesView.name() == "v4l2deviceprovider") + m_v4l2DevicePath = QString::fromUtf8(propertiesView["device.path"].toString()); + } + } - gstCamera.setStateSync(GST_STATE_NULL); - gstDecode.setStateSync(GST_STATE_NULL); + QCameraFormat f = findBestCameraFormat(camera); + auto caps = QGstCaps::fromCameraFormat(f); + auto gstNewDecode = QGstElement::createFromFactory( + f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); - gstCapsFilter.set("caps", caps); + QGstPipeline::modifyPipelineWhileNotRunning(gstCamera.getPipeline(), [&] { + gstCamera.setStateSync(GST_STATE_READY); // stop camera, as it may have active tasks - gstCameraBin.add(gstNewCamera, gstNewDecode); + qUnlinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); + gstCameraBin.stopAndRemoveElements(gstCamera, gstDecode); - gstNewDecode.link(gstVideoConvert); - gstCapsFilter.link(gstNewDecode); + gstCapsFilter.set("caps", caps); - if (!gstNewCamera.link(gstCapsFilter)) - qWarning() << "linking camera failed" << gstCamera.name() << caps.toString(); + gstCamera = std::move(gstNewCamera); + gstDecode = std::move(gstNewDecode); - // Start sending frames once pipeline is linked - // FIXME: put camera to READY state before linking to decoder as in the NULL state it does not know its true caps - gstCapsFilter.syncStateWithParent(); - gstNewDecode.syncStateWithParent(); - gstNewCamera.syncStateWithParent(); + gstCameraBin.add(gstCamera, gstDecode); + qLinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); - gstCamera = gstNewCamera; - gstDecode = gstNewDecode; + gstCameraBin.syncChildrenState(); + }); updateCameraProperties(); } @@ -157,29 +147,25 @@ bool QGstreamerCamera::setCameraFormat(const QCameraFormat &format) if (f.isNull()) f = findBestCameraFormat(m_cameraDevice); - auto caps = QGstMutableCaps::fromCameraFormat(f); + auto caps = QGstCaps::fromCameraFormat(f); - auto newGstDecode = QGstElement(f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); - gstCameraBin.add(newGstDecode); - newGstDecode.syncStateWithParent(); + auto newGstDecode = QGstElement::createFromFactory( + f.pixelFormat() == QVideoFrameFormat::Format_Jpeg ? "jpegdec" : "identity"); - gstCamera.staticPad("src").doInIdleProbe([&](){ - gstCamera.unlink(gstCapsFilter); - gstCapsFilter.unlink(gstDecode); - gstDecode.unlink(gstVideoConvert); + QGstPipeline::modifyPipelineWhileNotRunning(gstCamera.getPipeline(), [&] { + gstCamera.setStateSync(GST_STATE_READY); // stop camera, as it may have active tasks - gstCapsFilter.set("caps", caps); + qUnlinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); + gstCameraBin.stopAndRemoveElements(gstDecode); - newGstDecode.link(gstVideoConvert); - gstCapsFilter.link(newGstDecode); - if (!gstCamera.link(gstCapsFilter)) - qWarning() << "linking filtered camera to decoder failed" << gstCamera.name() << caps.toString(); - }); + gstCapsFilter.set("caps", caps); - gstCameraBin.remove(gstDecode); - gstDecode.setStateSync(GST_STATE_NULL); + gstDecode = std::move(newGstDecode); - gstDecode = newGstDecode; + gstCameraBin.add(gstDecode); + qLinkGstElements(gstCamera, gstCapsFilter, gstDecode, gstVideoConvert); + gstCameraBin.syncChildrenState(); + }); return true; } @@ -618,83 +604,78 @@ void QGstreamerCamera::setColorTemperature(int temperature) } #if QT_CONFIG(linux_v4l) +bool QGstreamerCamera::isV4L2Camera() const +{ + return !m_v4l2DevicePath.isEmpty(); +} + void QGstreamerCamera::initV4L2Controls() { v4l2AutoWhiteBalanceSupported = false; v4l2ColorTemperatureSupported = false; - QCamera::Features features; + QCamera::Features features{}; + Q_ASSERT(!m_v4l2DevicePath.isEmpty()); - const QString deviceName = v4l2Device(); - Q_ASSERT(!deviceName.isEmpty()); - v4l2FileDescriptor = qt_safe_open(deviceName.toLocal8Bit().constData(), O_RDONLY); - if (v4l2FileDescriptor == -1) { - qWarning() << "Unable to open the camera" << deviceName - << "for read to query the parameter info:" << qt_error_string(errno); - return; - } - - struct v4l2_queryctrl queryControl; - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_AUTO_WHITE_BALANCE; + withV4L2DeviceFileDescriptor([&](int fd) { + struct v4l2_queryctrl queryControl = {}; + queryControl.id = V4L2_CID_AUTO_WHITE_BALANCE; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2AutoWhiteBalanceSupported = true; - setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, true); - } + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2AutoWhiteBalanceSupported = true; + setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, true); + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_WHITE_BALANCE_TEMPERATURE; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinColorTemp = queryControl.minimum; - v4l2MaxColorTemp = queryControl.maximum; - v4l2ColorTemperatureSupported = true; - features |= QCamera::Feature::ColorTemperature; - } + queryControl = {}; + queryControl.id = V4L2_CID_WHITE_BALANCE_TEMPERATURE; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2MinColorTemp = queryControl.minimum; + v4l2MaxColorTemp = queryControl.maximum; + v4l2ColorTemperatureSupported = true; + features |= QCamera::Feature::ColorTemperature; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_EXPOSURE_AUTO; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2AutoExposureSupported = true; - } + queryControl = {}; + queryControl.id = V4L2_CID_EXPOSURE_AUTO; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2AutoExposureSupported = true; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_EXPOSURE_ABSOLUTE; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2ManualExposureSupported = true; - v4l2MinExposure = queryControl.minimum; - v4l2MaxExposure = queryControl.maximum; - features |= QCamera::Feature::ManualExposureTime; - } + queryControl = {}; + queryControl.id = V4L2_CID_EXPOSURE_ABSOLUTE; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2ManualExposureSupported = true; + v4l2MinExposure = queryControl.minimum; + v4l2MaxExposure = queryControl.maximum; + features |= QCamera::Feature::ManualExposureTime; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_AUTO_EXPOSURE_BIAS; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - v4l2MinExposureAdjustment = queryControl.minimum; - v4l2MaxExposureAdjustment = queryControl.maximum; - features |= QCamera::Feature::ExposureCompensation; - } + queryControl = {}; + queryControl.id = V4L2_CID_AUTO_EXPOSURE_BIAS; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + v4l2MinExposureAdjustment = queryControl.minimum; + v4l2MaxExposureAdjustment = queryControl.maximum; + features |= QCamera::Feature::ExposureCompensation; + } - ::memset(&queryControl, 0, sizeof(queryControl)); - queryControl.id = V4L2_CID_ISO_SENSITIVITY_AUTO; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - queryControl.id = V4L2_CID_ISO_SENSITIVITY; - if (::ioctl(v4l2FileDescriptor, VIDIOC_QUERYCTRL, &queryControl) == 0) { - features |= QCamera::Feature::IsoSensitivity; - minIsoChanged(queryControl.minimum); - maxIsoChanged(queryControl.minimum); + queryControl = {}; + queryControl.id = V4L2_CID_ISO_SENSITIVITY_AUTO; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + queryControl.id = V4L2_CID_ISO_SENSITIVITY; + if (::ioctl(fd, VIDIOC_QUERYCTRL, &queryControl) == 0) { + features |= QCamera::Feature::IsoSensitivity; + minIsoChanged(queryControl.minimum); + maxIsoChanged(queryControl.minimum); + } } - } + }); supportedFeaturesChanged(features); } int QGstreamerCamera::setV4L2ColorTemperature(int temperature) { - struct v4l2_control control; - ::memset(&control, 0, sizeof(control)); - if (v4l2AutoWhiteBalanceSupported) { setV4L2Parameter(V4L2_CID_AUTO_WHITE_BALANCE, temperature == 0 ? true : false); } else if (temperature == 0) { @@ -714,22 +695,77 @@ int QGstreamerCamera::setV4L2ColorTemperature(int temperature) bool QGstreamerCamera::setV4L2Parameter(quint32 id, qint32 value) { - struct v4l2_control control{id, value}; - if (::ioctl(v4l2FileDescriptor, VIDIOC_S_CTRL, &control) != 0) { - qWarning() << "Unable to set the V4L2 Parameter" << Qt::hex << id << "to" << value << qt_error_string(errno); - return false; - } - return true; + return withV4L2DeviceFileDescriptor([&](int fd) { + v4l2_control control{ id, value }; + if (::ioctl(fd, VIDIOC_S_CTRL, &control) != 0) { + qWarning() << "Unable to set the V4L2 Parameter" << Qt::hex << id << "to" << value + << qt_error_string(errno); + return false; + } + return true; + }); } int QGstreamerCamera::getV4L2Parameter(quint32 id) const { - struct v4l2_control control{id, 0}; - if (::ioctl(v4l2FileDescriptor, VIDIOC_G_CTRL, &control) != 0) { - qWarning() << "Unable to get the V4L2 Parameter" << Qt::hex << id << qt_error_string(errno); - return 0; - } - return control.value; + return withV4L2DeviceFileDescriptor([&](int fd) { + v4l2_control control{ id, 0 }; + if (::ioctl(fd, VIDIOC_G_CTRL, &control) != 0) { + qWarning() << "Unable to get the V4L2 Parameter" << Qt::hex << id + << qt_error_string(errno); + return 0; + } + return control.value; + }); +} + +QGstreamerCustomCamera::QGstreamerCustomCamera(QCamera *camera) + : QGstreamerCameraBase{ + camera, + }, + m_userProvidedGstElement{ + false, + } +{ +} + +QGstreamerCustomCamera::QGstreamerCustomCamera(QCamera *camera, QGstElement element) + : QGstreamerCameraBase{ + camera, + }, + gstCamera{ + std::move(element), + }, + m_userProvidedGstElement{ + true, + } +{ +} + +void QGstreamerCustomCamera::setCamera(const QCameraDevice &device) +{ + if (m_userProvidedGstElement) + return; + + gstCamera = QGstBin::createFromPipelineDescription(device.id(), /*name=*/nullptr, + /* ghostUnlinkedPads=*/true); +} + +bool QGstreamerCustomCamera::isActive() const +{ + return m_active; +} + +void QGstreamerCustomCamera::setActive(bool active) +{ + if (m_active == active) + return; + + m_active = active; + + emit activeChanged(active); } #endif + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h index ac57eb5ef..f43c01f34 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamercamera_p.h @@ -1,42 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERCAMERACONTROL_H #define QGSTREAMERCAMERACONTROL_H @@ -52,18 +15,28 @@ // We mean it. // -#include <QHash> #include <private/qplatformcamera_p.h> -#include "qgstreamermediacapture_p.h" -#include <qgst_p.h> +#include <private/qmultimediautils_p.h> + +#include <mediacapture/qgstreamermediacapture_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> QT_BEGIN_NAMESPACE -class QGstreamerCamera : public QPlatformCamera +class QGstreamerCameraBase : public QPlatformCamera { - Q_OBJECT public: - QGstreamerCamera(QCamera *camera); + using QPlatformCamera::QPlatformCamera; + + virtual QGstElement gstElement() const = 0; +}; + +class QGstreamerCamera : public QGstreamerCameraBase +{ +public: + static QMaybe<QPlatformCamera *> create(QCamera *camera); + virtual ~QGstreamerCamera(); bool isActive() const override; @@ -72,7 +45,7 @@ public: void setCamera(const QCameraDevice &camera) override; bool setCameraFormat(const QCameraFormat &format) override; - QGstElement gstElement() const { return gstCameraBin.element(); } + QGstElement gstElement() const override { return gstCameraBin; } #if QT_CONFIG(gstreamer_photography) GstPhotography *photography() const; #endif @@ -96,12 +69,13 @@ public: void setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) override; void setColorTemperature(int temperature) override; - QString v4l2Device() const { return m_v4l2Device; } - bool isV4L2Camera() const { return !m_v4l2Device.isEmpty(); } - private: + QGstreamerCamera(QCamera *camera); + void updateCameraProperties(); + #if QT_CONFIG(linux_v4l) + bool isV4L2Camera() const; void initV4L2Controls(); int setV4L2ColorTemperature(int temperature); bool setV4L2Parameter(quint32 id, qint32 value); @@ -117,7 +91,29 @@ private: qint32 v4l2MaxExposure = 0; qint32 v4l2MinExposureAdjustment = 0; qint32 v4l2MaxExposureAdjustment = 0; - int v4l2FileDescriptor = -1; + + template <typename Functor> + auto withV4L2DeviceFileDescriptor(Functor &&f) const + { + using ReturnType = std::invoke_result_t<Functor, int>; + Q_ASSERT(isV4L2Camera()); + + if (int gstreamerDeviceFd = gstCamera.getInt("device-fd"); gstreamerDeviceFd != -1) + return f(gstreamerDeviceFd); + + auto v4l2FileDescriptor = QFileDescriptorHandle{ + qt_safe_open(m_v4l2DevicePath.toLocal8Bit().constData(), O_RDONLY), + }; + if (!v4l2FileDescriptor) { + qWarning() << "Unable to open the camera" << m_v4l2DevicePath + << "for read to query the parameter info:" << qt_error_string(errno); + if constexpr (std::is_void_v<ReturnType>) + return; + else + return ReturnType{}; + } + return f(v4l2FileDescriptor.get()); + } #endif QCameraDevice m_cameraDevice; @@ -130,7 +126,25 @@ private: QGstElement gstVideoScale; bool m_active = false; - QString m_v4l2Device; + QString m_v4l2DevicePath; +}; + +class QGstreamerCustomCamera : public QGstreamerCameraBase +{ +public: + explicit QGstreamerCustomCamera(QCamera *); + explicit QGstreamerCustomCamera(QCamera *, QGstElement element); + + QGstElement gstElement() const override { return gstCamera; } + void setCamera(const QCameraDevice &) override; + + bool isActive() const override; + void setActive(bool) override; + +private: + QGstElement gstCamera; + bool m_active{}; + const bool m_userProvidedGstElement; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp index 91e48e9c7..9c21dc083 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture.cpp @@ -1,68 +1,117 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamerimagecapture_p.h" -#include <private/qplatformcamera_p.h> -#include <private/qplatformimagecapture_p.h> -#include <qgstvideobuffer_p.h> -#include <qgstutils_p.h> -#include <qgstreamermetadata_p.h> -#include <qvideoframeformat.h> -#include <private/qmediastoragelocation_p.h> -#include <QtCore/QDebug> -#include <QtCore/QDir> -#include <qstandardpaths.h> +#include <QtMultimedia/qvideoframeformat.h> +#include <QtMultimedia/private/qmediastoragelocation_p.h> +#include <QtMultimedia/private/qplatformcamera_p.h> +#include <QtMultimedia/private/qplatformimagecapture_p.h> +#include <QtMultimedia/private/qvideoframe_p.h> +#include <QtCore/qdebug.h> +#include <QtCore/qdir.h> +#include <QtCore/qstandardpaths.h> +#include <QtCore/qcoreapplication.h> +#include <QtCore/qloggingcategory.h> -#include <qloggingcategory.h> +#include <common/qgstreamermetadata_p.h> +#include <common/qgstvideobuffer_p.h> +#include <common/qgstutils_p.h> + +#include <utility> QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcImageCapture, "qt.multimedia.imageCapture") +namespace { +Q_LOGGING_CATEGORY(qLcImageCaptureGst, "qt.multimedia.imageCapture") -QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) - : QPlatformImageCapture(parent), - QGstreamerBufferProbe(ProbeBuffers) +struct ThreadPoolSingleton +{ + QObject m_context; + QMutex m_poolMutex; + QThreadPool *m_instance{}; + bool m_appUnderDestruction = false; + + QThreadPool *get(const QMutexLocker<QMutex> &) + { + if (m_instance) + return m_instance; + if (m_appUnderDestruction || !qApp) + return nullptr; + + using namespace std::chrono; + + m_instance = new QThreadPool(qApp); + m_instance->setMaxThreadCount(1); // 1 thread; + static constexpr auto expiryTimeout = minutes(5); + m_instance->setExpiryTimeout(round<milliseconds>(expiryTimeout).count()); + + QObject::connect(qApp, &QCoreApplication::aboutToQuit, &m_context, [&] { + // we need to make sure that thread-local QRhi is destroyed before the application to + // prevent QTBUG-124189 + QMutexLocker guard(&m_poolMutex); + delete m_instance; + m_instance = {}; + m_appUnderDestruction = true; + }); + + QObject::connect(qApp, &QCoreApplication::destroyed, &m_context, [&] { + m_appUnderDestruction = false; + }); + return m_instance; + } + + template <typename Functor> + QFuture<void> run(Functor &&f) + { + QMutexLocker guard(&m_poolMutex); + QThreadPool *pool = get(guard); + if (!pool) + return QFuture<void>{}; + + return QtConcurrent::run(pool, std::forward<Functor>(f)); + } +}; + +ThreadPoolSingleton s_threadPoolSingleton; + +}; // namespace + +QMaybe<QPlatformImageCapture *> QGstreamerImageCapture::create(QImageCapture *parent) { - bin = QGstBin("imageCaptureBin"); + static const auto error = qGstErrorMessageIfElementsNotAvailable( + "queue", "capsfilter", "videoconvert", "jpegenc", "jifmux", "fakesink"); + if (error) + return *error; + + return new QGstreamerImageCapture(parent); +} - queue = QGstElement("queue", "imageCaptureQueue"); +QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) + : QPlatformImageCapture(parent), + QGstreamerBufferProbe(ProbeBuffers), + bin{ + QGstBin::create("imageCaptureBin"), + }, + queue{ + QGstElement::createFromFactory("queue", "imageCaptureQueue"), + }, + filter{ + QGstElement::createFromFactory("capsfilter", "filter"), + }, + videoConvert{ + QGstElement::createFromFactory("videoconvert", "imageCaptureConvert"), + }, + encoder{ + QGstElement::createFromFactory("jpegenc", "jpegEncoder"), + }, + muxer{ + QGstElement::createFromFactory("jifmux", "jpegMuxer"), + }, + sink{ + QGstElement::createFromFactory("fakesink", "imageCaptureSink"), + } +{ // configures the queue to be fast, lightweight and non blocking queue.set("leaky", 2 /*downstream*/); queue.set("silent", true); @@ -70,37 +119,45 @@ QGstreamerImageCapture::QGstreamerImageCapture(QImageCapture *parent) queue.set("max-size-bytes", uint(0)); queue.set("max-size-time", quint64(0)); - videoConvert = QGstElement("videoconvert", "imageCaptureConvert"); - encoder = QGstElement("jpegenc", "jpegEncoder"); - muxer = QGstElement("jifmux", "jpegMuxer"); - sink = QGstElement("fakesink","imageCaptureSink"); // imageCaptureSink do not wait for a preroll buffer when going READY -> PAUSED // as no buffer will arrive until capture() is called sink.set("async", false); - bin.add(queue, videoConvert, encoder, muxer, sink); - queue.link(videoConvert, encoder, muxer, sink); + bin.add(queue, filter, videoConvert, encoder, muxer, sink); + qLinkGstElements(queue, filter, videoConvert, encoder, muxer, sink); bin.addGhostPad(queue, "sink"); addProbeToPad(queue.staticPad("src").pad(), false); sink.set("signal-handoffs", true); - g_signal_connect(sink.object(), "handoff", G_CALLBACK(&QGstreamerImageCapture::saveImageFilter), this); + m_handoffConnection = sink.connect("handoff", G_CALLBACK(&saveImageFilter), this); } QGstreamerImageCapture::~QGstreamerImageCapture() { bin.setStateSync(GST_STATE_NULL); + + // wait for pending futures + auto pendingFutures = [&] { + QMutexLocker guard(&m_mutex); + return std::move(m_pendingFutures); + }(); + + for (QFuture<void> &pendingImage : pendingFutures) + pendingImage.waitForFinished(); } bool QGstreamerImageCapture::isReadyForCapture() const { + QMutexLocker guard(&m_mutex); return m_session && !passImage && cameraActive; } int QGstreamerImageCapture::capture(const QString &fileName) { - QString path = QMediaStorageLocation::generateFileName(fileName, QStandardPaths::PicturesLocation, QLatin1String("jpg")); + using namespace Qt::Literals; + QString path = QMediaStorageLocation::generateFileName( + fileName, QStandardPaths::PicturesLocation, u"jpg"_s); return doCapture(path); } @@ -111,100 +168,153 @@ int QGstreamerImageCapture::captureToBuffer() int QGstreamerImageCapture::doCapture(const QString &fileName) { - qCDebug(qLcImageCapture) << "do capture"; - if (!m_session) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::ResourceError), - Q_ARG(QString, QPlatformImageCapture::msgImageCaptureNotSet())); - - qCDebug(qLcImageCapture) << "error 1"; - return -1; - } - if (!m_session->camera()) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::ResourceError), - Q_ARG(QString,tr("No camera available."))); - - qCDebug(qLcImageCapture) << "error 2"; - return -1; - } - if (passImage) { - //emit error in the next event loop, - //so application can associate it with returned request id. - QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection, - Q_ARG(int, -1), - Q_ARG(int, QImageCapture::NotReadyError), - Q_ARG(QString, QPlatformImageCapture::msgCameraNotReady())); - - qCDebug(qLcImageCapture) << "error 3"; - return -1; - } - m_lastId++; + qCDebug(qLcImageCaptureGst) << "do capture"; + + { + QMutexLocker guard(&m_mutex); + if (!m_session) { + invokeDeferred([this] { + emit error(-1, QImageCapture::ResourceError, + QPlatformImageCapture::msgImageCaptureNotSet()); + }); + + qCDebug(qLcImageCaptureGst) << "error 1"; + return -1; + } + if (!m_session->camera()) { + invokeDeferred([this] { + emit error(-1, QImageCapture::ResourceError, tr("No camera available.")); + }); - pendingImages.enqueue({m_lastId, fileName, QMediaMetaData{}}); - // let one image pass the pipeline - passImage = true; + qCDebug(qLcImageCaptureGst) << "error 2"; + return -1; + } + if (passImage) { + invokeDeferred([this] { + emit error(-1, QImageCapture::NotReadyError, + QPlatformImageCapture::msgCameraNotReady()); + }); + + qCDebug(qLcImageCaptureGst) << "error 3"; + return -1; + } + m_lastId++; + + pendingImages.enqueue({ m_lastId, fileName, QMediaMetaData{} }); + // let one image pass the pipeline + passImage = true; + } emit readyForCaptureChanged(false); return m_lastId; } +void QGstreamerImageCapture::setResolution(const QSize &resolution) +{ + QGstCaps padCaps = bin.staticPad("sink").currentCaps(); + if (padCaps.isNull()) { + qDebug() << "Camera not ready"; + return; + } + QGstCaps caps = padCaps.copy(); + if (caps.isNull()) + return; + + gst_caps_set_simple(caps.caps(), "width", G_TYPE_INT, resolution.width(), "height", G_TYPE_INT, + resolution.height(), nullptr); + filter.set("caps", caps); +} + +// HACK: gcc-10 and earlier reject [=,this] when building with c++17 +#if __cplusplus >= 202002L +# define EQ_THIS_CAPTURE =, this +#else +# define EQ_THIS_CAPTURE = +#endif + bool QGstreamerImageCapture::probeBuffer(GstBuffer *buffer) { + QMutexLocker guard(&m_mutex); + if (!passImage) return false; - qCDebug(qLcImageCapture) << "probe buffer"; + qCDebug(qLcImageCaptureGst) << "probe buffer"; - passImage = false; + QGstBufferHandle bufferHandle{ + buffer, + QGstBufferHandle::NeedsRef, + }; - emit readyForCaptureChanged(isReadyForCapture()); + passImage = false; - QGstCaps caps = gst_pad_get_current_caps(bin.staticPad("sink").pad()); - GstVideoInfo previewInfo; - gst_video_info_from_caps(&previewInfo, caps.get()); + bool ready = isReadyForCapture(); + invokeDeferred([this, ready] { + emit readyForCaptureChanged(ready); + }); + QGstCaps caps = bin.staticPad("sink").currentCaps(); auto memoryFormat = caps.memoryFormat(); - auto fmt = QGstCaps(caps).formatForCaps(&previewInfo); - auto *sink = m_session->gstreamerVideoSink(); - auto *gstBuffer = new QGstVideoBuffer(buffer, previewInfo, sink, fmt, memoryFormat); - QVideoFrame frame(gstBuffer, fmt); - QImage img = frame.toImage(); - if (img.isNull()) { - qDebug() << "received a null image"; - return true; - } - auto &imageData = pendingImages.head(); - - emit imageExposed(imageData.id); - - qCDebug(qLcImageCapture) << "Image available!"; - emit imageAvailable(imageData.id, frame); + GstVideoInfo previewInfo; + QVideoFrameFormat fmt; + auto optionalFormatAndVideoInfo = caps.formatAndVideoInfo(); + if (optionalFormatAndVideoInfo) + std::tie(fmt, previewInfo) = std::move(*optionalFormatAndVideoInfo); + + int futureId = futureIDAllocator += 1; + + // ensure QVideoFrame::toImage is executed on a worker thread that is joined before the + // qApplication is destroyed + QFuture<void> future = s_threadPoolSingleton.run([EQ_THIS_CAPTURE]() mutable { + QMutexLocker guard(&m_mutex); + auto scopeExit = qScopeGuard([&] { + m_pendingFutures.remove(futureId); + }); + + if (!m_session) { + qDebug() << "QGstreamerImageCapture::probeBuffer: no session"; + return; + } - emit imageCaptured(imageData.id, img); + auto *sink = m_session->gstreamerVideoSink(); + auto gstBuffer = std::make_unique<QGstVideoBuffer>(std::move(bufferHandle), previewInfo, + sink, fmt, memoryFormat); - QMediaMetaData metaData = this->metaData(); - metaData.insert(QMediaMetaData::Date, QDateTime::currentDateTime()); - metaData.insert(QMediaMetaData::Resolution, frame.size()); - imageData.metaData = metaData; + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(gstBuffer), fmt); + QImage img = frame.toImage(); + if (img.isNull()) { + qDebug() << "received a null image"; + return; + } - // ensure taginject injects this metaData - const auto &md = static_cast<const QGstreamerMetaData &>(metaData); - md.setMetaData(muxer.element()); + QMediaMetaData imageMetaData = metaData(); + imageMetaData.insert(QMediaMetaData::Resolution, frame.size()); + pendingImages.head().metaData = std::move(imageMetaData); + PendingImage pendingImage = pendingImages.head(); + + invokeDeferred([this, pendingImage = std::move(pendingImage), frame = std::move(frame), + img = std::move(img)]() mutable { + emit imageExposed(pendingImage.id); + qCDebug(qLcImageCaptureGst) << "Image available!"; + emit imageAvailable(pendingImage.id, frame); + emit imageCaptured(pendingImage.id, img); + emit imageMetadataAvailable(pendingImage.id, pendingImage.metaData); + }); + }); + + if (!future.isValid()) // during qApplication shutdown the threadpool becomes unusable + return true; - emit imageMetadataAvailable(imageData.id, metaData); + m_pendingFutures.insert(futureId, future); return true; } +#undef EQ_THIS_CAPTURE + void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session) { + QMutexLocker guard(&m_mutex); QGstreamerMediaCapture *captureSession = static_cast<QGstreamerMediaCapture *>(session); if (m_session == captureSession) return; @@ -225,71 +335,98 @@ void QGstreamerImageCapture::setCaptureSession(QPlatformMediaCaptureSession *ses return; } - connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, &QGstreamerImageCapture::onCameraChanged); + connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, + &QGstreamerImageCapture::onCameraChanged); onCameraChanged(); } +void QGstreamerImageCapture::setMetaData(const QMediaMetaData &m) +{ + { + QMutexLocker guard(&m_mutex); + QPlatformImageCapture::setMetaData(m); + } + + // ensure taginject injects this metaData + applyMetaDataToTagSetter(m, muxer); +} + void QGstreamerImageCapture::cameraActiveChanged(bool active) { - qCDebug(qLcImageCapture) << "cameraActiveChanged" << cameraActive << active; + qCDebug(qLcImageCaptureGst) << "cameraActiveChanged" << cameraActive << active; if (cameraActive == active) return; cameraActive = active; - qCDebug(qLcImageCapture) << "isReady" << isReadyForCapture(); + qCDebug(qLcImageCaptureGst) << "isReady" << isReadyForCapture(); emit readyForCaptureChanged(isReadyForCapture()); } void QGstreamerImageCapture::onCameraChanged() { + QMutexLocker guard(&m_mutex); if (m_session->camera()) { cameraActiveChanged(m_session->camera()->isActive()); - connect(m_session->camera(), &QPlatformCamera::activeChanged, this, &QGstreamerImageCapture::cameraActiveChanged); + connect(m_session->camera(), &QPlatformCamera::activeChanged, this, + &QGstreamerImageCapture::cameraActiveChanged); } else { cameraActiveChanged(false); } } -gboolean QGstreamerImageCapture::saveImageFilter(GstElement *element, - GstBuffer *buffer, - GstPad *pad, - void *appdata) +gboolean QGstreamerImageCapture::saveImageFilter(GstElement *, GstBuffer *buffer, GstPad *, + QGstreamerImageCapture *capture) { - Q_UNUSED(element); - Q_UNUSED(pad); - QGstreamerImageCapture *capture = static_cast<QGstreamerImageCapture *>(appdata); + capture->saveBufferToImage(buffer); + return true; +} - capture->passImage = false; +void QGstreamerImageCapture::saveBufferToImage(GstBuffer *buffer) +{ + QMutexLocker guard(&m_mutex); + passImage = false; - if (capture->pendingImages.isEmpty()) { - return true; - } + if (pendingImages.isEmpty()) + return; - auto imageData = capture->pendingImages.dequeue(); - if (imageData.filename.isEmpty()) { - return true; - } + PendingImage imageData = pendingImages.dequeue(); + if (imageData.filename.isEmpty()) + return; - qCDebug(qLcImageCapture) << "saving image as" << imageData.filename; + int id = futureIDAllocator++; + QGstBufferHandle bufferHandle{ + buffer, + QGstBufferHandle::NeedsRef, + }; + + QFuture<void> saveImageFuture = QtConcurrent::run([this, imageData, bufferHandle, + id]() mutable { + auto cleanup = qScopeGuard([&] { + QMutexLocker guard(&m_mutex); + m_pendingFutures.remove(id); + }); + + qCDebug(qLcImageCaptureGst) << "saving image as" << imageData.filename; + + QFile f(imageData.filename); + if (!f.open(QFile::WriteOnly)) { + qCDebug(qLcImageCaptureGst) << " could not open image file for writing"; + return; + } - QFile f(imageData.filename); - if (f.open(QFile::WriteOnly)) { GstMapInfo info; + GstBuffer *buffer = bufferHandle.get(); if (gst_buffer_map(buffer, &info, GST_MAP_READ)) { f.write(reinterpret_cast<const char *>(info.data), info.size); gst_buffer_unmap(buffer, &info); } f.close(); - static QMetaMethod savedSignal = QMetaMethod::fromSignal(&QGstreamerImageCapture::imageSaved); - savedSignal.invoke(capture, - Qt::QueuedConnection, - Q_ARG(int, imageData.id), - Q_ARG(QString, imageData.filename)); - } else { - qCDebug(qLcImageCapture) << " could not open image file for writing"; - } + QMetaObject::invokeMethod(this, [this, imageData = std::move(imageData)]() mutable { + emit imageSaved(imageData.id, imageData.filename); + }); + }); - return TRUE; + m_pendingFutures.insert(id, saveImageFuture); } QImageEncoderSettings QGstreamerImageCapture::imageSettings() const @@ -300,9 +437,14 @@ QImageEncoderSettings QGstreamerImageCapture::imageSettings() const void QGstreamerImageCapture::setImageSettings(const QImageEncoderSettings &settings) { if (m_settings != settings) { + QSize resolution = settings.resolution(); + if (m_settings.resolution() != resolution && !resolution.isEmpty()) + setResolution(resolution); + m_settings = settings; - // ### } } QT_END_NAMESPACE + +#include "moc_qgstreamerimagecapture_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h index 1f498fe0b..04a7c00b4 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamerimagecapture_p.h @@ -1,42 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERIMAGECAPTURECONTROL_H #define QGSTREAMERIMAGECAPTURECONTROL_H @@ -52,23 +15,26 @@ // We mean it. // -#include <private/qplatformimagecapture_p.h> -#include "qgstreamermediacapture_p.h" -#include "qgstreamerbufferprobe_p.h" +#include <QtMultimedia/private/qplatformimagecapture_p.h> +#include <QtMultimedia/private/qmultimediautils_p.h> -#include <qqueue.h> +#include <QtCore/qmutex.h> +#include <QtCore/qqueue.h> +#include <QtConcurrent/QtConcurrentRun> -#include <qgst_p.h> +#include <common/qgst_p.h> +#include <common/qgstreamerbufferprobe_p.h> +#include <mediacapture/qgstreamermediacapture_p.h> #include <gst/video/video.h> QT_BEGIN_NAMESPACE class QGstreamerImageCapture : public QPlatformImageCapture, private QGstreamerBufferProbe - { Q_OBJECT + public: - QGstreamerImageCapture(QImageCapture *parent); + static QMaybe<QPlatformImageCapture *> create(QImageCapture *parent); virtual ~QGstreamerImageCapture(); bool isReadyForCapture() const override; @@ -82,16 +48,26 @@ public: void setCaptureSession(QPlatformMediaCaptureSession *session); - QGstElement gstElement() const { return bin.element(); } + QGstElement gstElement() const { return bin; } + + void setMetaData(const QMediaMetaData &m) override; public Q_SLOTS: void cameraActiveChanged(bool active); void onCameraChanged(); private: + QGstreamerImageCapture(QImageCapture *parent); + + void setResolution(const QSize &resolution); int doCapture(const QString &fileName); - static gboolean saveImageFilter(GstElement *element, GstBuffer *buffer, GstPad *pad, void *appdata); + static gboolean saveImageFilter(GstElement *element, GstBuffer *buffer, GstPad *pad, + QGstreamerImageCapture *capture); + + void saveBufferToImage(GstBuffer *buffer); + mutable QRecursiveMutex + m_mutex; // guard all elements accessed from probeBuffer/saveBufferToImage QGstreamerMediaCapture *m_session = nullptr; int m_lastId = 0; QImageEncoderSettings m_settings; @@ -106,6 +82,7 @@ private: QGstBin bin; QGstElement queue; + QGstElement filter; QGstElement videoConvert; QGstElement encoder; QGstElement muxer; @@ -114,6 +91,17 @@ private: bool passImage = false; bool cameraActive = false; + + QGObjectHandlerScopedConnection m_handoffConnection; + + QMap<int, QFuture<void>> m_pendingFutures; + int futureIDAllocator = 0; + + template <typename Functor> + void invokeDeferred(Functor &&fn) + { + QMetaObject::invokeMethod(this, std::forward<decltype(fn)>(fn), Qt::QueuedConnection); + }; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp index 15cd076e9..d807ab804 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture.cpp @@ -1,58 +1,19 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamermediacapture_p.h" -#include "qgstreamermediaencoder_p.h" -#include "qgstreamerimagecapture_p.h" -#include "qgstreamercamera_p.h" -#include <qgstpipeline_p.h> - -#include "qgstreameraudioinput_p.h" -#include "qgstreameraudiooutput_p.h" -#include "qgstreamervideooutput_p.h" - -#include <qloggingcategory.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -QT_BEGIN_NAMESPACE +#include <mediacapture/qgstreamermediacapture_p.h> +#include <mediacapture/qgstreamermediaencoder_p.h> +#include <mediacapture/qgstreamerimagecapture_p.h> +#include <mediacapture/qgstreamercamera_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreameraudioinput_p.h> +#include <common/qgstreameraudiooutput_p.h> +#include <common/qgstreamervideooutput_p.h> -Q_LOGGING_CATEGORY(qLcMediaCapture, "qt.multimedia.capture") +#include <QtCore/qloggingcategory.h> +#include <QtCore/private/quniquehandle_p.h> +QT_BEGIN_NAMESPACE static void linkTeeToPad(QGstElement tee, QGstPad sink) { @@ -63,37 +24,42 @@ static void linkTeeToPad(QGstElement tee, QGstPad sink) source.link(sink); } -static void unlinkTeeFromPad(QGstElement tee, QGstPad sink) +QMaybe<QPlatformMediaCaptureSession *> QGstreamerMediaCapture::create() { - if (tee.isNull() || sink.isNull()) - return; + auto videoOutput = QGstreamerVideoOutput::create(); + if (!videoOutput) + return videoOutput.error(); - auto source = sink.peer(); - source.unlink(sink); + static const auto error = qGstErrorMessageIfElementsNotAvailable("tee", "capsfilter"); + if (error) + return *error; - tee.releaseRequestPad(source); + return new QGstreamerMediaCapture(videoOutput.value()); } - -QGstreamerMediaCapture::QGstreamerMediaCapture() - : gstPipeline("pipeline") +QGstreamerMediaCapture::QGstreamerMediaCapture(QGstreamerVideoOutput *videoOutput) + : capturePipeline(QGstPipeline::create("mediaCapturePipeline")), gstVideoOutput(videoOutput) { - gstVideoOutput = new QGstreamerVideoOutput(this); + gstVideoOutput->setParent(this); gstVideoOutput->setIsPreview(); - gstVideoOutput->setPipeline(gstPipeline); + gstVideoOutput->setPipeline(capturePipeline); // Use system clock to drive all elements in the pipeline. Otherwise, // the clock is sourced from the elements (e.g. from an audio source). // Since the elements are added and removed dynamically the clock would // also change causing lost of synchronization in the pipeline. - gst_pipeline_use_clock(gstPipeline.pipeline(), gst_system_clock_obtain()); + + QGstClockHandle systemClock{ + gst_system_clock_obtain(), + }; + gst_pipeline_use_clock(capturePipeline.pipeline(), systemClock.get()); // This is the recording pipeline with only live sources, thus the pipeline // will be always in the playing state. - gstPipeline.setState(GST_STATE_PLAYING); - gstPipeline.setInStoppedState(false); + capturePipeline.setState(GST_STATE_PLAYING); + gstVideoOutput->setActive(true); - gstPipeline.dumpGraph("initial"); + capturePipeline.dumpGraph("initial"); } QGstreamerMediaCapture::~QGstreamerMediaCapture() @@ -101,7 +67,7 @@ QGstreamerMediaCapture::~QGstreamerMediaCapture() setMediaRecorder(nullptr); setImageCapture(nullptr); setCamera(nullptr); - gstPipeline.setStateSync(GST_STATE_NULL); + capturePipeline.setStateSync(GST_STATE_NULL); } QPlatformCamera *QGstreamerMediaCapture::camera() @@ -109,52 +75,64 @@ QPlatformCamera *QGstreamerMediaCapture::camera() return gstCamera; } -void QGstreamerMediaCapture::setCamera(QPlatformCamera *camera) +void QGstreamerMediaCapture::setCamera(QPlatformCamera *platformCamera) { - QGstreamerCamera *control = static_cast<QGstreamerCamera *>(camera); - if (gstCamera == control) + auto *camera = static_cast<QGstreamerCameraBase *>(platformCamera); + if (gstCamera == camera) return; if (gstCamera) { - unlinkTeeFromPad(gstVideoTee, encoderVideoSink); - unlinkTeeFromPad(gstVideoTee, imageCaptureSink); + QObject::disconnect(gstCameraActiveConnection); + if (gstVideoTee) + setCameraActive(false); + } - auto camera = gstCamera->gstElement(); + gstCamera = camera; - gstPipeline.remove(camera); - gstPipeline.remove(gstVideoTee); - gstPipeline.remove(gstVideoOutput->gstElement()); + if (gstCamera) { + gstCameraActiveConnection = QObject::connect(camera, &QPlatformCamera::activeChanged, this, + &QGstreamerMediaCapture::setCameraActive); + if (gstCamera->isActive()) + setCameraActive(true); + } - camera.setStateSync(GST_STATE_NULL); - gstVideoTee.setStateSync(GST_STATE_NULL); - gstVideoOutput->gstElement().setStateSync(GST_STATE_NULL); + emit cameraChanged(); +} - gstVideoTee = {}; - gstCamera->setCaptureSession(nullptr); - } +void QGstreamerMediaCapture::setCameraActive(bool activate) +{ + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (activate) { + QGstElement cameraElement = gstCamera->gstElement(); + gstVideoTee = QGstElement::createFromFactory("tee", "videotee"); + gstVideoTee.set("allow-not-linked", true); - gstCamera = control; - if (gstCamera) { - QGstElement camera = gstCamera->gstElement(); - gstVideoTee = QGstElement("tee", "videotee"); - gstVideoTee.set("allow-not-linked", true); + capturePipeline.add(gstVideoOutput->gstElement(), cameraElement, gstVideoTee); - gstPipeline.add(gstVideoOutput->gstElement(), camera, gstVideoTee); + linkTeeToPad(gstVideoTee, encoderVideoSink); + linkTeeToPad(gstVideoTee, gstVideoOutput->gstElement().staticPad("sink")); + linkTeeToPad(gstVideoTee, imageCaptureSink); - linkTeeToPad(gstVideoTee, encoderVideoSink); - linkTeeToPad(gstVideoTee, gstVideoOutput->gstElement().staticPad("sink")); - linkTeeToPad(gstVideoTee, imageCaptureSink); + qLinkGstElements(cameraElement, gstVideoTee); - camera.link(gstVideoTee); + capturePipeline.syncChildrenState(); + } else { + if (encoderVideoCapsFilter) + qUnlinkGstElements(gstVideoTee, encoderVideoCapsFilter); + if (m_imageCapture) + qUnlinkGstElements(gstVideoTee, m_imageCapture->gstElement()); - gstVideoOutput->gstElement().setState(GST_STATE_PLAYING); - gstVideoTee.setState(GST_STATE_PLAYING); - camera.setState(GST_STATE_PLAYING); - } + auto camera = gstCamera->gstElement(); - gstPipeline.dumpGraph("camera"); + capturePipeline.stopAndRemoveElements(camera, gstVideoTee, + gstVideoOutput->gstElement()); - emit cameraChanged(); + gstVideoTee = {}; + gstCamera->setCaptureSession(nullptr); + } + }); + + capturePipeline.dumpGraph("camera"); } QPlatformImageCapture *QGstreamerMediaCapture::imageCapture() @@ -168,24 +146,25 @@ void QGstreamerMediaCapture::setImageCapture(QPlatformImageCapture *imageCapture if (m_imageCapture == control) return; - if (m_imageCapture) { - unlinkTeeFromPad(gstVideoTee, imageCaptureSink); - gstPipeline.remove(m_imageCapture->gstElement()); - m_imageCapture->gstElement().setStateSync(GST_STATE_NULL); - imageCaptureSink = {}; - m_imageCapture->setCaptureSession(nullptr); - } + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (m_imageCapture) { + qUnlinkGstElements(gstVideoTee, m_imageCapture->gstElement()); + capturePipeline.stopAndRemoveElements(m_imageCapture->gstElement()); + imageCaptureSink = {}; + m_imageCapture->setCaptureSession(nullptr); + } - m_imageCapture = control; - if (m_imageCapture) { - imageCaptureSink = m_imageCapture->gstElement().staticPad("sink"); - m_imageCapture->gstElement().setState(GST_STATE_PLAYING); - gstPipeline.add(m_imageCapture->gstElement()); - linkTeeToPad(gstVideoTee, imageCaptureSink); - m_imageCapture->setCaptureSession(this); - } + m_imageCapture = control; + if (m_imageCapture) { + imageCaptureSink = m_imageCapture->gstElement().staticPad("sink"); + capturePipeline.add(m_imageCapture->gstElement()); + m_imageCapture->gstElement().syncStateWithParent(); + linkTeeToPad(gstVideoTee, imageCaptureSink); + m_imageCapture->setCaptureSession(this); + } + }); - gstPipeline.dumpGraph("imageCapture"); + capturePipeline.dumpGraph("imageCapture"); emit imageCaptureChanged(); } @@ -203,7 +182,7 @@ void QGstreamerMediaCapture::setMediaRecorder(QPlatformMediaRecorder *recorder) m_mediaEncoder->setCaptureSession(this); emit encoderChanged(); - gstPipeline.dumpGraph("encoder"); + capturePipeline.dumpGraph("encoder"); } QPlatformMediaRecorder *QGstreamerMediaCapture::mediaRecorder() @@ -213,55 +192,62 @@ QPlatformMediaRecorder *QGstreamerMediaCapture::mediaRecorder() void QGstreamerMediaCapture::linkEncoder(QGstPad audioSink, QGstPad videoSink) { - if (!gstVideoTee.isNull() && !videoSink.isNull()) { - auto caps = gst_pad_get_current_caps(gstVideoTee.sink().pad()); + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (!gstVideoTee.isNull() && !videoSink.isNull()) { + QGstCaps caps = gstVideoTee.sink().currentCaps(); - encoderVideoCapsFilter = QGstElement("capsfilter", "encoderVideoCapsFilter"); - encoderVideoCapsFilter.set("caps", QGstMutableCaps(caps)); + encoderVideoCapsFilter = + QGstElement::createFromFactory("capsfilter", "encoderVideoCapsFilter"); + Q_ASSERT(encoderVideoCapsFilter); + encoderVideoCapsFilter.set("caps", caps); - gstPipeline.add(encoderVideoCapsFilter); + capturePipeline.add(encoderVideoCapsFilter); - encoderVideoCapsFilter.src().link(videoSink); - linkTeeToPad(gstVideoTee, encoderVideoCapsFilter.sink()); - encoderVideoCapsFilter.setState(GST_STATE_PLAYING); - encoderVideoSink = encoderVideoCapsFilter.sink(); - } + encoderVideoCapsFilter.src().link(videoSink); + linkTeeToPad(gstVideoTee, encoderVideoCapsFilter.sink()); + encoderVideoSink = encoderVideoCapsFilter.sink(); + } - if (!gstAudioTee.isNull() && !audioSink.isNull()) { - auto caps = gst_pad_get_current_caps(gstAudioTee.sink().pad()); + if (!gstAudioTee.isNull() && !audioSink.isNull()) { + QGstCaps caps = gstAudioTee.sink().currentCaps(); - encoderAudioCapsFilter = QGstElement("capsfilter", "encoderAudioCapsFilter"); - encoderAudioCapsFilter.set("caps", QGstMutableCaps(caps)); + encoderAudioCapsFilter = + QGstElement::createFromFactory("capsfilter", "encoderAudioCapsFilter"); + Q_ASSERT(encoderAudioCapsFilter); + encoderAudioCapsFilter.set("caps", caps); - gstPipeline.add(encoderAudioCapsFilter); + capturePipeline.add(encoderAudioCapsFilter); - encoderAudioCapsFilter.src().link(audioSink); - linkTeeToPad(gstAudioTee, encoderAudioCapsFilter.sink()); - encoderAudioCapsFilter.setState(GST_STATE_PLAYING); - encoderAudioSink = encoderAudioCapsFilter.sink(); - } + encoderAudioCapsFilter.src().link(audioSink); + linkTeeToPad(gstAudioTee, encoderAudioCapsFilter.sink()); + encoderAudioSink = encoderAudioCapsFilter.sink(); + } + }); } void QGstreamerMediaCapture::unlinkEncoder() { - if (!encoderVideoCapsFilter.isNull()) { - encoderVideoCapsFilter.src().unlinkPeer(); - unlinkTeeFromPad(gstVideoTee, encoderVideoCapsFilter.sink()); - gstPipeline.remove(encoderVideoCapsFilter); - encoderVideoCapsFilter.setStateSync(GST_STATE_NULL); - encoderVideoCapsFilter = {}; - } + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (encoderVideoCapsFilter) { + qUnlinkGstElements(gstVideoTee, encoderVideoCapsFilter); + capturePipeline.stopAndRemoveElements(encoderVideoCapsFilter); + encoderVideoCapsFilter = {}; + } - if (!encoderAudioCapsFilter.isNull()) { - encoderAudioCapsFilter.src().unlinkPeer(); - unlinkTeeFromPad(gstAudioTee, encoderAudioCapsFilter.sink()); - gstPipeline.remove(encoderAudioCapsFilter); - encoderAudioCapsFilter.setStateSync(GST_STATE_NULL); - encoderAudioCapsFilter = {}; - } + if (encoderAudioCapsFilter) { + qUnlinkGstElements(gstAudioTee, encoderAudioCapsFilter); + capturePipeline.stopAndRemoveElements(encoderAudioCapsFilter); + encoderAudioCapsFilter = {}; + } + + encoderAudioSink = {}; + encoderVideoSink = {}; + }); +} - encoderAudioSink = {}; - encoderVideoSink = {}; +const QGstPipeline &QGstreamerMediaCapture::pipeline() const +{ + return capturePipeline; } void QGstreamerMediaCapture::setAudioInput(QPlatformAudioInput *input) @@ -269,41 +255,39 @@ void QGstreamerMediaCapture::setAudioInput(QPlatformAudioInput *input) if (gstAudioInput == input) return; - if (gstAudioInput) { - unlinkTeeFromPad(gstAudioTee, encoderAudioSink); + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (gstAudioInput) { + if (encoderAudioCapsFilter) + qUnlinkGstElements(gstAudioTee, encoderAudioCapsFilter); + + if (gstAudioOutput) { + qUnlinkGstElements(gstAudioTee, gstAudioOutput->gstElement()); + capturePipeline.stopAndRemoveElements(gstAudioOutput->gstElement()); + } - if (gstAudioOutput) { - unlinkTeeFromPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - gstPipeline.remove(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setStateSync(GST_STATE_NULL); + capturePipeline.stopAndRemoveElements(gstAudioInput->gstElement(), gstAudioTee); + gstAudioTee = {}; } - gstPipeline.remove(gstAudioInput->gstElement()); - gstPipeline.remove(gstAudioTee); - gstAudioInput->gstElement().setStateSync(GST_STATE_NULL); - gstAudioTee.setStateSync(GST_STATE_NULL); - gstAudioTee = {}; - } + gstAudioInput = static_cast<QGstreamerAudioInput *>(input); + if (gstAudioInput) { + Q_ASSERT(gstAudioTee.isNull()); + gstAudioTee = QGstElement::createFromFactory("tee", "audiotee"); + gstAudioTee.set("allow-not-linked", true); + capturePipeline.add(gstAudioInput->gstElement(), gstAudioTee); + qLinkGstElements(gstAudioInput->gstElement(), gstAudioTee); - gstAudioInput = static_cast<QGstreamerAudioInput *>(input); - if (gstAudioInput) { - Q_ASSERT(gstAudioTee.isNull()); - gstAudioTee = QGstElement("tee", "audiotee"); - gstAudioTee.set("allow-not-linked", true); - gstPipeline.add(gstAudioInput->gstElement(), gstAudioTee); - gstAudioInput->gstElement().link(gstAudioTee); - - if (gstAudioOutput) { - gstPipeline.add(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setState(GST_STATE_PLAYING); - linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - } + if (gstAudioOutput) { + capturePipeline.add(gstAudioOutput->gstElement()); + gstAudioOutput->gstElement().setState(GST_STATE_PLAYING); + linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); + } - gstAudioTee.setState(GST_STATE_PLAYING); - gstAudioInput->gstElement().setStateSync(GST_STATE_PLAYING); + capturePipeline.syncChildrenState(); - linkTeeToPad(gstAudioTee, encoderAudioSink); - } + linkTeeToPad(gstAudioTee, encoderAudioSink); + } + }); } void QGstreamerMediaCapture::setVideoPreview(QVideoSink *sink) @@ -316,19 +300,20 @@ void QGstreamerMediaCapture::setAudioOutput(QPlatformAudioOutput *output) if (gstAudioOutput == output) return; - if (gstAudioOutput && gstAudioInput) { - // If audio input is set, the output is in the pipeline - unlinkTeeFromPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - gstPipeline.remove(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setStateSync(GST_STATE_NULL); - } + capturePipeline.modifyPipelineWhileNotRunning([&] { + if (gstAudioOutput && gstAudioInput) { + // If audio input is set, the output is in the pipeline + qUnlinkGstElements(gstAudioTee, gstAudioOutput->gstElement()); + capturePipeline.stopAndRemoveElements(gstAudioOutput->gstElement()); + } - gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); - if (gstAudioOutput && gstAudioInput) { - gstPipeline.add(gstAudioOutput->gstElement()); - gstAudioOutput->gstElement().setState(GST_STATE_PLAYING); - linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); - } + gstAudioOutput = static_cast<QGstreamerAudioOutput *>(output); + if (gstAudioOutput && gstAudioInput) { + capturePipeline.add(gstAudioOutput->gstElement()); + capturePipeline.syncChildrenState(); + linkTeeToPad(gstAudioTee, gstAudioOutput->gstElement().staticPad("sink")); + } + }); } QGstreamerVideoSink *QGstreamerMediaCapture::gstreamerVideoSink() const @@ -336,5 +321,6 @@ QGstreamerVideoSink *QGstreamerMediaCapture::gstreamerVideoSink() const return gstVideoOutput ? gstVideoOutput->gstreamerVideoSink() : nullptr; } - QT_END_NAMESPACE + +#include "moc_qgstreamermediacapture_p.cpp" diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h index 8a7b03f02..c44e31f0e 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediacapture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERCAPTURESERVICE_H #define QGSTREAMERCAPTURESERVICE_H @@ -54,14 +18,14 @@ #include <private/qplatformmediacapture_p.h> #include <private/qplatformmediaintegration_p.h> -#include <qgst_p.h> -#include <qgstpipeline_p.h> +#include <common/qgst_p.h> +#include <common/qgstpipeline_p.h> #include <qtimer.h> QT_BEGIN_NAMESPACE -class QGstreamerCamera; +class QGstreamerCameraBase; class QGstreamerImageCapture; class QGstreamerMediaEncoder; class QGstreamerAudioInput; @@ -69,12 +33,12 @@ class QGstreamerAudioOutput; class QGstreamerVideoOutput; class QGstreamerVideoSink; -class QGstreamerMediaCapture : public QPlatformMediaCaptureSession +class QGstreamerMediaCapture final : public QPlatformMediaCaptureSession { Q_OBJECT public: - QGstreamerMediaCapture(); + static QMaybe<QPlatformMediaCaptureSession *> create(); virtual ~QGstreamerMediaCapture(); QPlatformCamera *camera() override; @@ -95,17 +59,22 @@ public: void linkEncoder(QGstPad audioSink, QGstPad videoSink); void unlinkEncoder(); - QGstPipeline pipeline() const { return gstPipeline; } + const QGstPipeline &pipeline() const; QGstreamerVideoSink *gstreamerVideoSink() const; private: + void setCameraActive(bool activate); + + explicit QGstreamerMediaCapture(QGstreamerVideoOutput *videoOutput); + friend QGstreamerMediaEncoder; // Gst elements - QGstPipeline gstPipeline; + QGstPipeline capturePipeline; QGstreamerAudioInput *gstAudioInput = nullptr; - QGstreamerCamera *gstCamera = nullptr; + QGstreamerCameraBase *gstCamera = nullptr; + QMetaObject::Connection gstCameraActiveConnection; QGstElement gstAudioTee; QGstElement gstVideoTee; diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp index 387ba9a71..4ec10ca84 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder.cpp @@ -1,63 +1,31 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamermediaencoder_p.h" -#include "qgstreamerintegration_p.h" -#include "qgstreamerformatinfo_p.h" -#include "qgstpipeline_p.h" -#include "qgstreamermessage_p.h" -#include <private/qplatformcamera_p.h> -#include "qaudiodevice.h" -#include <private/qmediastoragelocation_p.h> - -#include <qdebug.h> -#include <qeventloop.h> -#include <qstandardpaths.h> -#include <qmimetype.h> -#include <qloggingcategory.h> +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <mediacapture/qgstreamermediaencoder_p.h> +#include <qgstreamerformatinfo_p.h> +#include <common/qgstpipeline_p.h> +#include <common/qgstreamermessage_p.h> +#include <common/qgst_debug_p.h> +#include <qgstreamerintegration_p.h> + +#include <QtMultimedia/private/qmediastoragelocation_p.h> +#include <QtMultimedia/private/qplatformcamera_p.h> +#include <QtMultimedia/qaudiodevice.h> + +#include <QtCore/qdebug.h> +#include <QtCore/qeventloop.h> +#include <QtCore/qstandardpaths.h> +#include <QtCore/qmimetype.h> +#include <QtCore/qloggingcategory.h> #include <gst/gsttagsetter.h> #include <gst/gstversion.h> #include <gst/video/video.h> #include <gst/pbutils/encoding-profile.h> -Q_LOGGING_CATEGORY(qLcMediaEncoder, "qt.multimedia.encoder") +static Q_LOGGING_CATEGORY(qLcMediaEncoderGst, "qt.multimedia.encoder") + +QT_BEGIN_NAMESPACE QGstreamerMediaEncoder::QGstreamerMediaEncoder(QMediaRecorder *parent) : QPlatformMediaRecorder(parent), @@ -65,15 +33,17 @@ QGstreamerMediaEncoder::QGstreamerMediaEncoder(QMediaRecorder *parent) videoPauseControl(*this) { signalDurationChangedTimer.setInterval(100); - signalDurationChangedTimer.callOnTimeout([this](){ durationChanged(duration()); }); + signalDurationChangedTimer.callOnTimeout(&signalDurationChangedTimer, [this]() { + durationChanged(duration()); + }); } QGstreamerMediaEncoder::~QGstreamerMediaEncoder() { - if (!gstPipeline.isNull()) { + if (!capturePipeline.isNull()) { finalize(); - gstPipeline.removeMessageFilter(this); - gstPipeline.setStateSync(GST_STATE_NULL); + capturePipeline.removeMessageFilter(this); + capturePipeline.setStateSync(GST_STATE_NULL); } } @@ -84,52 +54,60 @@ bool QGstreamerMediaEncoder::isLocationWritable(const QUrl &) const void QGstreamerMediaEncoder::handleSessionError(QMediaRecorder::Error code, const QString &description) { - error(code, description); + updateError(code, description); stop(); } -bool QGstreamerMediaEncoder::processBusMessage(const QGstreamerMessage &message) +bool QGstreamerMediaEncoder::processBusMessage(const QGstreamerMessage &msg) { - if (message.isNull()) - return false; - auto msg = message; - -// qCDebug(qLcMediaEncoder) << "received event from" << message.source().name() << Qt::hex << message.type(); -// if (message.type() == GST_MESSAGE_STATE_CHANGED) { -// GstState oldState; -// GstState newState; -// GstState pending; -// gst_message_parse_state_changed(gm, &oldState, &newState, &pending); -// qCDebug(qLcMediaEncoder) << "received state change from" << message.source().name() << oldState << newState << pending; -// } - if (msg.type() == GST_MESSAGE_ELEMENT) { - QGstStructure s = msg.structure(); - qCDebug(qLcMediaEncoder) << "received element message from" << msg.source().name() << s.name(); + constexpr bool traceStateChange = false; + constexpr bool traceAllEvents = false; + + if constexpr (traceAllEvents) + qCDebug(qLcMediaEncoderGst) << "received event:" << msg; + + switch (msg.type()) { + case GST_MESSAGE_ELEMENT: { + QGstStructureView s = msg.structure(); if (s.name() == "GstBinForwarded") - msg = QGstreamerMessage(s); - if (msg.isNull()) - return false; + return processBusMessage(s.getMessage()); + + qCDebug(qLcMediaEncoderGst) + << "received element message from" << msg.source().name() << s.name(); + return false; } - if (msg.type() == GST_MESSAGE_EOS) { - qCDebug(qLcMediaEncoder) << "received EOS from" << msg.source().name(); + case GST_MESSAGE_EOS: { + qCDebug(qLcMediaEncoderGst) << "received EOS from" << msg.source().name(); finalize(); return false; } - if (msg.type() == GST_MESSAGE_ERROR) { - GError *err; - gchar *debug; - gst_message_parse_error(msg.rawMessage(), &err, &debug); - error(QMediaRecorder::ResourceError, QString::fromUtf8(err->message)); - g_error_free(err); - g_free(debug); + case GST_MESSAGE_ERROR: { + qCDebug(qLcMediaEncoderGst) + << "received error:" << msg.source().name() << QCompactGstMessageAdaptor(msg); + + QUniqueGErrorHandle err; + QGString debug; + gst_message_parse_error(msg.message(), &err, &debug); + updateError(QMediaRecorder::ResourceError, QString::fromUtf8(err.get()->message)); if (!m_finalizing) stop(); finalize(); + return false; } - return false; + case GST_MESSAGE_STATE_CHANGED: { + if constexpr (traceStateChange) + qCDebug(qLcMediaEncoderGst) + << "received state change" << QCompactGstMessageAdaptor(msg); + + return false; + } + + default: + return false; + }; } qint64 QGstreamerMediaEncoder::duration() const @@ -142,13 +120,13 @@ static GstEncodingContainerProfile *createContainerProfile(const QMediaEncoderSe { auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo(); - QGstMutableCaps caps = formatInfo->formatCaps(settings.fileFormat()); + auto caps = formatInfo->formatCaps(settings.fileFormat()); - GstEncodingContainerProfile *profile = (GstEncodingContainerProfile *)gst_encoding_container_profile_new( - "container_profile", - (gchar *)"custom container profile", - const_cast<GstCaps *>(caps.get()), - nullptr); //preset + GstEncodingContainerProfile *profile = + (GstEncodingContainerProfile *)gst_encoding_container_profile_new( + "container_profile", (gchar *)"custom container profile", + const_cast<GstCaps *>(caps.caps()), + nullptr); // preset return profile; } @@ -156,15 +134,18 @@ static GstEncodingProfile *createVideoProfile(const QMediaEncoderSettings &setti { auto *formatInfo = QGstreamerIntegration::instance()->gstFormatsInfo(); - QGstMutableCaps caps = formatInfo->videoCaps(settings.mediaFormat()); + QGstCaps caps = formatInfo->videoCaps(settings.mediaFormat()); if (caps.isNull()) return nullptr; - GstEncodingVideoProfile *profile = gst_encoding_video_profile_new( - const_cast<GstCaps *>(caps.get()), - nullptr, - nullptr, //restriction - 0); //presence + QSize videoResolution = settings.videoResolution(); + if (videoResolution.isValid()) + caps.setResolution(videoResolution); + + GstEncodingVideoProfile *profile = + gst_encoding_video_profile_new(const_cast<GstCaps *>(caps.caps()), nullptr, + nullptr, // restriction + 0); // presence gst_encoding_video_profile_set_pass(profile, 0); gst_encoding_video_profile_set_variableframerate(profile, TRUE); @@ -180,11 +161,11 @@ static GstEncodingProfile *createAudioProfile(const QMediaEncoderSettings &setti if (caps.isNull()) return nullptr; - GstEncodingProfile *profile = (GstEncodingProfile *)gst_encoding_audio_profile_new( - const_cast<GstCaps *>(caps.get()), - nullptr, //preset - nullptr, //restriction - 0); //presence + GstEncodingProfile *profile = + (GstEncodingProfile *)gst_encoding_audio_profile_new(const_cast<GstCaps *>(caps.caps()), + nullptr, // preset + nullptr, // restriction + 0); // presence return profile; } @@ -281,7 +262,7 @@ void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings) const auto hasAudio = m_session->audioInput() != nullptr; if (!hasVideo && !hasAudio) { - error(QMediaRecorder::ResourceError, QMediaRecorder::tr("No camera or audio input")); + updateError(QMediaRecorder::ResourceError, QMediaRecorder::tr("No camera or audio input")); return; } @@ -292,16 +273,18 @@ void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings) auto location = QMediaStorageLocation::generateFileName(outputLocation().toLocalFile(), primaryLocation, container); QUrl actualSink = QUrl::fromLocalFile(QDir::currentPath()).resolved(location); - qCDebug(qLcMediaEncoder) << "recording new video to" << actualSink; + qCDebug(qLcMediaEncoderGst) << "recording new video to" << actualSink; Q_ASSERT(!actualSink.isEmpty()); - gstEncoder = QGstElement("encodebin", "encodebin"); + gstEncoder = QGstBin::createFromFactory("encodebin", "encodebin"); + Q_ASSERT(gstEncoder); auto *encodingProfile = createEncodingProfile(settings); g_object_set (gstEncoder.object(), "profile", encodingProfile, nullptr); gst_encoding_profile_unref(encodingProfile); - gstFileSink = QGstElement("filesink", "filesink"); + gstFileSink = QGstElement::createFromFactory("filesink", "filesink"); + Q_ASSERT(gstFileSink); gstFileSink.set("location", QFile::encodeName(actualSink.toLocalFile()).constData()); gstFileSink.set("async", false); @@ -327,17 +310,19 @@ void QGstreamerMediaEncoder::record(QMediaEncoderSettings &settings) videoPauseControl.installOn(videoSink); } - gstPipeline.add(gstEncoder, gstFileSink); - gstEncoder.link(gstFileSink); - m_metaData.setMetaData(gstEncoder.bin()); + capturePipeline.modifyPipelineWhileNotRunning([&] { + capturePipeline.add(gstEncoder, gstFileSink); + qLinkGstElements(gstEncoder, gstFileSink); + applyMetaDataToTagSetter(m_metaData, gstEncoder); - m_session->linkEncoder(audioSink, videoSink); + m_session->linkEncoder(audioSink, videoSink); - gstEncoder.syncStateWithParent(); - gstFileSink.syncStateWithParent(); + gstEncoder.syncStateWithParent(); + gstFileSink.syncStateWithParent(); + }); signalDurationChangedTimer.start(); - gstPipeline.dumpGraph("recording"); + capturePipeline.dumpGraph("recording"); durationChanged(0); stateChanged(QMediaRecorder::RecordingState); @@ -349,13 +334,14 @@ void QGstreamerMediaEncoder::pause() if (!m_session || m_finalizing || state() != QMediaRecorder::RecordingState) return; signalDurationChangedTimer.stop(); - gstPipeline.dumpGraph("before-pause"); + durationChanged(duration()); + capturePipeline.dumpGraph("before-pause"); stateChanged(QMediaRecorder::PausedState); } void QGstreamerMediaEncoder::resume() { - gstPipeline.dumpGraph("before-resume"); + capturePipeline.dumpGraph("before-resume"); if (!m_session || m_finalizing || state() != QMediaRecorder::PausedState) return; signalDurationChangedTimer.start(); @@ -366,12 +352,13 @@ void QGstreamerMediaEncoder::stop() { if (!m_session || m_finalizing || state() == QMediaRecorder::StoppedState) return; - qCDebug(qLcMediaEncoder) << "stop"; + durationChanged(duration()); + qCDebug(qLcMediaEncoderGst) << "stop"; m_finalizing = true; m_session->unlinkEncoder(); signalDurationChangedTimer.stop(); - qCDebug(qLcMediaEncoder) << ">>>>>>>>>>>>> sending EOS"; + qCDebug(qLcMediaEncoderGst) << ">>>>>>>>>>>>> sending EOS"; gstEncoder.sendEos(); } @@ -380,12 +367,9 @@ void QGstreamerMediaEncoder::finalize() if (!m_session || gstEncoder.isNull()) return; - qCDebug(qLcMediaEncoder) << "finalize"; + qCDebug(qLcMediaEncoderGst) << "finalize"; - gstPipeline.remove(gstEncoder); - gstPipeline.remove(gstFileSink); - gstEncoder.setStateSync(GST_STATE_NULL); - gstFileSink.setStateSync(GST_STATE_NULL); + capturePipeline.stopAndRemoveElements(gstEncoder, gstFileSink); gstFileSink = {}; gstEncoder = {}; m_finalizing = false; @@ -396,7 +380,7 @@ void QGstreamerMediaEncoder::setMetaData(const QMediaMetaData &metaData) { if (!m_session) return; - m_metaData = static_cast<const QGstreamerMetaData &>(metaData); + m_metaData = metaData; } QMediaMetaData QGstreamerMediaEncoder::metaData() const @@ -414,19 +398,22 @@ void QGstreamerMediaEncoder::setCaptureSession(QPlatformMediaCaptureSession *ses stop(); if (m_finalizing) { QEventLoop loop; - loop.connect(mediaRecorder(), SIGNAL(recorderStateChanged(RecorderState)), SLOT(quit())); + QObject::connect(mediaRecorder(), &QMediaRecorder::recorderStateChanged, &loop, + &QEventLoop::quit); loop.exec(); } - gstPipeline.removeMessageFilter(this); - gstPipeline = {}; + capturePipeline.removeMessageFilter(this); + capturePipeline = {}; } m_session = captureSession; if (!m_session) return; - gstPipeline = captureSession->gstPipeline; - gstPipeline.set("message-forward", true); - gstPipeline.installMessageFilter(this); + capturePipeline = captureSession->capturePipeline; + capturePipeline.set("message-forward", true); + capturePipeline.installMessageFilter(this); } + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h index 554b79207..56e8c193b 100644 --- a/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h +++ b/src/plugins/multimedia/gstreamer/mediacapture/qgstreamermediaencoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERENCODERCONTROL_H @@ -52,14 +16,14 @@ // We mean it. // -#include <private/qplatformmediarecorder_p.h> -#include "qgstreamermediacapture_p.h" -#include "qgstreamermetadata_p.h" +#include <mediacapture/qgstreamermediacapture_p.h> +#include <common/qgstreamermetadata_p.h> +#include <QtMultimedia/private/qplatformmediarecorder_p.h> #include <QtCore/qurl.h> #include <QtCore/qdir.h> -#include <qelapsedtimer.h> -#include <qtimer.h> +#include <QtCore/qelapsedtimer.h> +#include <QtCore/qtimer.h> QT_BEGIN_NAMESPACE @@ -69,7 +33,7 @@ class QGstreamerMessage; class QGstreamerMediaEncoder : public QPlatformMediaRecorder, QGstreamerBusMessageFilter { public: - QGstreamerMediaEncoder(QMediaRecorder *parent); + explicit QGstreamerMediaEncoder(QMediaRecorder *parent); virtual ~QGstreamerMediaEncoder(); bool isLocationWritable(const QUrl &sink) const override; @@ -92,7 +56,7 @@ private: private: struct PauseControl { - PauseControl(QPlatformMediaRecorder &encoder) : encoder(encoder) {} + explicit PauseControl(QPlatformMediaRecorder &encoder) : encoder(encoder) { } GstPadProbeReturn processBuffer(QGstPad pad, GstPadProbeInfo *info); void installOn(QGstPad pad); @@ -112,10 +76,10 @@ private: void finalize(); QGstreamerMediaCapture *m_session = nullptr; - QGstreamerMetaData m_metaData; + QMediaMetaData m_metaData; QTimer signalDurationChangedTimer; - QGstPipeline gstPipeline; + QGstPipeline capturePipeline; QGstBin gstEncoder; QGstElement gstFileSink; diff --git a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp index 8c17ed7e7..a657fc52f 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp +++ b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo.cpp @@ -1,56 +1,22 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#include <common/qglist_helper_p.h> #include "qgstreamerformatinfo_p.h" -#include "qgstutils_p.h" +#include <gst/gst.h> QT_BEGIN_NAMESPACE -QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructure structure) +QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!name || strncmp(name, "audio/", 6)) + if (!name || (strncmp(name, "audio/", 6) != 0)) return QMediaFormat::AudioCodec::Unspecified; name += 6; - if (!strcmp(name, "mpeg")) { + if (name == "mpeg"sv) { auto version = structure["mpegversion"].toInt(); if (version == 1) { auto layer = structure["layer"]; @@ -59,91 +25,120 @@ QMediaFormat::AudioCodec QGstreamerFormatInfo::audioCodecForCaps(QGstStructure s } if (version == 4) return QMediaFormat::AudioCodec::AAC; - } else if (!strcmp(name, "x-ac3")) { + return QMediaFormat::AudioCodec::Unspecified; + } + if (name == "x-ac3"sv) return QMediaFormat::AudioCodec::AC3; - } else if (!strcmp(name, "x-eac3")) { + + if (name == "x-eac3"sv) return QMediaFormat::AudioCodec::EAC3; - } else if (!strcmp(name, "x-flac")) { + + if (name == "x-flac"sv) return QMediaFormat::AudioCodec::FLAC; - } else if (!strcmp(name, "x-alac")) { + + if (name == "x-alac"sv) return QMediaFormat::AudioCodec::ALAC; - } else if (!strcmp(name, "x-true-hd")) { + + if (name == "x-true-hd"sv) return QMediaFormat::AudioCodec::DolbyTrueHD; - } else if (!strcmp(name, "x-vorbis")) { + + if (name == "x-vorbis"sv) return QMediaFormat::AudioCodec::Vorbis; - } else if (!strcmp(name, "x-opus")) { + + if (name == "x-opus"sv) return QMediaFormat::AudioCodec::Opus; - } else if (!strcmp(name, "x-wav")) { + + if (name == "x-wav"sv) return QMediaFormat::AudioCodec::Wave; - } else if (!strcmp(name, "x-wma")) { + + if (name == "x-wma"sv) return QMediaFormat::AudioCodec::WMA; - } + return QMediaFormat::AudioCodec::Unspecified; } -QMediaFormat::VideoCodec QGstreamerFormatInfo::videoCodecForCaps(QGstStructure structure) +QMediaFormat::VideoCodec QGstreamerFormatInfo::videoCodecForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!name || strncmp(name, "video/", 6)) + if (!name || (strncmp(name, "video/", 6) != 0)) return QMediaFormat::VideoCodec::Unspecified; name += 6; - if (!strcmp(name, "mpeg")) { + if (name == "mpeg"sv) { auto version = structure["mpegversion"].toInt(); if (version == 1) return QMediaFormat::VideoCodec::MPEG1; - else if (version == 2) + if (version == 2) return QMediaFormat::VideoCodec::MPEG2; - else if (version == 4) + if (version == 4) return QMediaFormat::VideoCodec::MPEG4; - } else if (!strcmp(name, "x-h264")) { + return QMediaFormat::VideoCodec::Unspecified; + } + if (name == "x-h264"sv) return QMediaFormat::VideoCodec::H264; + #if GST_CHECK_VERSION(1, 17, 0) // x265enc seems to be broken on 1.16 at least - } else if (!strcmp(name, "x-h265")) { + if (name == "x-h265"sv) return QMediaFormat::VideoCodec::H265; #endif - } else if (!strcmp(name, "x-vp8")) { + + if (name == "x-vp8"sv) return QMediaFormat::VideoCodec::VP8; - } else if (!strcmp(name, "x-vp9")) { + + if (name == "x-vp9"sv) return QMediaFormat::VideoCodec::VP9; - } else if (!strcmp(name, "x-av1")) { + + if (name == "x-av1"sv) return QMediaFormat::VideoCodec::AV1; - } else if (!strcmp(name, "x-theora")) { + + if (name == "x-theora"sv) return QMediaFormat::VideoCodec::Theora; - } else if (!strcmp(name, "x-jpeg")) { + + if (name == "x-jpeg"sv) return QMediaFormat::VideoCodec::MotionJPEG; - } else if (!strcmp(name, "x-wmv")) { + + if (name == "x-wmv"sv) return QMediaFormat::VideoCodec::WMV; - } + return QMediaFormat::VideoCodec::Unspecified; } -QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructure structure) +QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!strcmp(name, "video/x-ms-asf")) { + if (name == "video/x-ms-asf"sv) return QMediaFormat::FileFormat::WMV; - } else if (!strcmp(name, "video/x-msvideo")) { + + if (name == "video/x-msvideo"sv) return QMediaFormat::FileFormat::AVI; - } else if (!strcmp(name, "video/x-matroska")) { + + if (name == "video/x-matroska"sv) return QMediaFormat::FileFormat::Matroska; - } else if (!strcmp(name, "video/quicktime")) { - auto variant = structure["variant"].toString(); + + if (name == "video/quicktime"sv) { + const char *variant = structure["variant"].toString(); if (!variant) return QMediaFormat::FileFormat::QuickTime; - else if (!strcmp(variant, "iso")) + if (variant == "iso"sv) return QMediaFormat::FileFormat::MPEG4; - } else if (!strcmp(name, "video/ogg")) { + } + if (name == "video/ogg"sv) return QMediaFormat::FileFormat::Ogg; - } else if (!strcmp(name, "video/webm")) { + + if (name == "video/webm"sv) return QMediaFormat::FileFormat::WebM; - } else if (!strcmp(name, "audio/x-m4a")) { + + if (name == "audio/x-m4a"sv) return QMediaFormat::FileFormat::Mpeg4Audio; - } else if (!strcmp(name, "audio/x-wav")) { + + if (name == "audio/x-wav"sv) return QMediaFormat::FileFormat::Wave; - } else if (!strcmp(name, "audio/mpeg")) { + + if (name == "audio/mpeg"sv) { auto mpegversion = structure["mpegversion"].toInt(); if (mpegversion == 1) { auto layer = structure["layer"]; @@ -151,23 +146,28 @@ QMediaFormat::FileFormat QGstreamerFormatInfo::fileFormatForCaps(QGstStructure s return QMediaFormat::FileFormat::MP3; } } + return QMediaFormat::UnspecifiedFormat; } -QImageCapture::FileFormat QGstreamerFormatInfo::imageFormatForCaps(QGstStructure structure) +QImageCapture::FileFormat QGstreamerFormatInfo::imageFormatForCaps(QGstStructureView structure) { + using namespace std::string_view_literals; const char *name = structure.name().data(); - if (!strcmp(name, "image/jpeg")) { + if (name == "image/jpeg"sv) return QImageCapture::JPEG; - } else if (!strcmp(name, "image/png")) { + + if (name == "image/png"sv) return QImageCapture::PNG; - } else if (!strcmp(name, "image/webp")) { + + if (name == "image/webp"sv) return QImageCapture::WebP; - } else if (!strcmp(name, "image/tiff")) { + + if (name == "image/tiff"sv) return QImageCapture::Tiff; - } + return QImageCapture::UnspecifiedFormat; } @@ -181,21 +181,16 @@ static QPair<QList<QMediaFormat::AudioCodec>, QList<QMediaFormat::VideoCodec>> g GList *elementList = gst_element_factory_list_get_elements(decode ? GST_ELEMENT_FACTORY_TYPE_DECODER : GST_ELEMENT_FACTORY_TYPE_ENCODER, GST_RANK_MARGINAL); - GList *element = elementList; - while (element) { - GstElementFactory *factory = (GstElementFactory *)element->data; - element = element->next; - - const GList *padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; - + for (GstElementFactory *factory : + QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { if (padTemplate->direction == padDirection) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); auto a = QGstreamerFormatInfo::audioCodecForCaps(structure); if (a != QMediaFormat::AudioCodec::Unspecified && !audio.contains(a)) audio.append(a); @@ -219,25 +214,23 @@ QList<QGstreamerFormatInfo::CodecMap> QGstreamerFormatInfo::getMuxerList(bool de GstPadDirection padDirection = demuxer ? GST_PAD_SINK : GST_PAD_SRC; - GList *elementList = gst_element_factory_list_get_elements(demuxer ? GST_ELEMENT_FACTORY_TYPE_DEMUXER : GST_ELEMENT_FACTORY_TYPE_MUXER, - GST_RANK_MARGINAL); - GList *element = elementList; - while (element) { - GstElementFactory *factory = (GstElementFactory *)element->data; - element = element->next; + GList *elementList = gst_element_factory_list_get_elements( + demuxer ? GST_ELEMENT_FACTORY_TYPE_DEMUXER : GST_ELEMENT_FACTORY_TYPE_MUXER, + GST_RANK_MARGINAL); + for (GstElementFactory *factory : + QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { QList<QMediaFormat::FileFormat> fileFormats; - const GList *padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { if (padTemplate->direction == padDirection) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); auto fmt = fileFormatForCaps(structure); if (fmt != QMediaFormat::UnspecifiedFormat) fileFormats.append(fmt); @@ -250,18 +243,17 @@ QList<QGstreamerFormatInfo::CodecMap> QGstreamerFormatInfo::getMuxerList(bool de QList<QMediaFormat::AudioCodec> audioCodecs; QList<QMediaFormat::VideoCodec> videoCodecs; - padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { // check the other side for supported inputs/outputs if (padTemplate->direction != padDirection) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + auto caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); bool acceptsRawAudio = false; for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); if (structure.name() == "audio/x-raw") acceptsRawAudio = true; auto audio = audioCodecForCaps(structure); @@ -290,7 +282,7 @@ QList<QGstreamerFormatInfo::CodecMap> QGstreamerFormatInfo::getMuxerList(bool de } } if (!audioCodecs.isEmpty() || !videoCodecs.isEmpty()) { - for (auto f : qAsConst(fileFormats)) { + for (auto f : std::as_const(fileFormats)) { muxers.append({f, audioCodecs, videoCodecs}); if (f == QMediaFormat::MPEG4 && !fileFormats.contains(QMediaFormat::Mpeg4Audio)) { muxers.append({QMediaFormat::Mpeg4Audio, audioCodecs, {}}); @@ -313,21 +305,17 @@ static QList<QImageCapture::FileFormat> getImageFormatList() GList *elementList = gst_element_factory_list_get_elements(GST_ELEMENT_FACTORY_TYPE_ENCODER, GST_RANK_MARGINAL); - GList *element = elementList; - while (element) { - GstElementFactory *factory = (GstElementFactory *)element->data; - element = element->next; - - const GList *padTemplates = gst_element_factory_get_static_pad_templates(factory); - while (padTemplates) { - GstStaticPadTemplate *padTemplate = (GstStaticPadTemplate *)padTemplates->data; - padTemplates = padTemplates->next; + for (GstElementFactory *factory : + QGstUtils::GListRangeAdaptor<GstElementFactory *>(elementList)) { + for (GstStaticPadTemplate *padTemplate : + QGstUtils::GListRangeAdaptor<GstStaticPadTemplate *>( + gst_element_factory_get_static_pad_templates(factory))) { if (padTemplate->direction == GST_PAD_SRC) { - QGstMutableCaps caps = gst_static_caps_get(&padTemplate->static_caps); + QGstCaps caps = QGstCaps(gst_static_caps_get(&padTemplate->static_caps), QGstCaps::HasRef); for (int i = 0; i < caps.size(); i++) { - QGstStructure structure = caps.at(i); + QGstStructureView structure = caps.at(i); auto f = QGstreamerFormatInfo::imageFormatForCaps(structure); if (f != QImageCapture::UnspecifiedFormat) { // qDebug() << structure.toString() << f; @@ -387,7 +375,7 @@ QGstreamerFormatInfo::QGstreamerFormatInfo() QGstreamerFormatInfo::~QGstreamerFormatInfo() = default; -QGstMutableCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const +QGstCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const { auto format = f.fileFormat(); Q_ASSERT(format != QMediaFormat::UnspecifiedFormat); @@ -407,14 +395,14 @@ QGstMutableCaps QGstreamerFormatInfo::formatCaps(const QMediaFormat &f) const "audio/x-flac", // FLAC "audio/x-wav" // Wave }; - return gst_caps_from_string(capsForFormat[format]); + return QGstCaps(gst_caps_from_string(capsForFormat[format]), QGstCaps::HasRef); } -QGstMutableCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const +QGstCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const { auto codec = f.audioCodec(); if (codec == QMediaFormat::AudioCodec::Unspecified) - return nullptr; + return {}; const char *capsForCodec[(int)QMediaFormat::AudioCodec::LastAudioCodec + 1] = { "audio/mpeg, mpegversion=(int)1, layer=(int)3", // MP3 @@ -429,14 +417,14 @@ QGstMutableCaps QGstreamerFormatInfo::audioCaps(const QMediaFormat &f) const "audio/x-wma", // WMA "audio/x-alac", // ALAC }; - return gst_caps_from_string(capsForCodec[(int)codec]); + return QGstCaps(gst_caps_from_string(capsForCodec[(int)codec]), QGstCaps::HasRef); } -QGstMutableCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const +QGstCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const { auto codec = f.videoCodec(); if (codec == QMediaFormat::VideoCodec::Unspecified) - return nullptr; + return {}; const char *capsForCodec[(int)QMediaFormat::VideoCodec::LastVideoCodec + 1] = { "video/mpeg, mpegversion=(int)1", // MPEG1, @@ -451,7 +439,7 @@ QGstMutableCaps QGstreamerFormatInfo::videoCaps(const QMediaFormat &f) const "audio/x-wmv", // WMV "video/x-jpeg", // MotionJPEG, }; - return gst_caps_from_string(capsForCodec[(int)codec]); + return QGstCaps(gst_caps_from_string(capsForCodec[(int)codec]), QGstCaps::HasRef); } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h index 3cfea9dc5..bba10edb9 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h +++ b/src/plugins/multimedia/gstreamer/qgstreamerformatinfo_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERFORMATINFO_H #define QGSTREAMERFORMATINFO_H @@ -52,9 +16,8 @@ // #include <private/qplatformmediaformatinfo_p.h> -#include <qhash.h> #include <qlist.h> -#include <qgstutils_p.h> +#include <common/qgst_p.h> QT_BEGIN_NAMESPACE @@ -64,14 +27,14 @@ public: QGstreamerFormatInfo(); ~QGstreamerFormatInfo(); - QGstMutableCaps formatCaps(const QMediaFormat &f) const; - QGstMutableCaps audioCaps(const QMediaFormat &f) const; - QGstMutableCaps videoCaps(const QMediaFormat &f) const; + QGstCaps formatCaps(const QMediaFormat &f) const; + QGstCaps audioCaps(const QMediaFormat &f) const; + QGstCaps videoCaps(const QMediaFormat &f) const; - static QMediaFormat::AudioCodec audioCodecForCaps(QGstStructure structure); - static QMediaFormat::VideoCodec videoCodecForCaps(QGstStructure structure); - static QMediaFormat::FileFormat fileFormatForCaps(QGstStructure structure); - static QImageCapture::FileFormat imageFormatForCaps(QGstStructure structure); + static QMediaFormat::AudioCodec audioCodecForCaps(QGstStructureView structure); + static QMediaFormat::VideoCodec videoCodecForCaps(QGstStructureView structure); + static QMediaFormat::FileFormat fileFormatForCaps(QGstStructureView structure); + static QImageCapture::FileFormat imageFormatForCaps(QGstStructureView structure); QList<CodecMap> getMuxerList(bool demuxer, QList<QMediaFormat::AudioCodec> audioCodecs, QList<QMediaFormat::VideoCodec> videoCodecs); }; diff --git a/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp b/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp index 8ed2fda03..87c514f2e 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp +++ b/src/plugins/multimedia/gstreamer/qgstreamerintegration.cpp @@ -1,150 +1,242 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "qgstreamerintegration_p.h" -#include "qgstreamervideodevices_p.h" -#include "qgstreamermediaplayer_p.h" -#include "qgstreamermediacapture_p.h" -#include "qgstreameraudiodecoder_p.h" -#include "qgstreamercamera_p.h" -#include "qgstreamermediaencoder_p.h" -#include "qgstreamerimagecapture_p.h" -#include "qgstreamerformatinfo_p.h" -#include "qgstreamervideosink_p.h" -#include "qgstreameraudioinput_p.h" -#include "qgstreameraudiooutput_p.h" -#include <QtMultimedia/private/qplatformmediaplugin_p.h> - -#include <memory> +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <qgstreamerintegration_p.h> +#include <qgstreamerformatinfo_p.h> +#include <qgstreamervideodevices_p.h> +#include <audio/qgstreameraudiodevice_p.h> +#include <audio/qgstreameraudiodecoder_p.h> +#include <common/qgstreameraudioinput_p.h> +#include <common/qgstreameraudiooutput_p.h> +#include <common/qgstreamermediaplayer_p.h> +#include <common/qgstreamervideosink_p.h> +#include <mediacapture/qgstreamercamera_p.h> +#include <mediacapture/qgstreamerimagecapture_p.h> +#include <mediacapture/qgstreamermediacapture_p.h> +#include <mediacapture/qgstreamermediaencoder_p.h> + +#include <QtCore/qloggingcategory.h> +#include <QtMultimedia/private/qmediaplayer_p.h> +#include <QtMultimedia/private/qmediacapturesession_p.h> +#include <QtMultimedia/private/qcameradevice_p.h> QT_BEGIN_NAMESPACE -class QGstreamerMediaPlugin : public QPlatformMediaPlugin +static thread_local bool inCustomCameraConstruction = false; +static thread_local QGstElement pendingCameraElement{}; + +QGStreamerPlatformSpecificInterfaceImplementation:: + ~QGStreamerPlatformSpecificInterfaceImplementation() = default; + +QAudioDevice QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerAudioInput( + const QByteArray &gstreamerPipeline) +{ + return qMakeCustomGStreamerAudioInput(gstreamerPipeline); +} + +QAudioDevice QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerAudioOutput( + const QByteArray &gstreamerPipeline) +{ + return qMakeCustomGStreamerAudioOutput(gstreamerPipeline); +} + +QCamera *QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerCamera( + const QByteArray &gstreamerPipeline, QObject *parent) { - Q_OBJECT - Q_PLUGIN_METADATA(IID QPlatformMediaPlugin_iid FILE "gstreamer.json") + QCameraDevicePrivate *info = new QCameraDevicePrivate; + info->id = gstreamerPipeline; + QCameraDevice device = info->create(); + + inCustomCameraConstruction = true; + auto guard = qScopeGuard([] { + inCustomCameraConstruction = false; + }); -public: - QGstreamerMediaPlugin() - : QPlatformMediaPlugin() - {} + return new QCamera(device, parent); +} - QPlatformMediaIntegration* create(const QString &name) override - { - if (name == QLatin1String("gstreamer")) - return new QGstreamerIntegration; +QCamera * +QGStreamerPlatformSpecificInterfaceImplementation::makeCustomGStreamerCamera(GstElement *element, + QObject *parent) +{ + QCameraDevicePrivate *info = new QCameraDevicePrivate; + info->id = "Custom Camera from GstElement"; + QCameraDevice device = info->create(); + + pendingCameraElement = QGstElement{ + element, + QGstElement::NeedsRef, + }; + + inCustomCameraConstruction = true; + auto guard = qScopeGuard([] { + inCustomCameraConstruction = false; + Q_ASSERT(!pendingCameraElement); + }); + + return new QCamera(device, parent); +} + +GstPipeline *QGStreamerPlatformSpecificInterfaceImplementation::gstPipeline(QMediaPlayer *player) +{ + auto *priv = reinterpret_cast<QMediaPlayerPrivate *>(QMediaPlayerPrivate::get(player)); + if (!priv) return nullptr; - } + + QGstreamerMediaPlayer *gstreamerPlayer = dynamic_cast<QGstreamerMediaPlayer *>(priv->control); + return gstreamerPlayer ? gstreamerPlayer->pipeline().pipeline() : nullptr; +} + +GstPipeline * +QGStreamerPlatformSpecificInterfaceImplementation::gstPipeline(QMediaCaptureSession *session) +{ + auto *priv = QMediaCaptureSessionPrivate::get(session); + if (!priv) + return nullptr; + + QGstreamerMediaCapture *gstreamerCapture = + dynamic_cast<QGstreamerMediaCapture *>(priv->captureSession.get()); + return gstreamerCapture ? gstreamerCapture->pipeline().pipeline() : nullptr; +} + +Q_LOGGING_CATEGORY(lcGstreamer, "qt.multimedia.gstreamer") + +namespace { + +void rankDownPlugin(GstRegistry *reg, const char *name) +{ + QGstPluginFeatureHandle pluginFeature{ + gst_registry_lookup_feature(reg, name), + QGstPluginFeatureHandle::HasRef, + }; + if (pluginFeature) + gst_plugin_feature_set_rank(pluginFeature.get(), GST_RANK_PRIMARY - 1); +} + +// https://gstreamer.freedesktop.org/documentation/vaapi/index.html +constexpr auto vaapiPluginNames = { + "vaapidecodebin", "vaapih264dec", "vaapih264enc", "vaapih265dec", + "vaapijpegdec", "vaapijpegenc", "vaapimpeg2dec", "vaapipostproc", + "vaapisink", "vaapivp8dec", "vaapivp9dec", +}; + +// https://gstreamer.freedesktop.org/documentation/va/index.html +constexpr auto vaPluginNames = { + "vaav1dec", "vacompositor", "vadeinterlace", "vah264dec", "vah264enc", "vah265dec", + "vajpegdec", "vampeg2dec", "vapostproc", "vavp8dec", "vavp9dec", +}; + +// https://gstreamer.freedesktop.org/documentation/nvcodec/index.html +constexpr auto nvcodecPluginNames = { + "cudaconvert", "cudaconvertscale", "cudadownload", "cudaipcsink", "cudaipcsrc", + "cudascale", "cudaupload", "nvautogpuh264enc", "nvautogpuh265enc", "nvav1dec", + "nvcudah264enc", "nvcudah265enc", "nvd3d11h264enc", "nvd3d11h265enc", "nvh264dec", + "nvh264enc", "nvh265dec", "nvh265enc", "nvjpegdec", "nvjpegenc", + "nvmpeg2videodec", "nvmpeg4videodec", "nvmpegvideodec", "nvvp8dec", "nvvp9dec", }; +} // namespace + QGstreamerIntegration::QGstreamerIntegration() + : QPlatformMediaIntegration(QLatin1String("gstreamer")) { gst_init(nullptr, nullptr); - m_videoDevices = new QGstreamerVideoDevices(this); - m_formatsInfo = new QGstreamerFormatInfo(); + qCDebug(lcGstreamer) << "Using gstreamer version: " << gst_version_string(); + + GstRegistry *reg = gst_registry_get(); + + if constexpr (!GST_CHECK_VERSION(1, 22, 0)) { + GstRegistry* reg = gst_registry_get(); + for (const char *name : vaapiPluginNames) + rankDownPlugin(reg, name); + } + + if (qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_VA")) { + for (const char *name : vaPluginNames) + rankDownPlugin(reg, name); + } + + if (qEnvironmentVariableIsSet("QT_GSTREAMER_DISABLE_NVCODEC")) { + for (const char *name : nvcodecPluginNames) + rankDownPlugin(reg, name); + } } -QGstreamerIntegration::~QGstreamerIntegration() +QPlatformMediaFormatInfo *QGstreamerIntegration::createFormatInfo() { - delete m_formatsInfo; + return new QGstreamerFormatInfo(); } -QPlatformMediaFormatInfo *QGstreamerIntegration::formatInfo() +QPlatformVideoDevices *QGstreamerIntegration::createVideoDevices() { - return m_formatsInfo; + return new QGstreamerVideoDevices(this); } -const QGstreamerFormatInfo *QGstreamerIntegration::gstFormatsInfo() const +const QGstreamerFormatInfo *QGstreamerIntegration::gstFormatsInfo() { - return m_formatsInfo; + return static_cast<const QGstreamerFormatInfo *>(formatInfo()); } -QPlatformAudioDecoder *QGstreamerIntegration::createAudioDecoder(QAudioDecoder *decoder) +QMaybe<QPlatformAudioDecoder *> QGstreamerIntegration::createAudioDecoder(QAudioDecoder *decoder) { - return new QGstreamerAudioDecoder(decoder); + return QGstreamerAudioDecoder::create(decoder); } -QPlatformMediaCaptureSession *QGstreamerIntegration::createCaptureSession() +QMaybe<QPlatformMediaCaptureSession *> QGstreamerIntegration::createCaptureSession() { - return new QGstreamerMediaCapture(); + return QGstreamerMediaCapture::create(); } -QPlatformMediaPlayer *QGstreamerIntegration::createPlayer(QMediaPlayer *player) +QMaybe<QPlatformMediaPlayer *> QGstreamerIntegration::createPlayer(QMediaPlayer *player) { - return new QGstreamerMediaPlayer(player); + return QGstreamerMediaPlayer::create(player); } -QPlatformCamera *QGstreamerIntegration::createCamera(QCamera *camera) +QMaybe<QPlatformCamera *> QGstreamerIntegration::createCamera(QCamera *camera) { - return new QGstreamerCamera(camera); + if (inCustomCameraConstruction) { + QGstElement element = std::exchange(pendingCameraElement, {}); + return element ? new QGstreamerCustomCamera{ camera, std::move(element) } + : new QGstreamerCustomCamera{ camera }; + } + + return QGstreamerCamera::create(camera); } -QPlatformMediaRecorder *QGstreamerIntegration::createRecorder(QMediaRecorder *recorder) +QMaybe<QPlatformMediaRecorder *> QGstreamerIntegration::createRecorder(QMediaRecorder *recorder) { return new QGstreamerMediaEncoder(recorder); } -QPlatformImageCapture *QGstreamerIntegration::createImageCapture(QImageCapture *imageCapture) +QMaybe<QPlatformImageCapture *> QGstreamerIntegration::createImageCapture(QImageCapture *imageCapture) { - return new QGstreamerImageCapture(imageCapture); + return QGstreamerImageCapture::create(imageCapture); } -QPlatformVideoSink *QGstreamerIntegration::createVideoSink(QVideoSink *sink) +QMaybe<QPlatformVideoSink *> QGstreamerIntegration::createVideoSink(QVideoSink *sink) { return new QGstreamerVideoSink(sink); } -QPlatformAudioInput *QGstreamerIntegration::createAudioInput(QAudioInput *q) +QMaybe<QPlatformAudioInput *> QGstreamerIntegration::createAudioInput(QAudioInput *q) { - return new QGstreamerAudioInput(q); + return QGstreamerAudioInput::create(q); } -QPlatformAudioOutput *QGstreamerIntegration::createAudioOutput(QAudioOutput *q) +QMaybe<QPlatformAudioOutput *> QGstreamerIntegration::createAudioOutput(QAudioOutput *q) { - return new QGstreamerAudioOutput(q); + return QGstreamerAudioOutput::create(q); } -GstDevice *QGstreamerIntegration::videoDevice(const QByteArray &id) const +GstDevice *QGstreamerIntegration::videoDevice(const QByteArray &id) { - return m_videoDevices ? static_cast<QGstreamerVideoDevices*>(m_videoDevices)->videoDevice(id) : nullptr; + const auto devices = videoDevices(); + return devices ? static_cast<QGstreamerVideoDevices *>(devices)->videoDevice(id) : nullptr; } -QT_END_NAMESPACE +QAbstractPlatformSpecificInterface *QGstreamerIntegration::platformSpecificInterface() +{ + return &m_platformSpecificImplementation; +} -#include "qgstreamerintegration.moc" +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h b/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h index 4b8d02efd..229bbd48e 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h +++ b/src/plugins/multimedia/gstreamer/qgstreamerintegration_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERINTEGRATION_H #define QGSTREAMERINTEGRATION_H @@ -51,39 +15,63 @@ // We mean it. // -#include <private/qplatformmediaintegration_p.h> +#include <QtMultimedia/private/qplatformmediaintegration_p.h> +#include <QtMultimedia/private/qgstreamer_platformspecificinterface_p.h> + #include <gst/gst.h> QT_BEGIN_NAMESPACE class QGstreamerFormatInfo; +class QGStreamerPlatformSpecificInterfaceImplementation : public QGStreamerPlatformSpecificInterface +{ +public: + ~QGStreamerPlatformSpecificInterfaceImplementation() override; + + QAudioDevice makeCustomGStreamerAudioInput(const QByteArray &gstreamerPipeline) override; + QAudioDevice makeCustomGStreamerAudioOutput(const QByteArray &gstreamerPipeline) override; + QCamera *makeCustomGStreamerCamera(const QByteArray &gstreamerPipeline, + QObject *parent) override; + + QCamera *makeCustomGStreamerCamera(GstElement *, QObject *parent) override; + + GstPipeline *gstPipeline(QMediaPlayer *) override; + GstPipeline *gstPipeline(QMediaCaptureSession *) override; +}; + class QGstreamerIntegration : public QPlatformMediaIntegration { public: QGstreamerIntegration(); - ~QGstreamerIntegration(); - static QGstreamerIntegration *instance() { return static_cast<QGstreamerIntegration *>(QPlatformMediaIntegration::instance()); } - QPlatformMediaFormatInfo *formatInfo() override; + static QGstreamerIntegration *instance() + { + return static_cast<QGstreamerIntegration *>(QPlatformMediaIntegration::instance()); + } + + QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *decoder) override; + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *player) override; + QMaybe<QPlatformCamera *> createCamera(QCamera *) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *) override; + + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *sink) override; - QPlatformAudioDecoder *createAudioDecoder(QAudioDecoder *decoder) override; - QPlatformMediaCaptureSession *createCaptureSession() override; - QPlatformMediaPlayer *createPlayer(QMediaPlayer *player) override; - QPlatformCamera *createCamera(QCamera *) override; - QPlatformMediaRecorder *createRecorder(QMediaRecorder *) override; - QPlatformImageCapture *createImageCapture(QImageCapture *) override; + QMaybe<QPlatformAudioInput *> createAudioInput(QAudioInput *) override; + QMaybe<QPlatformAudioOutput *> createAudioOutput(QAudioOutput *) override; - QPlatformVideoSink *createVideoSink(QVideoSink *sink) override; + const QGstreamerFormatInfo *gstFormatsInfo(); + GstDevice *videoDevice(const QByteArray &id); - QPlatformAudioInput *createAudioInput(QAudioInput *) override; - QPlatformAudioOutput *createAudioOutput(QAudioOutput *) override; + QAbstractPlatformSpecificInterface *platformSpecificInterface() override; - const QGstreamerFormatInfo *gstFormatsInfo() const; - GstDevice *videoDevice(const QByteArray &id) const; +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; + QPlatformVideoDevices *createVideoDevices() override; -private: - QGstreamerFormatInfo *m_formatsInfo; + QGStreamerPlatformSpecificInterfaceImplementation m_platformSpecificImplementation; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamerplugin.cpp b/src/plugins/multimedia/gstreamer/qgstreamerplugin.cpp new file mode 100644 index 000000000..66ad7f712 --- /dev/null +++ b/src/plugins/multimedia/gstreamer/qgstreamerplugin.cpp @@ -0,0 +1,28 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QtMultimedia/private/qplatformmediaplugin_p.h> + +#include <qgstreamerintegration_p.h> + +QT_BEGIN_NAMESPACE + +class QGstreamerMediaPlugin : public QPlatformMediaPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID QPlatformMediaPlugin_iid FILE "gstreamer.json") + +public: + QGstreamerMediaPlugin() = default; + + QPlatformMediaIntegration* create(const QString &name) override + { + if (name == u"gstreamer") + return new QGstreamerIntegration; + return nullptr; + } +}; + +QT_END_NAMESPACE + +#include "qgstreamerplugin.moc" diff --git a/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp b/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp index e7595b4cc..78ac16eb4 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp +++ b/src/plugins/multimedia/gstreamer/qgstreamervideodevices.cpp @@ -1,201 +1,158 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qgstreamervideodevices_p.h" -#include "qmediadevices.h" -#include "private/qcameradevice_p.h" +#include <QtMultimedia/qmediadevices.h> +#include <QtMultimedia/private/qcameradevice_p.h> -#include "qgstreameraudiosource_p.h" -#include "qgstreameraudiosink_p.h" -#include "qgstreameraudiodevice_p.h" -#include "qgstutils_p.h" +#include <common/qgst_p.h> +#include <common/qgstutils_p.h> +#include <common/qglist_helper_p.h> QT_BEGIN_NAMESPACE -static gboolean deviceMonitor(GstBus *, GstMessage *message, gpointer m) +static gboolean deviceMonitorCallback(GstBus *, GstMessage *message, gpointer m) { auto *manager = static_cast<QGstreamerVideoDevices *>(m); - GstDevice *device = nullptr; + QGstDeviceHandle device; - switch (GST_MESSAGE_TYPE (message)) { + switch (GST_MESSAGE_TYPE(message)) { case GST_MESSAGE_DEVICE_ADDED: gst_message_parse_device_added(message, &device); - manager->addDevice(device); + manager->addDevice(std::move(device)); break; case GST_MESSAGE_DEVICE_REMOVED: gst_message_parse_device_removed(message, &device); - manager->removeDevice(device); + manager->removeDevice(std::move(device)); break; default: break; } - if (device) - gst_object_unref (device); return G_SOURCE_CONTINUE; } QGstreamerVideoDevices::QGstreamerVideoDevices(QPlatformMediaIntegration *integration) - : QPlatformVideoDevices(integration) + : QPlatformVideoDevices(integration), + m_deviceMonitor{ + gst_device_monitor_new(), + } { - GstDeviceMonitor *monitor; - GstBus *bus; + gst_device_monitor_add_filter(m_deviceMonitor.get(), "Video/Source", nullptr); - monitor = gst_device_monitor_new(); + QGstBusHandle bus{ + gst_device_monitor_get_bus(m_deviceMonitor.get()), + }; + gst_bus_add_watch(bus.get(), deviceMonitorCallback, this); + gst_device_monitor_start(m_deviceMonitor.get()); - gst_device_monitor_add_filter (monitor, nullptr, nullptr); + GList *devices = gst_device_monitor_get_devices(m_deviceMonitor.get()); - bus = gst_device_monitor_get_bus(monitor); - gst_bus_add_watch(bus, deviceMonitor, this); - gst_object_unref(bus); - - gst_device_monitor_start(monitor); + for (GstDevice *device : QGstUtils::GListRangeAdaptor<GstDevice *>(devices)) { + addDevice(QGstDeviceHandle{ + device, + QGstDeviceHandle::HasRef, + }); + } - auto devices = gst_device_monitor_get_devices(monitor); + g_list_free(devices); +} - while (devices) { - GstDevice *device = static_cast<GstDevice *>(devices->data); - addDevice(device); - gst_object_unref(device); - devices = g_list_delete_link(devices, devices); - } +QGstreamerVideoDevices::~QGstreamerVideoDevices() +{ + gst_device_monitor_stop(m_deviceMonitor.get()); } QList<QCameraDevice> QGstreamerVideoDevices::videoDevices() const { QList<QCameraDevice> devices; - for (auto *d : qAsConst(m_videoSources)) { - QGstStructure properties = gst_device_get_properties(d); - if (!properties.isNull()) { - QCameraDevicePrivate *info = new QCameraDevicePrivate; - auto *desc = gst_device_get_display_name(d); - info->description = QString::fromUtf8(desc); - g_free(desc); - - info->id = properties["device.path"].toString(); - auto def = properties["is-default"].toBool(); + for (const auto &device : m_videoSources) { + QCameraDevicePrivate *info = new QCameraDevicePrivate; + + QGString desc{ + gst_device_get_display_name(device.gstDevice.get()), + }; + info->description = desc.toQString(); + info->id = device.id; + + QUniqueGstStructureHandle properties{ + gst_device_get_properties(device.gstDevice.get()), + }; + if (properties) { + QGstStructureView view{ properties }; + auto def = view["is-default"].toBool(); info->isDefault = def && *def; - if (def) - devices.prepend(info->create()); - else - devices.append(info->create()); - properties.free(); - QGstCaps caps = gst_device_get_caps(d); - if (!caps.isNull()) { - QList<QCameraFormat> formats; - QSet<QSize> photoResolutions; - - int size = caps.size(); - for (int i = 0; i < size; ++i) { - auto cap = caps.at(i); - - QSize resolution = cap.resolution(); - if (!resolution.isValid()) - continue; - - auto pixelFormat = cap.pixelFormat(); - auto frameRate = cap.frameRateRange(); - - auto *f = new QCameraFormatPrivate{ - QSharedData(), - pixelFormat, - resolution, - frameRate.min, - frameRate.max - }; - formats << f->create(); - photoResolutions.insert(resolution); - } - info->videoFormats = formats; - // ### sort resolutions? - info->photoResolutions = photoResolutions.values(); + } + + if (info->isDefault) + devices.prepend(info->create()); + else + devices.append(info->create()); + + auto caps = QGstCaps(gst_device_get_caps(device.gstDevice.get()), QGstCaps::HasRef); + if (!caps.isNull()) { + QList<QCameraFormat> formats; + QSet<QSize> photoResolutions; + + int size = caps.size(); + for (int i = 0; i < size; ++i) { + auto cap = caps.at(i); + + QSize resolution = cap.resolution(); + if (!resolution.isValid()) + continue; + + auto pixelFormat = cap.pixelFormat(); + auto frameRate = cap.frameRateRange(); + + auto *f = new QCameraFormatPrivate{ QSharedData(), pixelFormat, resolution, + frameRate.min, frameRate.max }; + formats << f->create(); + photoResolutions.insert(resolution); } + info->videoFormats = formats; + // ### sort resolutions? + info->photoResolutions = photoResolutions.values(); } } return devices; } -void QGstreamerVideoDevices::addDevice(GstDevice *device) +void QGstreamerVideoDevices::addDevice(QGstDeviceHandle device) { - gchar *type = gst_device_get_device_class(device); -// qDebug() << "adding device:" << device << type << gst_device_get_display_name(device) << gst_structure_to_string(gst_device_get_properties(device)); - gst_object_ref(device); - if (!strcmp(type, "Video/Source") || !strcmp(type, "Source/Video")) { - m_videoSources.insert(device); - videoInputsChanged(); - } else { - gst_object_unref(device); - } - g_free(type); -} + Q_ASSERT(gst_device_has_classes(device.get(), "Video/Source")); -void QGstreamerVideoDevices::removeDevice(GstDevice *device) -{ -// qDebug() << "removing device:" << device << gst_device_get_display_name(device); - if (m_videoSources.remove(device)) - videoInputsChanged(); + auto it = std::find_if(m_videoSources.begin(), m_videoSources.end(), + [&](const QGstRecordDevice &a) { return a.gstDevice == device; }); + + if (it != m_videoSources.end()) + return; - gst_object_unref(device); + m_videoSources.push_back(QGstRecordDevice{ + std::move(device), + QByteArray::number(m_idGenerator), + }); + emit videoInputsChanged(); + m_idGenerator++; } -static GstDevice *getDevice(const QSet<GstDevice *> &devices, const char *key, const QByteArray &id) +void QGstreamerVideoDevices::removeDevice(QGstDeviceHandle device) { - GstDevice *gstDevice = nullptr; - for (auto *d : devices) { - QGstStructure properties = gst_device_get_properties(d); - if (!properties.isNull()) { - auto *name = properties[key].toString(); - if (id == name) { - gstDevice = d; - } - } - properties.free(); - if (gstDevice) - break; + auto it = std::find_if(m_videoSources.begin(), m_videoSources.end(), + [&](const QGstRecordDevice &a) { return a.gstDevice == device; }); + + if (it != m_videoSources.end()) { + m_videoSources.erase(it); + emit videoInputsChanged(); } - return gstDevice; } GstDevice *QGstreamerVideoDevices::videoDevice(const QByteArray &id) const { - return getDevice(m_videoSources, "device.path", id); + auto it = std::find_if(m_videoSources.begin(), m_videoSources.end(), + [&](const QGstRecordDevice &a) { return a.id == id; }); + return it != m_videoSources.end() ? it->gstDevice.get() : nullptr; } QT_END_NAMESPACE diff --git a/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h b/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h index cd0acc053..a321ae66b 100644 --- a/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h +++ b/src/plugins/multimedia/gstreamer/qgstreamervideodevices_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QGSTREAMERMEDIADEVICES_H #define QGSTREAMERMEDIADEVICES_H @@ -53,24 +17,36 @@ #include <private/qplatformvideodevices_p.h> #include <gst/gst.h> -#include <qset.h> #include <qaudiodevice.h> +#include <vector> + +#include <common/qgst_handle_types_p.h> QT_BEGIN_NAMESPACE class QGstreamerVideoDevices : public QPlatformVideoDevices { public: - QGstreamerVideoDevices(QPlatformMediaIntegration *integration); + explicit QGstreamerVideoDevices(QPlatformMediaIntegration *integration); + ~QGstreamerVideoDevices(); QList<QCameraDevice> videoDevices() const override; GstDevice *videoDevice(const QByteArray &id) const; - void addDevice(GstDevice *); - void removeDevice(GstDevice *); + void addDevice(QGstDeviceHandle); + void removeDevice(QGstDeviceHandle); private: - QSet<GstDevice *> m_videoSources; + struct QGstRecordDevice + { + QGstDeviceHandle gstDevice; + QByteArray id; + }; + + quint64 m_idGenerator = 0; + std::vector<QGstRecordDevice> m_videoSources; + + QGstDeviceMonitorHandle m_deviceMonitor; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/qnx/CMakeLists.txt b/src/plugins/multimedia/qnx/CMakeLists.txt index 18d062343..e1ac0ffa3 100644 --- a/src/plugins/multimedia/qnx/CMakeLists.txt +++ b/src/plugins/multimedia/qnx/CMakeLists.txt @@ -1,10 +1,13 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + qt_internal_add_plugin(QQnxMediaPlugin OUTPUT_NAME qnxmediaplugin PLUGIN_TYPE multimedia SOURCES - SOURCES camera/qqnxcamera.cpp camera/qqnxcamera_p.h + camera/qqnxplatformcamera.cpp camera/qqnxplatformcamera_p.h camera/qqnxcameraframebuffer.cpp camera/qqnxcameraframebuffer_p.h camera/qqnximagecapture.cpp camera/qqnximagecapture_p.h common/qqnxaudioinput.cpp common/qqnxaudioinput_p.h diff --git a/src/plugins/multimedia/qnx/camera/qqnxcamera.cpp b/src/plugins/multimedia/qnx/camera/qqnxcamera.cpp index 35008be45..6976221bd 100644 --- a/src/plugins/multimedia/qnx/camera/qqnxcamera.cpp +++ b/src/plugins/multimedia/qnx/camera/qqnxcamera.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxcamera_p.h" #include "qqnxcameraframebuffer_p.h" #include "qqnxmediacapturesession_p.h" @@ -46,21 +10,15 @@ #include <private/qmediastoragelocation_p.h> -static constexpr camera_focusmode_t qnxFocusMode(QCamera::FocusMode mode) +QDebug &operator<<(QDebug &d, const QQnxCamera::VideoFormat &f) { - switch (mode) { - default: - case QCamera::FocusModeAuto: - case QCamera::FocusModeAutoFar: - case QCamera::FocusModeInfinity: - return CAMERA_FOCUSMODE_CONTINUOUS_AUTO; - case QCamera::FocusModeAutoNear: - return CAMERA_FOCUSMODE_CONTINUOUS_MACRO; - case QCamera::FocusModeHyperfocal: - return CAMERA_FOCUSMODE_EDOF; - case QCamera::FocusModeManual: - return CAMERA_FOCUSMODE_MANUAL; - } + d << "VideoFormat - width=" << f.width + << "height=" << f.height + << "rotation=" << f.rotation + << "frameRate=" << f.frameRate + << "frameType=" << f.frameType; + + return d; } static QString statusToString(camera_devstatus_t status) @@ -101,10 +59,28 @@ static QString statusToString(camera_devstatus_t status) QT_BEGIN_NAMESPACE -QQnxCamera::QQnxCamera(QCamera *parent) - : QPlatformCamera(parent) +QQnxCamera::QQnxCamera(camera_unit_t unit, QObject *parent) + : QObject(parent) + , m_cameraUnit(unit) { - setCamera(QMediaDevices::defaultVideoInput()); + if (!m_handle.open(m_cameraUnit, CAMERA_MODE_RW)) + qWarning("QQnxCamera: Failed to open camera (0x%x)", m_handle.lastError()); + + if (camera_set_vf_mode(m_handle.get(), CAMERA_VFMODE_VIDEO) != CAMERA_EOK) { + qWarning("QQnxCamera: unable to configure viewfinder mode"); + return; + } + + if (camera_set_vf_property(m_handle.get(), CAMERA_IMGPROP_CREATEWINDOW, 0, + CAMERA_IMGPROP_RENDERTOWINDOW, 0) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to set camera properties"); + return; + } + + updateZoomLimits(); + updateSupportedWhiteBalanceValues(); + + m_valid = true; } QQnxCamera::~QQnxCamera() @@ -112,43 +88,38 @@ QQnxCamera::~QQnxCamera() stop(); } -bool QQnxCamera::isActive() const -{ - return m_handle.isOpen() && m_viewfinderActive; -} - -void QQnxCamera::setActive(bool active) +camera_unit_t QQnxCamera::unit() const { - if (active) - start(); - else - stop(); + return m_cameraUnit; } -void QQnxCamera::start() +QString QQnxCamera::name() const { - if (isActive()) - return; + char name[CAMERA_LOCATION_NAMELEN]; - updateCameraFeatures(); - - if (camera_set_vf_property(m_handle.get(), CAMERA_IMGPROP_CREATEWINDOW, 0, - CAMERA_IMGPROP_RENDERTOWINDOW, 0) != CAMERA_EOK) { - qWarning("QQnxCamera: failed to set camera properties"); - return; + if (camera_get_location_property(m_cameraUnit, + CAMERA_LOCATION_NAME, &name, CAMERA_LOCATION_END) != CAMERA_EOK) { + qWarning("QQnxCamera: unable to obtain camera name"); + return {}; } - constexpr camera_vfmode_t mode = CAMERA_VFMODE_DEFAULT; + return QString::fromUtf8(name); +} - if (!supportedVfModes().contains(mode)) { - qWarning("QQnxCamera: unsupported viewfinder mode"); - return; - } +bool QQnxCamera::isValid() const +{ + return m_valid; +} - if (camera_set_vf_mode(m_handle.get(), mode) != CAMERA_EOK) { - qWarning("QQnxCamera: unable to configure viewfinder mode"); +bool QQnxCamera::isActive() const +{ + return m_handle.isOpen() && m_viewfinderActive; +} + +void QQnxCamera::start() +{ + if (isActive()) return; - } if (camera_start_viewfinder(m_handle.get(), viewfinderCallback, statusCallback, this) != CAMERA_EOK) { @@ -157,12 +128,6 @@ void QQnxCamera::start() } m_viewfinderActive = true; - - if (m_session) - m_videoSink = m_session->videoSink(); - - if (isVideoEncodingSupported() && m_outputUrl.isValid()) - startVideoRecording(); } void QQnxCamera::stop() @@ -177,35 +142,17 @@ void QQnxCamera::stop() qWarning("QQnxCamera: Failed to stop camera"); m_viewfinderActive = false; - - m_videoSink = nullptr; } -void QQnxCamera::setCamera(const QCameraDevice &camera) -{ - if (m_camera == camera) - return; - - stop(); - - m_handle = {}; - - m_camera = camera; - m_cameraUnit = camera_unit_t(camera.id().toUInt()); - - if (!m_handle.open(m_cameraUnit, CAMERA_MODE_RO|CAMERA_MODE_PWRITE)) - qWarning("QQnxCamera: Failed to open camera (0x%x)", m_handle.lastError()); -} - -bool QQnxCamera::setCameraFormat(const QCameraFormat &format) +bool QQnxCamera::setCameraFormat(uint32_t width, uint32_t height, double frameRate) { if (!m_handle.isOpen()) return false; const camera_error_t error = camera_set_vf_property(m_handle.get(), - CAMERA_IMGPROP_WIDTH, format.resolution().width(), - CAMERA_IMGPROP_HEIGHT, format.resolution().height(), - CAMERA_IMGPROP_FRAMERATE, format.maxFrameRate()); + CAMERA_IMGPROP_WIDTH, width, + CAMERA_IMGPROP_HEIGHT, height, + CAMERA_IMGPROP_FRAMERATE, frameRate); if (error != CAMERA_EOK) { qWarning("QQnxCamera: failed to set camera format"); @@ -215,132 +162,264 @@ bool QQnxCamera::setCameraFormat(const QCameraFormat &format) return true; } -void QQnxCamera::setCaptureSession(QPlatformMediaCaptureSession *session) +bool QQnxCamera::isFocusModeSupported(camera_focusmode_t mode) const { - if (m_session == session) - return; - m_session = static_cast<QQnxMediaCaptureSession *>(session); + return supportedFocusModes().contains(mode); } -bool QQnxCamera::isFocusModeSupported(QCamera::FocusMode mode) const +bool QQnxCamera::setFocusMode(camera_focusmode_t mode) { - return supportedFocusModes().contains(::qnxFocusMode(mode)); + if (!isActive()) + return false; + + const camera_error_t result = camera_set_focus_mode(m_handle.get(), mode); + + if (result != CAMERA_EOK) { + qWarning("QQnxCamera: Unable to set focus mode (0x%x)", result); + return false; + } + + focusModeChanged(mode); + + return true; } -void QQnxCamera::setFocusMode(QCamera::FocusMode mode) +camera_focusmode_t QQnxCamera::focusMode() const { if (!isActive()) - return; + return CAMERA_FOCUSMODE_OFF; + + camera_focusmode_t mode; - const camera_error_t result = camera_set_focus_mode(m_handle.get(), ::qnxFocusMode(mode)); + const camera_error_t result = camera_get_focus_mode(m_handle.get(), &mode); if (result != CAMERA_EOK) { qWarning("QQnxCamera: Unable to set focus mode (0x%x)", result); - return; + return CAMERA_FOCUSMODE_OFF; } - focusModeChanged(mode); + return mode; +} + +QQnxCamera::VideoFormat QQnxCamera::vfFormat() const +{ + VideoFormat f = {}; + + if (camera_get_vf_property(m_handle.get(), + CAMERA_IMGPROP_WIDTH, &f.width, + CAMERA_IMGPROP_HEIGHT, &f.height, + CAMERA_IMGPROP_ROTATION, &f.rotation, + CAMERA_IMGPROP_FRAMERATE, &f.frameRate, + CAMERA_IMGPROP_FORMAT, &f.frameType) != CAMERA_EOK) { + qWarning("QQnxCamera: Failed to query video finder frameType"); + } + + return f; +} + +void QQnxCamera::setVfFormat(const VideoFormat &f) +{ + const bool active = isActive(); + + if (active) + stop(); + + if (camera_set_vf_property(m_handle.get(), + CAMERA_IMGPROP_WIDTH, f.width, + CAMERA_IMGPROP_HEIGHT, f.height, + CAMERA_IMGPROP_ROTATION, f.rotation, + CAMERA_IMGPROP_FRAMERATE, f.frameRate, + CAMERA_IMGPROP_FORMAT, f.frameType) != CAMERA_EOK) { + qWarning("QQnxCamera: Failed to set video finder frameType"); + } + + if (active) + start(); +} + +QQnxCamera::VideoFormat QQnxCamera::recordingFormat() const +{ + VideoFormat f = {}; + + if (camera_get_video_property(m_handle.get(), + CAMERA_IMGPROP_WIDTH, &f.width, + CAMERA_IMGPROP_HEIGHT, &f.height, + CAMERA_IMGPROP_ROTATION, &f.rotation, + CAMERA_IMGPROP_FRAMERATE, &f.frameRate, + CAMERA_IMGPROP_FORMAT, &f.frameType) != CAMERA_EOK) { + qWarning("QQnxCamera: Failed to query recording frameType"); + } + + return f; +} + +void QQnxCamera::setRecordingFormat(const VideoFormat &f) +{ + if (camera_set_video_property(m_handle.get(), + CAMERA_IMGPROP_WIDTH, f.width, + CAMERA_IMGPROP_HEIGHT, f.height, + CAMERA_IMGPROP_ROTATION, f.rotation, + CAMERA_IMGPROP_FRAMERATE, f.frameRate, + CAMERA_IMGPROP_FORMAT, f.frameType) != CAMERA_EOK) { + qWarning("QQnxCamera: Failed to set recording frameType"); + } } void QQnxCamera::setCustomFocusPoint(const QPointF &point) { - // get the size of the viewfinder - int width = 0; - int height = 0; - auto result = camera_get_vf_property(m_handle.get(), - CAMERA_IMGPROP_WIDTH, width, - CAMERA_IMGPROP_HEIGHT, height); - if (result != CAMERA_EOK) + const QSize vfSize = viewFinderSize(); + + if (vfSize.isEmpty()) return; + const auto toUint32 = [](double value) { + return static_cast<uint32_t>(std::max(0.0, value)); + }; + // define a 40x40 pixel focus region around the custom focus point - camera_region_t focusRegion; - focusRegion.left = qMax(0, static_cast<int>(point.x() * width) - 20); - focusRegion.top = qMax(0, static_cast<int>(point.y() * height) - 20); - focusRegion.width = 40; - focusRegion.height = 40; + constexpr int pixelSize = 40; - result = camera_set_focus_regions(m_handle.get(), 1, &focusRegion); - if (result != CAMERA_EOK) { - qWarning("QQnxCamera: Unable to set focus region (0x%x)", result); - return; - } - auto qnxMode = ::qnxFocusMode(focusMode()); - result = camera_set_focus_mode(m_handle.get(), qnxMode); - if (result != CAMERA_EOK) { - qWarning("QQnxCamera: Unable to set focus region (0x%x)", result); + const auto left = toUint32(point.x() * vfSize.width() - pixelSize / 2); + const auto top = toUint32(point.y() * vfSize.height() - pixelSize / 2); + + camera_region_t focusRegion { + .left = left, + .top = top, + .width = pixelSize, + .height = pixelSize, + .extra = 0 + }; + + if (camera_set_focus_regions(m_handle.get(), 1, &focusRegion) != CAMERA_EOK) { + qWarning("QQnxCamera: Unable to set focus region"); return; } - customFocusPointChanged(point); + + if (setFocusMode(focusMode())) + customFocusPointChanged(point); } -void QQnxCamera::setFocusDistance(float distance) +void QQnxCamera::setManualFocusStep(int step) { - if (!isActive() || !isFocusModeSupported(QCamera::FocusModeManual)) + if (!isActive()) { + qWarning("QQnxCamera: Failed to set focus distance - view finder not active"); return; + } - const int maxDistance = maxFocusDistance(); - - if (maxDistance < 0) + if (!isFocusModeSupported(CAMERA_FOCUSMODE_MANUAL)) { + qWarning("QQnxCamera: Failed to set focus distance - manual focus mode not supported"); return; + } - const int qnxDistance = maxDistance * std::min(distance, 1.0f); - - if (camera_set_manual_focus_step(m_handle.get(), qnxDistance) != CAMERA_EOK) + if (camera_set_manual_focus_step(m_handle.get(), step) != CAMERA_EOK) qWarning("QQnxCamera: Failed to set focus distance"); } -int QQnxCamera::maxFocusDistance() const +int QQnxCamera::manualFocusStep() const { - if (!isActive() || !isFocusModeSupported(QCamera::FocusModeManual)) - return -1; + return focusStep().step; +} - int maxstep; - int step; +int QQnxCamera::maxFocusStep() const +{ + return focusStep().maxStep; +} - if (camera_get_manual_focus_step(m_handle.get(), &maxstep, &step) != CAMERA_EOK) { +QQnxCamera::FocusStep QQnxCamera::focusStep() const +{ + constexpr FocusStep invalidStep { -1, -1 }; + + if (!isActive()) { + qWarning("QQnxCamera: Failed to query max focus distance - view finder not active"); + return invalidStep; + } + + if (!isFocusModeSupported(CAMERA_FOCUSMODE_MANUAL)) { + qWarning("QQnxCamera: Failed to query max focus distance - " + "manual focus mode not supported"); + return invalidStep; + } + + FocusStep focusStep; + + if (camera_get_manual_focus_step(m_handle.get(), + &focusStep.maxStep, &focusStep.step) != CAMERA_EOK) { qWarning("QQnxCamera: Unable to query camera focus step"); - return -1; + return invalidStep; } - return maxstep; + return focusStep; } -void QQnxCamera::zoomTo(float factor, float) + +QSize QQnxCamera::viewFinderSize() const { - if (!isActive()) - return; + // get the size of the viewfinder + int width = 0; + int height = 0; - if (maxZoom <= minZoom) - return; - // QNX has an integer based API. Interpolate between the levels according to the factor we get - const float max = maxZoomFactor(); - const float min = minZoomFactor(); - if (max <= min) - return; - factor = qBound(min, factor, max) - min; - uint zoom = minZoom + (uint)qRound(factor*(maxZoom - minZoom)/(max - min)); + if (camera_get_vf_property(m_handle.get(), + CAMERA_IMGPROP_WIDTH, width, + CAMERA_IMGPROP_HEIGHT, height) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to query view finder size"); + return {}; + } - auto error = camera_set_vf_property(m_handle.get(), CAMERA_IMGPROP_ZOOMFACTOR, zoom); - if (error == CAMERA_EOK) - zoomFactorChanged(factor); + return { width, height }; } -void QQnxCamera::setExposureCompensation(float ev) +uint32_t QQnxCamera::minimumZoomLevel() const +{ + return m_minZoom; +} + +uint32_t QQnxCamera::maximumZoomLevel() const +{ + return m_maxZoom; +} + +bool QQnxCamera::isSmoothZoom() const +{ + return m_smoothZoom; +} + +double QQnxCamera::zoomRatio(uint32_t zoomLevel) const +{ + double ratio; + + if (camera_get_zoom_ratio_from_zoom_level(m_handle.get(), zoomLevel, &ratio) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to query zoom ratio from zoom level"); + return 0.0; + } + + return ratio; +} + +bool QQnxCamera::setZoomFactor(uint32_t factor) +{ + if (camera_set_vf_property(m_handle.get(), CAMERA_IMGPROP_ZOOMFACTOR, factor) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to set zoom factor"); + return false; + } + + return true; +} + +void QQnxCamera::setEvOffset(float ev) { if (!isActive()) return; if (camera_set_ev_offset(m_handle.get(), ev) != CAMERA_EOK) - qWarning("QQnxCamera: Failed to setup exposure compensation"); + qWarning("QQnxCamera: Failed to set up exposure compensation"); } -int QQnxCamera::isoSensitivity() const +uint32_t QQnxCamera::manualIsoSensitivity() const { if (!isActive()) return 0; - unsigned int isoValue; + uint32_t isoValue; if (camera_get_manual_iso(m_handle.get(), &isoValue) != CAMERA_EOK) { qWarning("QQnxCamera: Failed to query ISO value"); @@ -350,18 +429,16 @@ int QQnxCamera::isoSensitivity() const return isoValue; } -void QQnxCamera::setManualIsoSensitivity(int value) +void QQnxCamera::setManualIsoSensitivity(uint32_t value) { if (!isActive()) return; - const unsigned int isoValue = std::max(0, value); - - if (camera_set_manual_iso(m_handle.get(), isoValue) != CAMERA_EOK) + if (camera_set_manual_iso(m_handle.get(), value) != CAMERA_EOK) qWarning("QQnxCamera: Failed to set ISO value"); } -void QQnxCamera::setManualExposureTime(float seconds) +void QQnxCamera::setManualExposureTime(double seconds) { if (!isActive()) return; @@ -370,95 +447,120 @@ void QQnxCamera::setManualExposureTime(float seconds) qWarning("QQnxCamera: Failed to set exposure time"); } -float QQnxCamera::exposureTime() const +double QQnxCamera::manualExposureTime() const { if (!isActive()) - return 0; + return 0.0; double shutterSpeed; if (camera_get_manual_shutter_speed(m_handle.get(), &shutterSpeed) != CAMERA_EOK) { qWarning("QQnxCamera: Failed to get exposure time"); - return 0; + return 0.0; } - return static_cast<float>(shutterSpeed); + return shutterSpeed; } -bool QQnxCamera::isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const +bool QQnxCamera::hasFeature(camera_feature_t feature) const { - if (!whiteBalanceModesChecked) { - whiteBalanceModesChecked = true; - unsigned numWhiteBalanceValues = 0; - auto error = camera_get_supported_manual_white_balance_values(m_handle.get(), 0, &numWhiteBalanceValues, - nullptr, &continuousColorTemperatureSupported); - if (error == CAMERA_EOK) { - manualColorTemperatureValues.resize(numWhiteBalanceValues); - auto error = camera_get_supported_manual_white_balance_values(m_handle.get(), numWhiteBalanceValues, &numWhiteBalanceValues, - manualColorTemperatureValues.data(), - &continuousColorTemperatureSupported); + return camera_has_feature(m_handle.get(), feature); +} - minColorTemperature = 1024*1014; // large enough :) - for (int temp : qAsConst(manualColorTemperatureValues)) { - minColorTemperature = qMin(minColorTemperature, temp); - maxColorTemperature = qMax(maxColorTemperature, temp); - } - } else { - maxColorTemperature = 0; - } +void QQnxCamera::setWhiteBalanceMode(camera_whitebalancemode_t mode) +{ + if (!isActive()) + return; + + if (camera_set_whitebalance_mode(m_handle.get(), mode) != CAMERA_EOK) + qWarning("QQnxCamera: failed to set whitebalance mode"); +} + +camera_whitebalancemode_t QQnxCamera::whiteBalanceMode() const +{ + if (!isActive()) + return CAMERA_WHITEBALANCEMODE_OFF; + + camera_whitebalancemode_t mode; + + if (camera_get_whitebalance_mode(m_handle.get(), &mode) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to get white balance mode"); + return CAMERA_WHITEBALANCEMODE_OFF; } - if (maxColorTemperature != 0) - return true; - return mode == QCamera::WhiteBalanceAuto; + return mode; } -void QQnxCamera::setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) +void QQnxCamera::setManualWhiteBalance(uint32_t value) { - if (mode == QCamera::WhiteBalanceAuto) { - camera_set_whitebalance_mode(m_handle.get(), CAMERA_WHITEBALANCEMODE_AUTO); + if (!isActive()) return; - } - camera_set_whitebalance_mode(m_handle.get(), CAMERA_WHITEBALANCEMODE_MANUAL); - setColorTemperature(colorTemperatureForWhiteBalance(mode)); + + if (camera_set_manual_white_balance(m_handle.get(), value) != CAMERA_EOK) + qWarning("QQnxCamera: failed to set manual white balance"); } -void QQnxCamera::setColorTemperature(int temperature) +uint32_t QQnxCamera::manualWhiteBalance() const { + if (!isActive()) + return 0; - if (maxColorTemperature == 0) - return; + uint32_t value; - unsigned bestTemp = 0; - if (!continuousColorTemperatureSupported) { - // find the closest match - int delta = 1024*1024; - for (unsigned temp : qAsConst(manualColorTemperatureValues)) { - int d = qAbs(int(temp) - temperature); - if (d < delta) { - bestTemp = temp; - delta = d; - } - } - } else { - bestTemp = (unsigned)qBound(minColorTemperature, temperature, maxColorTemperature); + if (camera_get_manual_white_balance(m_handle.get(), &value) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to get manual white balance"); + return 0; } - auto error = camera_set_manual_white_balance(m_handle.get(), bestTemp); + return value; } -void QQnxCamera::startVideoRecording() +bool QQnxCamera::startVideoRecording(const QString &filename) { - const QString container = m_encoderSettings.mimeType().preferredSuffix(); - const QString location = QMediaStorageLocation::generateFileName(m_outputUrl.toLocalFile(), - QStandardPaths::MoviesLocation, container); + // when preview is video, we must ensure that the recording properties + // match the view finder properties + if (hasFeature(CAMERA_FEATURE_PREVIEWISVIDEO)) { + VideoFormat newFormat = vfFormat(); - if (camera_start_video(m_handle.get(), qPrintable(location), - nullptr, nullptr, nullptr) != CAMERA_EOK) { - qWarning("QQnxCamera: failed to start video encoding"); - } else { + const QList<camera_frametype_t> recordingTypes = supportedRecordingFrameTypes(); + + // find a suitable matching frame type in case the current view finder + // frametype is not supported + if (newFormat.frameType != recordingFormat().frameType + && !recordingTypes.contains(newFormat.frameType)) { + + bool found = false; + + for (const camera_frametype_t type : supportedVfFrameTypes()) { + if (recordingTypes.contains(type)) { + newFormat.frameType = type; + found = true; + break; + } + } + + if (found) { + m_originalVfFormat = vfFormat(); + + // reconfigure and restart the view finder + setVfFormat(newFormat); + } else { + qWarning("QQnxCamera: failed to find suitable frame type for recording - aborting"); + return false; + } + } + + setRecordingFormat(newFormat); + } + + if (camera_start_video(m_handle.get(), qPrintable(filename), + nullptr, nullptr, nullptr) == CAMERA_EOK) { m_recordingVideo = true; + } else { + qWarning("QQnxCamera: failed to start video encoding"); } + + return m_recordingVideo; } void QQnxCamera::stopVideoRecording() @@ -467,6 +569,12 @@ void QQnxCamera::stopVideoRecording() if (camera_stop_video(m_handle.get()) != CAMERA_EOK) qWarning("QQnxCamera: error when stopping video recording"); + + // restore original vf format + if (m_originalVfFormat) { + setVfFormat(*m_originalVfFormat); + m_originalVfFormat.reset(); + } } bool QQnxCamera::isVideoEncodingSupported() const @@ -477,47 +585,50 @@ bool QQnxCamera::isVideoEncodingSupported() const return camera_has_feature(m_handle.get(), CAMERA_FEATURE_VIDEO); } -void QQnxCamera::setOutputUrl(const QUrl &url) +camera_handle_t QQnxCamera::handle() const { - m_outputUrl = url; + return m_handle.get(); } -void QQnxCamera::setMediaEncoderSettings(const QMediaEncoderSettings &settings) +void QQnxCamera::updateZoomLimits() { - m_encoderSettings = settings; -} + bool smooth; -camera_handle_t QQnxCamera::handle() const -{ - return m_handle.get(); + if (camera_get_zoom_limits(m_handle.get(), &m_minZoom, &m_maxZoom, &smooth) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to update zoom limits - using default values"); + m_minZoom = m_maxZoom = 0; + } } -void QQnxCamera::updateCameraFeatures() +void QQnxCamera::updateSupportedWhiteBalanceValues() { - whiteBalanceModesChecked = false; + uint32_t numSupported = 0; - bool smooth; - const camera_error_t error = camera_get_zoom_limits(m_handle.get(), - &minZoom, &maxZoom, &smooth); - - if (error == CAMERA_EOK) { - double level; - camera_get_zoom_ratio_from_zoom_level(m_handle.get(), minZoom, &level); - minimumZoomFactorChanged(level); - camera_get_zoom_ratio_from_zoom_level(m_handle.get(), maxZoom, &level); - maximumZoomFactorChanged(level); - } else { - minZoom = maxZoom = 1; + const camera_error_t result = camera_get_supported_manual_white_balance_values( + m_handle.get(), 0, &numSupported, nullptr, &m_continuousWhiteBalanceValues); + + if (result != CAMERA_EOK) { + if (result == CAMERA_EOPNOTSUPP) + qWarning("QQnxCamera: white balance not supported"); + else + qWarning("QQnxCamera: unable to query manual white balance value count"); + + m_supportedWhiteBalanceValues.clear(); + + return; } - QCamera::Features features = {}; + m_supportedWhiteBalanceValues.resize(numSupported); - if (camera_has_feature(m_handle.get(), CAMERA_FEATURE_REGIONFOCUS)) - features |= QCamera::Feature::CustomFocusPoint; + if (camera_get_supported_manual_white_balance_values(m_handle.get(), + m_supportedWhiteBalanceValues.size(), + &numSupported, + m_supportedWhiteBalanceValues.data(), + &m_continuousWhiteBalanceValues) != CAMERA_EOK) { + qWarning("QQnxCamera: unable to query manual white balance values"); - minimumZoomFactorChanged(minZoom); - maximumZoomFactorChanged(maxZoom); - supportedFeaturesChanged(features); + m_supportedWhiteBalanceValues.clear(); + } } QList<camera_vfmode_t> QQnxCamera::supportedVfModes() const @@ -530,19 +641,78 @@ QList<camera_res_t> QQnxCamera::supportedVfResolutions() const return queryValues(camera_get_supported_vf_resolutions); } +QList<camera_frametype_t> QQnxCamera::supportedVfFrameTypes() const +{ + return queryValues(camera_get_supported_vf_frame_types); +} + QList<camera_focusmode_t> QQnxCamera::supportedFocusModes() const { return queryValues(camera_get_focus_modes); } +QList<double> QQnxCamera::specifiedVfFrameRates(camera_frametype_t frameType, + camera_res_t resolution) const +{ + uint32_t numSupported = 0; + + if (camera_get_specified_vf_framerates(m_handle.get(), frameType, resolution, + 0, &numSupported, nullptr, nullptr) != CAMERA_EOK) { + qWarning("QQnxCamera: unable to query specified framerates count"); + return {}; + } + + QList<double> values(numSupported); + + if (camera_get_specified_vf_framerates(m_handle.get(), frameType, resolution, + values.size(), &numSupported, values.data(), nullptr) != CAMERA_EOK) { + qWarning("QQnxCamera: unable to query specified framerates values"); + return {}; + } + + return values; +} + +QList<camera_frametype_t> QQnxCamera::supportedRecordingFrameTypes() const +{ + return queryValues(camera_get_video_frame_types); +} + +QList<uint32_t> QQnxCamera::supportedWhiteBalanceValues() const +{ + return m_supportedWhiteBalanceValues; +} + +bool QQnxCamera::hasContinuousWhiteBalanceValues() const +{ + return m_continuousWhiteBalanceValues; +} + +QList<camera_unit_t> QQnxCamera::supportedUnits() +{ + unsigned int numSupported = 0; + + if (camera_get_supported_cameras(0, &numSupported, nullptr) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to query supported camera unit count"); + return {}; + } + + QList<camera_unit_t> cameraUnits(numSupported); + + if (camera_get_supported_cameras(cameraUnits.size(), &numSupported, + cameraUnits.data()) != CAMERA_EOK) { + qWarning("QQnxCamera: failed to enumerate supported camera units"); + return {}; + } + + return cameraUnits; +} + template <typename T, typename U> QList<T> QQnxCamera::queryValues(QueryFuncPtr<T,U> func) const { static_assert(std::is_integral_v<U>, "Parameter U must be of integral type"); - if (!isActive()) - return {}; - U numSupported = 0; if (func(m_handle.get(), 0, &numSupported, nullptr) != CAMERA_EOK) { @@ -562,9 +732,6 @@ QList<T> QQnxCamera::queryValues(QueryFuncPtr<T,U> func) const void QQnxCamera::handleVfBuffer(camera_buffer_t *buffer) { - if (!m_videoSink) - return; - // process the frame on this thread before locking the mutex auto frame = std::make_unique<QQnxCameraFrameBuffer>(buffer); @@ -573,9 +740,8 @@ void QQnxCamera::handleVfBuffer(camera_buffer_t *buffer) m_currentFrame = std::move(frame); m_currentFrameMutex.unlock(); - QMetaObject::invokeMethod(this, "processFrame", Qt::QueuedConnection); + Q_EMIT frameAvailable(); } - } void QQnxCamera::handleVfStatus(camera_devstatus_t status, uint16_t extraData) @@ -590,34 +756,46 @@ void QQnxCamera::handleStatusChange(camera_devstatus_t status, uint16_t extraDat Q_UNUSED(extraData); switch (status) { - case CAMERA_STATUS_DISCONNECTED: - case CAMERA_STATUS_POWERDOWN: - case CAMERA_STATUS_VIDEOVF: + case CAMERA_STATUS_BUFFER_UNDERFLOW: + case CAMERA_STATUS_CAPTURECOMPLETE: case CAMERA_STATUS_CAPTURE_ABORTED: + case CAMERA_STATUS_CONNECTED: + case CAMERA_STATUS_DISCONNECTED: + case CAMERA_STATUS_FILESIZE_ERROR: + case CAMERA_STATUS_FILESIZE_LIMIT_WARNING: case CAMERA_STATUS_FILESIZE_WARNING: + case CAMERA_STATUS_FLASH_LEVEL_CHANGE: case CAMERA_STATUS_FOCUS_CHANGE: - case CAMERA_STATUS_RESOURCENOTAVAIL: - case CAMERA_STATUS_VIEWFINDER_ERROR: + case CAMERA_STATUS_FRAME_DROPPED: + case CAMERA_STATUS_LOWLIGHT: case CAMERA_STATUS_MM_ERROR: - case CAMERA_STATUS_FILESIZE_ERROR: case CAMERA_STATUS_NOSPACE_ERROR: - case CAMERA_STATUS_BUFFER_UNDERFLOW: - Q_EMIT(status, ::statusToString(status)); - stop(); - break; - default: + case CAMERA_STATUS_PHOTOVF: + case CAMERA_STATUS_POWERDOWN: + case CAMERA_STATUS_POWERUP: + case CAMERA_STATUS_RESOURCENOTAVAIL: + case CAMERA_STATUS_UNKNOWN: + case CAMERA_STATUS_VIDEOLIGHT_CHANGE: + case CAMERA_STATUS_VIDEOLIGHT_LEVEL_CHANGE: + case CAMERA_STATUS_VIDEOVF: + case CAMERA_STATUS_VIDEO_PAUSE: + case CAMERA_STATUS_VIDEO_RESUME: + case CAMERA_STATUS_VIEWFINDER_ACTIVE: + case CAMERA_STATUS_VIEWFINDER_ERROR: + case CAMERA_STATUS_VIEWFINDER_FREEZE: + case CAMERA_STATUS_VIEWFINDER_SUSPEND: + case CAMERA_STATUS_VIEWFINDER_UNFREEZE: + case CAMERA_STATUS_VIEWFINDER_UNSUSPEND: + qDebug() << "QQnxCamera:" << ::statusToString(status); break; } } -void QQnxCamera::processFrame() +std::unique_ptr<QQnxCameraFrameBuffer> QQnxCamera::takeCurrentFrame() { QMutexLocker l(&m_currentFrameMutex); - const QVideoFrame actualFrame(m_currentFrame.get(), - QVideoFrameFormat(m_currentFrame->size(), m_currentFrame->pixelFormat())); - - m_videoSink->setVideoFrame(actualFrame); + return std::move(m_currentFrame); } void QQnxCamera::viewfinderCallback(camera_handle_t handle, camera_buffer_t *buffer, void *arg) @@ -638,3 +816,5 @@ void QQnxCamera::statusCallback(camera_handle_t handle, camera_devstatus_t statu } QT_END_NAMESPACE + +#include "moc_qqnxcamera_p.cpp" diff --git a/src/plugins/multimedia/qnx/camera/qqnxcamera_p.h b/src/plugins/multimedia/qnx/camera/qqnxcamera_p.h index aab112000..a4ddbfed6 100644 --- a/src/plugins/multimedia/qnx/camera/qqnxcamera_p.h +++ b/src/plugins/multimedia/qnx/camera/qqnxcamera_p.h @@ -1,43 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ -#ifndef QQnxCamera_H -#define QQnxCamera_H +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QQNXCAMERA_H +#define QQNXCAMERA_H // // W A R N I N G @@ -50,17 +15,18 @@ // We mean it. // -#include <private/qplatformcamera_p.h> -#include <private/qplatformmediarecorder_p.h> +#include "qqnxcamerahandle_p.h" #include <QtCore/qlist.h> #include <QtCore/qmutex.h> +#include <QtCore/qobject.h> #include <QtCore/qurl.h> #include <camera/camera_api.h> #include <camera/camera_3a.h> #include <memory> +#include <optional> QT_BEGIN_NAMESPACE @@ -68,156 +34,128 @@ class QQnxCameraFrameBuffer; class QQnxMediaCaptureSession; class QQnxVideoSink; -class CameraHandle +class QQnxCamera : public QObject { + Q_OBJECT public: - CameraHandle() = default; - - explicit CameraHandle(camera_handle_t h) - : m_handle (h) {} - - explicit CameraHandle(CameraHandle &&other) - : m_handle(other.m_handle) - , m_lastError(other.m_lastError) - { - other = CameraHandle(); - } - - CameraHandle(const CameraHandle&) = delete; + explicit QQnxCamera(camera_unit_t unit, QObject *parent = nullptr); + ~QQnxCamera(); - CameraHandle& operator=(CameraHandle&& other) - { - m_handle = other.m_handle; - m_lastError = other.m_lastError; + camera_unit_t unit() const; - other = CameraHandle(); + QString name() const; - return *this; - } + bool isValid() const; - ~CameraHandle() - { - close(); - } + bool isActive() const; + void start(); + void stop(); - bool open(camera_unit_t unit, uint32_t mode) - { - if (isOpen()) { - m_lastError = CAMERA_EALREADY; - return false; - } + bool startVideoRecording(const QString &filename); + void stopVideoRecording(); - return cacheError(camera_open, unit, mode, &m_handle); - } + bool setCameraFormat(uint32_t width, uint32_t height, double frameRate); - bool close() - { - if (!isOpen()) - return true; + bool isFocusModeSupported(camera_focusmode_t mode) const; + bool setFocusMode(camera_focusmode_t mode); + camera_focusmode_t focusMode() const; - const bool success = cacheError(camera_close, m_handle); - m_handle = CAMERA_HANDLE_INVALID; + void setCustomFocusPoint(const QPointF &point); - return success; - } + void setManualFocusStep(int step); + int manualFocusStep() const; + int maxFocusStep() const; - camera_handle_t get() const - { - return m_handle; - } + QSize viewFinderSize() const; - bool isOpen() const - { - return m_handle != CAMERA_HANDLE_INVALID; - } + uint32_t minimumZoomLevel() const; + uint32_t maximumZoomLevel() const; + bool isSmoothZoom() const; + double zoomRatio(uint32_t zoomLevel) const; + bool setZoomFactor(uint32_t factor); - camera_error_t lastError() const - { - return m_lastError; - } + void setEvOffset(float ev); -private: - template <typename Func, typename ...Args> - bool cacheError(Func f, Args &&...args) - { - m_lastError = f(std::forward<Args>(args)...); + uint32_t manualIsoSensitivity() const; + void setManualIsoSensitivity(uint32_t value); + void setManualExposureTime(double seconds); + double manualExposureTime() const; - return m_lastError == CAMERA_EOK; - } + void setWhiteBalanceMode(camera_whitebalancemode_t mode); + camera_whitebalancemode_t whiteBalanceMode() const; - camera_handle_t m_handle = CAMERA_HANDLE_INVALID; - camera_error_t m_lastError = CAMERA_EOK; -}; + void setManualWhiteBalance(uint32_t value); + uint32_t manualWhiteBalance() const; + bool hasFeature(camera_feature_t feature) const; -class QQnxCamera : public QPlatformCamera -{ - Q_OBJECT -public: - explicit QQnxCamera(QCamera *parent); - ~QQnxCamera(); + camera_handle_t handle() const; - bool isActive() const override; - void setActive(bool active) override; - void start(); - void stop(); + QList<camera_vfmode_t> supportedVfModes() const; + QList<camera_res_t> supportedVfResolutions() const; + QList<camera_frametype_t> supportedVfFrameTypes() const; + QList<camera_focusmode_t> supportedFocusModes() const; + QList<double> specifiedVfFrameRates(camera_frametype_t frameType, + camera_res_t resolution) const; - void setCamera(const QCameraDevice &camera) override; + QList<camera_frametype_t> supportedRecordingFrameTypes() const; - bool setCameraFormat(const QCameraFormat &format) override; + QList<uint32_t> supportedWhiteBalanceValues() const; - void setCaptureSession(QPlatformMediaCaptureSession *session) override; + bool hasContinuousWhiteBalanceValues() const; - bool isFocusModeSupported(QCamera::FocusMode mode) const override; - void setFocusMode(QCamera::FocusMode mode) override; + static QList<camera_unit_t> supportedUnits(); - void setCustomFocusPoint(const QPointF &point) override; + std::unique_ptr<QQnxCameraFrameBuffer> takeCurrentFrame(); - void setFocusDistance(float distance) override; +Q_SIGNALS: + void focusModeChanged(camera_focusmode_t mode); + void customFocusPointChanged(const QPointF &point); + void minimumZoomFactorChanged(double factor); - int maxFocusDistance() const; + double maximumZoomFactorChanged(double factor); - void zoomTo(float /*newZoomFactor*/, float /*rate*/ = -1.) override; + void frameAvailable(); - void setExposureCompensation(float ev) override; +private: + struct FocusStep + { + int step; // current step + int maxStep; // max supported step + }; - int isoSensitivity() const override; - void setManualIsoSensitivity(int value) override; - void setManualExposureTime(float seconds) override; - float exposureTime() const override; + FocusStep focusStep() const; - bool isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const override; - void setWhiteBalanceMode(QCamera::WhiteBalanceMode /*mode*/) override; - void setColorTemperature(int /*temperature*/) override; + struct VideoFormat + { + uint32_t width; + uint32_t height; + uint32_t rotation; + double frameRate; + camera_frametype_t frameType; + }; - void setOutputUrl(const QUrl &url); - void setMediaEncoderSettings(const QMediaEncoderSettings &settings); + friend QDebug &operator<<(QDebug&, const VideoFormat&); - camera_handle_t handle() const; + VideoFormat vfFormat() const; + void setVfFormat(const VideoFormat &format); - QList<camera_vfmode_t> supportedVfModes() const; - QList<camera_res_t> supportedVfResolutions() const; - QList<camera_focusmode_t> supportedFocusModes() const; + VideoFormat recordingFormat() const; + void setRecordingFormat(const VideoFormat &format); -private: - void updateCameraFeatures(); + void updateZoomLimits(); + void updateSupportedWhiteBalanceValues(); void setColorTemperatureInternal(unsigned temp); - void startVideoRecording(); - void stopVideoRecording(); - bool isVideoEncodingSupported() const; void handleVfBuffer(camera_buffer_t *buffer); - Q_INVOKABLE void processFrame(); - // viewfinder callback void handleVfStatus(camera_devstatus_t status, uint16_t extraData); // our handler running on main thread - void handleStatusChange(camera_devstatus_t status, uint16_t extraData); - + Q_INVOKABLE void handleStatusChange(camera_devstatus_t status, uint16_t extraData); template <typename T, typename U> using QueryFuncPtr = camera_error_t (*)(camera_handle_t, U, U *, T *); @@ -225,9 +163,6 @@ private: template <typename T, typename U> QList<T> queryValues(QueryFuncPtr<T, U> func) const; - template <typename ...Args> - using CamAPIFunc = camera_error_t (*)(camera_handle_t, Args...); - static void viewfinderCallback(camera_handle_t handle, camera_buffer_t *buffer, void *arg); @@ -235,35 +170,32 @@ private: uint16_t extraData, void *arg); QQnxMediaCaptureSession *m_session = nullptr; - QQnxVideoSink *m_videoSink = nullptr; - QCameraDevice m_camera; camera_unit_t m_cameraUnit = CAMERA_UNIT_NONE; - QUrl m_outputUrl; - - QMediaEncoderSettings m_encoderSettings; - - CameraHandle m_handle; + QQnxCameraHandle m_handle; - uint minZoom = 1; - uint maxZoom = 1; - mutable bool whiteBalanceModesChecked = false; - mutable bool continuousColorTemperatureSupported = false; - mutable int minColorTemperature = 0; - mutable int maxColorTemperature = 0; - mutable QList<unsigned> manualColorTemperatureValues; + uint32_t m_minZoom = 0; + uint32_t m_maxZoom = 0; QMutex m_currentFrameMutex; + QList<uint32_t> m_supportedWhiteBalanceValues; + std::unique_ptr<QQnxCameraFrameBuffer> m_currentFrame; + std::optional<VideoFormat> m_originalVfFormat; + bool m_viewfinderActive = false; bool m_recordingVideo = false; + bool m_valid = false; + bool m_smoothZoom = false; + bool m_continuousWhiteBalanceValues = false; }; QT_END_NAMESPACE Q_DECLARE_METATYPE(camera_devstatus_t) +Q_DECLARE_METATYPE(uint16_t) #endif diff --git a/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer.cpp b/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer.cpp index ba7c7b17f..6595c5d42 100644 --- a/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer.cpp +++ b/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer.cpp @@ -1,46 +1,31 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxcameraframebuffer_p.h" #include <limits> +template <typename T> +static constexpr int toInt(T value) +{ + if constexpr (sizeof(T) >= sizeof(int)) { + if (std::is_signed_v<T>) { + return static_cast<int>(std::clamp<T>(value, + std::numeric_limits<int>::min(), std::numeric_limits<int>::max())); + } else { + return static_cast<int>(std::min<T>(value, std::numeric_limits<int>::max())); + } + } else { + return static_cast<int>(value); + } +} + +template <typename T> +static constexpr QSize frameSize(const T &frame) +{ + return { toInt(frame.width), toInt(frame.height) }; +} + static constexpr QVideoFrameFormat::PixelFormat frameTypeToPixelFormat(camera_frametype_t type) { switch (type) { @@ -120,18 +105,18 @@ static QAbstractVideoBuffer::MapData mapData(const camera_frame_nv12_t &frame, { return { - .nPlanes = 2, + .planeCount = 2, .bytesPerLine = { - frame.stride, - frame.uv_stride + toInt(frame.stride), + toInt(frame.uv_stride) }, .data = { baseAddress, baseAddress + frame.uv_offset }, - .size = { - frame.stride * frame.height, - frame.uv_stride * frame.height / 2 + .dataSize = { + toInt(frame.stride * frame.height), + toInt(frame.uv_stride * frame.height / 2) } }; } @@ -140,15 +125,15 @@ static QAbstractVideoBuffer::MapData mapData(const camera_frame_rgb8888_t &frame unsigned char *baseAddress) { return { - .nPlanes = 1, + .planeCount = 1, .bytesPerLine = { - frame.stride + toInt(frame.stride) }, .data = { baseAddress }, - .size = { - frame.stride * frame.height, + .dataSize = { + toInt(frame.stride * frame.height), } }; } @@ -157,15 +142,15 @@ static QAbstractVideoBuffer::MapData mapData(const camera_frame_gray8_t &frame, unsigned char *baseAddress) { return { - .nPlanes = 1, + .planeCount = 1, .bytesPerLine = { - frame.stride + toInt(frame.stride) }, .data = { baseAddress }, - .size = { - frame.stride * frame.height + .dataSize = { + toInt(frame.stride * frame.height) } }; } @@ -174,15 +159,15 @@ static QAbstractVideoBuffer::MapData mapData(const camera_frame_cbycry_t &frame, unsigned char *baseAddress) { return { - .nPlanes = 1, + .planeCount = 1, .bytesPerLine = { - frame.stride + toInt(frame.stride) }, .data = { baseAddress }, - .size = { - frame.bufsize, + .dataSize = { + toInt(frame.bufsize), } }; } @@ -191,9 +176,9 @@ static QAbstractVideoBuffer::MapData mapData(const camera_frame_ycbcr420p_t &fra unsigned char *baseAddress) { return { - .nPlanes = 3, + .planeCount = 3, .bytesPerLine = { - frame.y_stride, + toInt(frame.y_stride), frame.cb_stride, frame.cr_stride, }, @@ -202,10 +187,10 @@ static QAbstractVideoBuffer::MapData mapData(const camera_frame_ycbcr420p_t &fra baseAddress + frame.cb_offset, baseAddress + frame.cr_offset, }, - .size = { - frame.y_stride * frame.height, - frame.cb_stride * frame.height / 2, - frame.cr_stride * frame.height / 2 + .dataSize = { + toInt(frame.y_stride * frame.height), + toInt(frame.cb_stride * frame.height / 2), + toInt(frame.cr_stride * frame.height / 2) } }; } @@ -214,15 +199,15 @@ static QAbstractVideoBuffer::MapData mapData(const camera_frame_ycbycr_t &frame, unsigned char *baseAddress) { return { - .nPlanes = 1, + .planeCount = 1, .bytesPerLine = { - frame.stride + toInt(frame.stride) }, .data = { baseAddress }, - .size = { - frame.stride * frame.height + .dataSize = { + toInt(frame.stride * frame.height) } }; } @@ -250,17 +235,6 @@ static QAbstractVideoBuffer::MapData mapData(const camera_buffer_t *buffer, return {}; } -static constexpr int toInt(uint32_t value) -{ - return static_cast<int>(std::min<uint32_t>(value, std::numeric_limits<int>::max())); -} - -template <typename T> -static constexpr QSize frameSize(const T &frame) -{ - return { toInt(frame.width), toInt(frame.height) }; -} - static constexpr QSize frameSize(const camera_buffer_t *buffer) { switch (buffer->frametype) { @@ -286,10 +260,10 @@ static constexpr QSize frameSize(const camera_buffer_t *buffer) QT_BEGIN_NAMESPACE QQnxCameraFrameBuffer::QQnxCameraFrameBuffer(const camera_buffer_t *buffer, QRhi *rhi) - : QAbstractVideoBuffer(rhi ? QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, rhi) - , m_rhi(rhi) - , m_pixelFormat(::frameTypeToPixelFormat(buffer->frametype)) - , m_dataSize(::bufferDataSize(buffer)) + : QHwVideoBuffer(rhi ? QVideoFrame::RhiTextureHandle : QVideoFrame::NoHandle, rhi), + m_rhi(rhi), + m_pixelFormat(::frameTypeToPixelFormat(buffer->frametype)), + m_dataSize(::bufferDataSize(buffer)) { if (m_dataSize <= 0) return; @@ -303,12 +277,7 @@ QQnxCameraFrameBuffer::QQnxCameraFrameBuffer(const camera_buffer_t *buffer, QRhi m_frameSize = ::frameSize(buffer); } -QVideoFrame::MapMode QQnxCameraFrameBuffer::mapMode() const -{ - return QVideoFrame::ReadOnly; -} - -QAbstractVideoBuffer::MapData QQnxCameraFrameBuffer::map(QVideoFrame::MapMode) +QAbstractVideoBuffer::MapData QQnxCameraFrameBuffer::map(QtVideo::MapMode) { return m_mapData; } diff --git a/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer_p.h b/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer_p.h index 7ff603dc0..20f724552 100644 --- a/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer_p.h +++ b/src/plugins/multimedia/qnx/camera/qqnxcameraframebuffer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXCAMERAFRAMEBUFFER_H #define QQNXCAMERAFRAMEBUFFER_H @@ -50,7 +14,7 @@ // We mean it. // -#include <private/qabstractvideobuffer_p.h> +#include <private/qhwvideobuffer_p.h> #include <QtCore/qsize.h> @@ -62,7 +26,7 @@ QT_BEGIN_NAMESPACE class QRhi; -class QQnxCameraFrameBuffer : public QAbstractVideoBuffer +class QQnxCameraFrameBuffer : public QHwVideoBuffer { public: explicit QQnxCameraFrameBuffer(const camera_buffer_t *buffer, QRhi *rhi = nullptr); @@ -70,18 +34,9 @@ public: QQnxCameraFrameBuffer(const QQnxCameraFrameBuffer&) = delete; QQnxCameraFrameBuffer& operator=(const QQnxCameraFrameBuffer&) = delete; - QVideoFrame::MapMode mapMode() const override; - MapData map(QVideoFrame::MapMode mode) override; + MapData map(QtVideo::MapMode mode) override; void unmap() override; -#if 0 - virtual void mapTextures() {} - virtual quint64 textureHandle(int /*plane*/) const { return 0; } - virtual std::unique_ptr<QRhiTexture> texture(int /*plane*/) const; - - virtual QMatrix4x4 externalTextureMatrix() const { return {}; } -#endif - QVideoFrameFormat::PixelFormat pixelFormat() const; QSize size() const; diff --git a/src/plugins/multimedia/qnx/camera/qqnxcamerahandle_p.h b/src/plugins/multimedia/qnx/camera/qqnxcamerahandle_p.h new file mode 100644 index 000000000..3d7863dc2 --- /dev/null +++ b/src/plugins/multimedia/qnx/camera/qqnxcamerahandle_p.h @@ -0,0 +1,102 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QQNXCAMERAHANDLE_P_H +#define QQNXCAMERAHANDLE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <camera/camera_api.h> + +#include <utility> + +class QQnxCameraHandle +{ +public: + QQnxCameraHandle() = default; + + explicit QQnxCameraHandle(camera_handle_t h) + : m_handle (h) {} + + explicit QQnxCameraHandle(QQnxCameraHandle &&other) + : m_handle(other.m_handle) + , m_lastError(other.m_lastError) + { + other = QQnxCameraHandle(); + } + + QQnxCameraHandle(const QQnxCameraHandle&) = delete; + + QQnxCameraHandle& operator=(QQnxCameraHandle&& other) + { + m_handle = other.m_handle; + m_lastError = other.m_lastError; + + other = QQnxCameraHandle(); + + return *this; + } + + ~QQnxCameraHandle() + { + close(); + } + + bool open(camera_unit_t unit, uint32_t mode) + { + if (isOpen()) { + m_lastError = CAMERA_EALREADY; + return false; + } + + return cacheError(camera_open, unit, mode, &m_handle); + } + + bool close() + { + if (!isOpen()) + return true; + + const bool success = cacheError(camera_close, m_handle); + m_handle = CAMERA_HANDLE_INVALID; + + return success; + } + + camera_handle_t get() const + { + return m_handle; + } + + bool isOpen() const + { + return m_handle != CAMERA_HANDLE_INVALID; + } + + camera_error_t lastError() const + { + return m_lastError; + } + +private: + template <typename Func, typename ...Args> + bool cacheError(Func f, Args &&...args) + { + m_lastError = f(std::forward<Args>(args)...); + + return m_lastError == CAMERA_EOK; + } + + camera_handle_t m_handle = CAMERA_HANDLE_INVALID; + camera_error_t m_lastError = CAMERA_EOK; +}; + +#endif diff --git a/src/plugins/multimedia/qnx/camera/qqnximagecapture.cpp b/src/plugins/multimedia/qnx/camera/qqnximagecapture.cpp index 6557267b2..3983dddbb 100644 --- a/src/plugins/multimedia/qnx/camera/qqnximagecapture.cpp +++ b/src/plugins/multimedia/qnx/camera/qqnximagecapture.cpp @@ -1,49 +1,60 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnximagecapture_p.h" +#include "qqnxplatformcamera_p.h" #include "qqnxmediacapturesession_p.h" #include "qqnxcamera_p.h" #include "qfile.h" +#include <private/qmediastoragelocation_p.h> + +#include <QtCore/qfileinfo.h> +#include <QtCore/qfuture.h> +#include <QtCore/qpromise.h> +#include <QtCore/qthread.h> + #include <camera/camera_api.h> +using namespace Qt::Literals::StringLiterals; + +static QString formatExtension(QImageCapture::FileFormat format) +{ + switch (format) { + case QImageCapture::JPEG: + return u"jpg"_s; + case QImageCapture::PNG: + return u"png"_s; + case QImageCapture::WebP: + case QImageCapture::Tiff: + case QImageCapture::UnspecifiedFormat: + break; + } + + return {}; +} + +static QString resolveFileName(const QString &fileName, QImageCapture::FileFormat format) +{ + const QString extension = formatExtension(format); + + if (extension.isEmpty()) + return {}; + + if (fileName.isEmpty()) { + return QMediaStorageLocation::generateFileName(QString(), + QStandardPaths::PicturesLocation, extension); + } + + if (fileName.endsWith(extension)) + return QFileInfo(fileName).canonicalFilePath(); + + QString path = fileName; + path.append(u".%1"_s.arg(extension)); + + return QFileInfo(path).canonicalFilePath(); +} + QT_BEGIN_NAMESPACE QQnxImageCapture::QQnxImageCapture(QImageCapture *parent) @@ -53,131 +64,130 @@ QQnxImageCapture::QQnxImageCapture(QImageCapture *parent) bool QQnxImageCapture::isReadyForCapture() const { - if (!m_session) - return false; - auto *camera = static_cast<QQnxCamera *>(m_session->camera()); - // ### add can take photo - return camera && camera->isActive(); + return m_camera && m_camera->isActive(); } -#if 0 -static void imageCaptureShutterCallback(camera_handle_t handle, void *context) +int QQnxImageCapture::capture(const QString &fileName) { - Q_UNUSED(handle); + if (!isReadyForCapture()) { + Q_EMIT error(-1, QImageCapture::NotReadyError, QPlatformImageCapture::msgCameraNotReady()); + return -1; + } - const auto *data = static_cast<QQnxImageCapture::PendingImage *>(context); + // default to PNG format if no format has been specified + const QImageCapture::FileFormat format = + m_settings.format() == QImageCapture::UnspecifiedFormat + ? QImageCapture::PNG : m_settings.format(); - // We are inside a worker thread here, so emit imageExposed inside the main thread - QMetaObject::invokeMethod(data->imageCapture, "imageExposed", Qt::QueuedConnection, Q_ARG(int, data->id)); + const QString resolvedFileName = resolveFileName(fileName, format); + + if (resolvedFileName.isEmpty()) { + const QString errorMessage = (u"Invalid file format: %1"_s).arg( + QImageCapture::fileFormatName(format)); + + Q_EMIT error(-1, QImageCapture::NotSupportedFeatureError, errorMessage); + return -1; + } + + const int id = m_lastId++; + + auto callback = [this, id, fn=std::move(resolvedFileName)](const QVideoFrame &frame) { + saveFrame(id, frame, fn); + }; + + m_camera->requestVideoFrame(std::move(callback)); + + return id; } -static void imageCaptureImageCallback(camera_handle_t handle, camera_buffer_t *buffer, void *context) +int QQnxImageCapture::captureToBuffer() { - Q_UNUSED(handle); + if (!isReadyForCapture()) { + Q_EMIT error(-1, QImageCapture::NotReadyError, QPlatformImageCapture::msgCameraNotReady()); + return -1; + } - auto *data = static_cast<QQnxImageCapture::PendingImage *>(context); + const int id = m_lastId++; - if (buffer->frametype != CAMERA_FRAMETYPE_JPEG) { - // ### Fix this, we can support other formats as well! + auto callback = [this, id](const QVideoFrame &frame) { decodeFrame(id, frame); }; - // We are inside a worker thread here, so emit error signal inside the main thread - QMetaObject::invokeMethod(data->imageCapture, "error", Qt::QueuedConnection, - Q_ARG(int, data->id), - Q_ARG(QImageCapture::Error, QImageCapture::FormatError), - Q_ARG(QString, QCamera::tr("Camera provides image in unsupported format"))); - return; - } + m_camera->requestVideoFrame(std::move(callback)); - const QByteArray rawData = QByteArray::fromRawData((const char*)buffer->framebuf, buffer->framedesc.jpeg.bufsize); - - QImage image; - const bool ok = image.loadFromData(rawData, "JPG"); - if (!ok) { - const QString errorMessage = QCamera::tr("Could not load JPEG data from frame"); - // We are inside a worker thread here, so emit error signal inside the main thread - QMetaObject::invokeMethod(data->imageCapture, "error", Qt::QueuedConnection, - Q_ARG(int, data->id), - Q_ARG(QImageCapture::Error, QImageCapture::FormatError), - Q_ARG(QString, errorMessage)); - return; + return id; +} + +QFuture<QImage> QQnxImageCapture::decodeFrame(int id, const QVideoFrame &frame) +{ + if (!frame.isValid()) { + Q_EMIT error(id, QImageCapture::NotReadyError, u"Invalid frame"_s); + return {}; } + QPromise<QImage> promise; + QFuture<QImage> future = promise.future(); - // We are inside a worker thread here, so invoke imageCaptured inside the main thread - QMetaObject::invokeMethod(data->imageCapture, "imageCaptured", Qt::QueuedConnection, - Q_ARG(int, data->id), - Q_ARG(QImage, image)); - - if (!data->filename.isEmpty()) { - QFile file(data->filename); - if (file.exists()) { - const QString errorMessage = QCamera::tr("Could not overwrite existing file"); - // We are inside a worker thread here, so emit error signal inside the main thread - QMetaObject::invokeMethod(data->imageCapture, "error", Qt::QueuedConnection, - Q_ARG(int, data->id), - Q_ARG(QImageCapture::Error, QImageCapture::ResourceError), - Q_ARG(QString, errorMessage)); - return; - } - if (!file.open(QFile::WriteOnly)) { - const QString errorMessage = QCamera::tr("Could not write image to file"); - // We are inside a worker thread here, so emit error signal inside the main thread - QMetaObject::invokeMethod(data->imageCapture, "error", Qt::QueuedConnection, - Q_ARG(int, data->id), - Q_ARG(QImageCapture::Error, QImageCapture::ResourceError), - Q_ARG(QString, errorMessage)); - return; + // converting a QVideoFrame to QImage is an expensive operation + // run it on a background thread to prevent it from stalling the UI + auto runner = [frame, promise=std::move(promise)]() mutable { + promise.start(); + promise.addResult(frame.toImage()); + promise.finish(); + }; + + auto *worker = QThread::create(std::move(runner)); + + auto onFinished = [this, worker, id, future]() mutable { + worker->deleteLater(); + + if (future.isValid()) { + Q_EMIT imageCaptured(id, future.result()); + } else { + qWarning("QQnxImageCapture: failed to capture image to buffer"); } - file.write(rawData); - file.close(); - QMetaObject::invokeMethod(data->imageCapture, "imageSaved", Qt::QueuedConnection, - Q_ARG(int, data->id), - Q_ARG(QString, data->filename)); - } - delete data; -} -#endif + }; + connect(worker, &QThread::finished, this, std::move(onFinished)); -int QQnxImageCapture::capture(const QString &fileName) -{ - auto *camera = static_cast<QQnxCamera *>(m_session->camera()); - // ### add can take photo - if (!camera || !camera->isActive()) { - emit error(-1, QImageCapture::NotReadyError, QPlatformImageCapture::msgCameraNotReady()); - return -1; - } + Q_EMIT imageExposed(id); - // prepare context object for callback - PendingImage *pending = new PendingImage; - pending->id = m_lastId; - pending->filename = fileName; - pending->imageCapture = this; - m_lastId++; - -#if 0 - // ### camera_take_photo doesn't exist anymore afaict - const camera_error_t result = camera_take_photo(camera->handle(), - imageCaptureShutterCallback, - 0, - 0, - imageCaptureImageCallback, - pending, false); - - if (result != CAMERA_EOK) { - qWarning() << "Unable to take photo:" << result; - emit error(-1, QImageCapture::NotReadyError, QPlatformImageCapture::msgCameraNotReady()); - return -1; - } -#endif + worker->start(); - return m_lastId; + return future; } -int QQnxImageCapture::captureToBuffer() +void QQnxImageCapture::saveFrame(int id, const QVideoFrame &frame, const QString &fileName) { - // ### implement me - return -1; + QFuture<QImage> decodeFuture = decodeFrame(id, frame); + + if (decodeFuture.isCanceled()) + return; + + QPromise<bool> promise; + QFuture<bool> saveFuture = promise.future(); + + // writing a QImage to disk is a _very_ expensive operation + // run it on a background thread to prevent it from stalling the UI + auto runner = [future=std::move(decodeFuture), + promise=std::move(promise), fileName]() mutable { + promise.start(); + promise.addResult(future.result().save(fileName)); + promise.finish(); + }; + + auto *worker = QThread::create(std::move(runner)); + + auto onFinished = [this, worker, id, future=std::move(saveFuture), fn=std::move(fileName)]() { + worker->deleteLater(); + + if (future.isValid() && future.result()) + Q_EMIT imageSaved(id, fn); + else + Q_EMIT error(id, QImageCapture::NotSupportedFeatureError, u"Failed to save image"_s); + }; + + connect(worker, &QThread::finished, this, std::move(onFinished)); + + worker->start(); } QImageEncoderSettings QQnxImageCapture::imageSettings() const @@ -195,22 +205,53 @@ void QQnxImageCapture::setCaptureSession(QQnxMediaCaptureSession *captureSession if (m_session == captureSession) return; - bool oldReadyForCapture = isReadyForCapture(); + if (m_session) + m_session->disconnect(this); + + m_session = captureSession; + if (m_session) { - disconnect(m_session, nullptr, this, nullptr); + connect(m_session, &QQnxMediaCaptureSession::cameraChanged, + this, &QQnxImageCapture::onCameraChanged); } - m_session = captureSession; + onCameraChanged(); +} - bool readyForCapture = isReadyForCapture(); - if (readyForCapture != oldReadyForCapture) - emit readyForCaptureChanged(readyForCapture); +void QQnxImageCapture::onCameraChanged() +{ + if (m_camera) + m_camera->disconnect(this); + + m_camera = m_session ? static_cast<QQnxPlatformCamera*>(m_session->camera()) : nullptr; - if (!m_session) + if (m_camera) { + connect(m_camera, &QQnxPlatformCamera::activeChanged, + this, &QQnxImageCapture::onCameraChanged); + } + + updateReadyForCapture(); +} + +void QQnxImageCapture::onCameraActiveChanged(bool active) +{ + Q_UNUSED(active); + + updateReadyForCapture(); +} + +void QQnxImageCapture::updateReadyForCapture() +{ + const bool readyForCapture = isReadyForCapture(); + + if (m_lastReadyForCapture == readyForCapture) return; -// connect(m_session, &QPlatformMediaCaptureSession::cameraChanged, this, &QGstreamerImageCapture::onCameraChanged); -// onCameraChanged(); + m_lastReadyForCapture = readyForCapture; + + Q_EMIT readyForCaptureChanged(m_lastReadyForCapture); } QT_END_NAMESPACE + +#include "moc_qqnximagecapture_p.cpp" diff --git a/src/plugins/multimedia/qnx/camera/qqnximagecapture_p.h b/src/plugins/multimedia/qnx/camera/qqnximagecapture_p.h index b4cc13057..832039654 100644 --- a/src/plugins/multimedia/qnx/camera/qqnximagecapture_p.h +++ b/src/plugins/multimedia/qnx/camera/qqnximagecapture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQnxImageCapture_H #define QQnxImageCapture_H @@ -51,11 +15,15 @@ // #include <private/qplatformimagecapture_p.h> -#include <qqueue.h> + +#include <QtCore/qfuture.h> QT_BEGIN_NAMESPACE class QQnxMediaCaptureSession; +class QQnxPlatformCamera; + +class QThread; class QQnxImageCapture : public QPlatformImageCapture { @@ -73,18 +41,21 @@ public: void setCaptureSession(QQnxMediaCaptureSession *session); - struct PendingImage { - int id; - QString filename; - QMediaMetaData metaData; - QQnxImageCapture *imageCapture; - }; - private: + QFuture<QImage> decodeFrame(int id, const QVideoFrame &frame); + void saveFrame(int id, const QVideoFrame &frame, const QString &fileName); + + void onCameraChanged(); + void onCameraActiveChanged(bool active); + void updateReadyForCapture(); + QQnxMediaCaptureSession *m_session = nullptr; + QQnxPlatformCamera *m_camera = nullptr; int m_lastId = 0; QImageEncoderSettings m_settings; + + bool m_lastReadyForCapture = false; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/qnx/camera/qqnxplatformcamera.cpp b/src/plugins/multimedia/qnx/camera/qqnxplatformcamera.cpp new file mode 100644 index 000000000..b604f4561 --- /dev/null +++ b/src/plugins/multimedia/qnx/camera/qqnxplatformcamera.cpp @@ -0,0 +1,426 @@ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#undef QT_NO_CONTEXTLESS_CONNECT // Remove after porting connect() calls + +#include "qqnxplatformcamera_p.h" +#include "qqnxcameraframebuffer_p.h" +#include "qqnxmediacapturesession_p.h" +#include "qqnxvideosink_p.h" + +#include <qcameradevice.h> +#include <qmediadevices.h> + +#include <private/qmediastoragelocation_p.h> +#include <private/qvideoframe_p.h> + +#include <camera/camera_api.h> +#include <camera/camera_3a.h> + +#include <algorithm> +#include <array> + +#include <dlfcn.h> + +struct FocusModeMapping +{ + QCamera::FocusMode qt; + camera_focusmode_t qnx; +}; + +constexpr std::array<FocusModeMapping, 6> focusModes {{ + { QCamera::FocusModeAuto, CAMERA_FOCUSMODE_CONTINUOUS_AUTO }, + { QCamera::FocusModeAutoFar, CAMERA_FOCUSMODE_CONTINUOUS_AUTO }, + { QCamera::FocusModeInfinity, CAMERA_FOCUSMODE_CONTINUOUS_AUTO }, + { QCamera::FocusModeAutoNear, CAMERA_FOCUSMODE_CONTINUOUS_MACRO }, + { QCamera::FocusModeHyperfocal, CAMERA_FOCUSMODE_EDOF }, + { QCamera::FocusModeManual, CAMERA_FOCUSMODE_MANUAL }, +}}; + +template <typename Mapping, typename From, typename To, size_t N> +static constexpr To convert(const std::array<Mapping, N> &mapping, + From Mapping::* from, To Mapping::* to, From value, To defaultValue) +{ + for (const Mapping &m : mapping) { + const auto fromValue = m.*from; + const auto toValue = m.*to; + + if (value == fromValue) + return toValue; + } + + return defaultValue; +} + +static constexpr camera_focusmode_t qnxFocusMode(QCamera::FocusMode mode) +{ + return convert(focusModes, &FocusModeMapping::qt, + &FocusModeMapping::qnx, mode, CAMERA_FOCUSMODE_CONTINUOUS_AUTO); +} + +static constexpr QCamera::FocusMode qtFocusMode(camera_focusmode_t mode) +{ + return convert(focusModes, &FocusModeMapping::qnx, + &FocusModeMapping::qt, mode, QCamera::FocusModeAuto); +} + +QT_BEGIN_NAMESPACE + +QQnxPlatformCamera::QQnxPlatformCamera(QCamera *parent) + : QPlatformCamera(parent) +{ + if (parent) + setCamera(parent->cameraDevice()); + else + setCamera(QMediaDevices::defaultVideoInput()); +} + +QQnxPlatformCamera::~QQnxPlatformCamera() +{ + stop(); +} + +bool QQnxPlatformCamera::isActive() const +{ + return m_qnxCamera && m_qnxCamera->isActive(); +} + +void QQnxPlatformCamera::setActive(bool active) +{ + if (active) + start(); + else + stop(); +} + +void QQnxPlatformCamera::start() +{ + if (!m_qnxCamera || isActive()) + return; + + if (m_session) + m_videoSink = m_session->videoSink(); + + m_qnxCamera->start(); + + Q_EMIT activeChanged(true); +} + +void QQnxPlatformCamera::stop() +{ + if (!m_qnxCamera) + return; + + m_qnxCamera->stop(); + + m_videoSink = nullptr; + + Q_EMIT activeChanged(false); +} + +void QQnxPlatformCamera::setCamera(const QCameraDevice &camera) +{ + if (m_cameraDevice == camera) + return; + + const auto cameraUnit = static_cast<camera_unit_t>(camera.id().toUInt()); + + m_qnxCamera = std::make_unique<QQnxCamera>(cameraUnit); + + connect(m_qnxCamera.get(), &QQnxCamera::focusModeChanged, + [this](camera_focusmode_t mode) { Q_EMIT focusModeChanged(qtFocusMode(mode)); }); + connect(m_qnxCamera.get(), &QQnxCamera::customFocusPointChanged, + this, &QQnxPlatformCamera::customFocusPointChanged); + connect(m_qnxCamera.get(), &QQnxCamera::frameAvailable, + this, &QQnxPlatformCamera::onFrameAvailable, Qt::QueuedConnection); + + m_cameraDevice = camera; + + updateCameraFeatures(); +} + +bool QQnxPlatformCamera::setCameraFormat(const QCameraFormat &format) +{ + const QSize resolution = format.resolution(); + + if (resolution.isEmpty()) { + qWarning("QQnxPlatformCamera: invalid resolution requested"); + return false; + } + + return m_qnxCamera->setCameraFormat(resolution.width(), + resolution.height(), format.maxFrameRate()); +} + +void QQnxPlatformCamera::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + if (m_session == session) + return; + + m_session = static_cast<QQnxMediaCaptureSession *>(session); +} + +bool QQnxPlatformCamera::isFocusModeSupported(QCamera::FocusMode mode) const +{ + if (!m_qnxCamera) + return false; + + return m_qnxCamera->supportedFocusModes().contains(::qnxFocusMode(mode)); +} + +void QQnxPlatformCamera::setFocusMode(QCamera::FocusMode mode) +{ + if (!m_qnxCamera) + return; + + m_qnxCamera->setFocusMode(::qnxFocusMode(mode)); +} + +void QQnxPlatformCamera::setCustomFocusPoint(const QPointF &point) +{ + if (!m_qnxCamera) + return; + + m_qnxCamera->setCustomFocusPoint(point); +} + +void QQnxPlatformCamera::setFocusDistance(float distance) +{ + if (!m_qnxCamera) + return; + + const int maxDistance = m_qnxCamera->maxFocusStep(); + + if (maxDistance < 0) + return; + + const int qnxDistance = maxDistance * std::min(distance, 1.0f); + + m_qnxCamera->setManualFocusStep(qnxDistance); +} + +void QQnxPlatformCamera::zoomTo(float factor, float) +{ + if (!m_qnxCamera) + return; + + const uint32_t minZoom = m_qnxCamera->minimumZoomLevel(); + const uint32_t maxZoom = m_qnxCamera->maximumZoomLevel(); + + if (maxZoom <= minZoom) + return; + + // QNX has an integer based API. Interpolate between the levels according to the factor we get + const float max = maxZoomFactor(); + const float min = minZoomFactor(); + + if (max <= min) + return; + + factor = qBound(min, factor, max) - min; + + const uint32_t zoom = minZoom + + static_cast<uint32_t>(qRound(factor*(maxZoom - minZoom)/(max - min))); + + if (m_qnxCamera->setZoomFactor(zoom)) + zoomFactorChanged(factor); +} + +void QQnxPlatformCamera::setExposureCompensation(float ev) +{ + if (!m_qnxCamera) + return; + + m_qnxCamera->setEvOffset(ev); +} + +int QQnxPlatformCamera::isoSensitivity() const +{ + if (!m_qnxCamera) + return 0; + + return m_qnxCamera->manualIsoSensitivity(); +} + +void QQnxPlatformCamera::setManualIsoSensitivity(int value) +{ + if (!m_qnxCamera) + return; + + const uint32_t isoValue = std::max(0, value); + + m_qnxCamera->setManualIsoSensitivity(isoValue); +} + +void QQnxPlatformCamera::setManualExposureTime(float seconds) +{ + if (!m_qnxCamera) + return; + + m_qnxCamera->setManualExposureTime(seconds); +} + +float QQnxPlatformCamera::exposureTime() const +{ + if (!m_qnxCamera) + return 0.0; + + return static_cast<float>(m_qnxCamera->manualExposureTime()); +} + +bool QQnxPlatformCamera::isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const +{ + if (m_maxColorTemperature != 0) + return true; + + return mode == QCamera::WhiteBalanceAuto; +} + +void QQnxPlatformCamera::setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) +{ + if (!m_qnxCamera) + return; + + if (mode == QCamera::WhiteBalanceAuto) { + m_qnxCamera->setWhiteBalanceMode(CAMERA_WHITEBALANCEMODE_AUTO); + } else { + m_qnxCamera->setWhiteBalanceMode(CAMERA_WHITEBALANCEMODE_MANUAL); + setColorTemperature(colorTemperatureForWhiteBalance(mode)); + } +} + +void QQnxPlatformCamera::setColorTemperature(int temperature) +{ + if (!m_qnxCamera) + return; + + const auto normalizedTemp = std::clamp<uint32_t>(std::max(0, temperature), + m_minColorTemperature, m_maxColorTemperature); + + if (m_qnxCamera->hasContinuousWhiteBalanceValues()) { + m_qnxCamera->setManualWhiteBalance(normalizedTemp); + } else { + uint32_t delta = std::numeric_limits<uint32_t>::max(); + uint32_t closestTemp = 0; + + for (uint32_t value : m_qnxCamera->supportedWhiteBalanceValues()) { + const auto &[min, max] = std::minmax(value, normalizedTemp); + const uint32_t currentDelta = max - min; + + if (currentDelta < delta) { + closestTemp = value; + delta = currentDelta; + } + } + + m_qnxCamera->setManualWhiteBalance(closestTemp); + } +} + +bool QQnxPlatformCamera::startVideoRecording() +{ + if (!m_qnxCamera) { + qWarning("QQnxPlatformCamera: cannot start video recording - no no camera assigned"); + return false; + } + + if (!isVideoEncodingSupported()) { + qWarning("QQnxPlatformCamera: cannot start video recording - not supported"); + return false; + } + + if (!m_qnxCamera->isActive()) { + qWarning("QQnxPlatformCamera: cannot start video recording - camera not started"); + return false; + } + + const QString container = m_encoderSettings.mimeType().preferredSuffix(); + const QString location = QMediaStorageLocation::generateFileName(m_outputUrl.toLocalFile(), + QStandardPaths::MoviesLocation, container); + +#if 0 + { + static void *libScreen = nullptr; + + if (!libScreen) + libScreen = dlopen("/usr/lib/libscreen.so.1", RTLD_GLOBAL); + } +#endif + + qDebug() << "Recording to" << location; + return m_qnxCamera->startVideoRecording(location); +} + +void QQnxPlatformCamera::requestVideoFrame(VideoFrameCallback cb) +{ + m_videoFrameRequests.emplace_back(std::move(cb)); +} + +bool QQnxPlatformCamera::isVideoEncodingSupported() const +{ + return m_qnxCamera && m_qnxCamera->hasFeature(CAMERA_FEATURE_VIDEO); +} + +void QQnxPlatformCamera::setOutputUrl(const QUrl &url) +{ + m_outputUrl = url; +} + +void QQnxPlatformCamera::setMediaEncoderSettings(const QMediaEncoderSettings &settings) +{ + m_encoderSettings = settings; +} + +void QQnxPlatformCamera::updateCameraFeatures() +{ + if (!m_qnxCamera) + return; + + QCamera::Features features = {}; + + if (m_qnxCamera->hasFeature(CAMERA_FEATURE_REGIONFOCUS)) + features |= QCamera::Feature::CustomFocusPoint; + + supportedFeaturesChanged(features); + + minimumZoomFactorChanged(m_qnxCamera->minimumZoomLevel()); + maximumZoomFactorChanged(m_qnxCamera->maximumZoomLevel()); + + const QList<uint32_t> wbValues = m_qnxCamera->supportedWhiteBalanceValues(); + + if (wbValues.isEmpty()) { + m_minColorTemperature = m_maxColorTemperature = 0; + } else { + const auto &[minTemp, maxTemp] = std::minmax_element(wbValues.begin(), wbValues.end()); + + m_minColorTemperature = *minTemp; + m_maxColorTemperature = *maxTemp; + } +} + +void QQnxPlatformCamera::onFrameAvailable() +{ + if (!m_videoSink) + return; + + std::unique_ptr<QQnxCameraFrameBuffer> currentFrameBuffer = m_qnxCamera->takeCurrentFrame(); + + if (!currentFrameBuffer) + return; + + QVideoFrameFormat format(currentFrameBuffer->size(), currentFrameBuffer->pixelFormat()); + const QVideoFrame actualFrame = + QVideoFramePrivate::createFrame(std::move(currentFrameBuffer), std::move(format)); + + m_videoSink->setVideoFrame(actualFrame); + + if (!m_videoFrameRequests.empty()) { + VideoFrameCallback cb = std::move(m_videoFrameRequests.front()); + m_videoFrameRequests.pop_front(); + cb(actualFrame); + } +} + +QT_END_NAMESPACE + +#include "moc_qqnxplatformcamera_p.cpp" diff --git a/src/plugins/multimedia/qnx/camera/qqnxplatformcamera_p.h b/src/plugins/multimedia/qnx/camera/qqnxplatformcamera_p.h new file mode 100644 index 000000000..3cbd17a4f --- /dev/null +++ b/src/plugins/multimedia/qnx/camera/qqnxplatformcamera_p.h @@ -0,0 +1,113 @@ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only +#ifndef QQNXPLATFORMCAMERA_H +#define QQNXPLATFORMCAMERA_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qqnxcamera_p.h" + +#include <private/qplatformcamera_p.h> +#include <private/qplatformmediarecorder_p.h> + +#include <QtCore/qlist.h> +#include <QtCore/qmutex.h> +#include <QtCore/qurl.h> + +#include <deque> +#include <functional> +#include <memory> + +QT_BEGIN_NAMESPACE + +class QQnxPlatformCameraFrameBuffer; +class QQnxMediaCaptureSession; +class QQnxVideoSink; +class QQnxCameraFrameBuffer; + +class QQnxPlatformCamera : public QPlatformCamera +{ + Q_OBJECT +public: + explicit QQnxPlatformCamera(QCamera *parent); + ~QQnxPlatformCamera(); + + bool isActive() const override; + void setActive(bool active) override; + void start(); + void stop(); + + void setCamera(const QCameraDevice &camera) override; + + bool setCameraFormat(const QCameraFormat &format) override; + + void setCaptureSession(QPlatformMediaCaptureSession *session) override; + + bool isFocusModeSupported(QCamera::FocusMode mode) const override; + void setFocusMode(QCamera::FocusMode mode) override; + + void setCustomFocusPoint(const QPointF &point) override; + + void setFocusDistance(float distance) override; + + void zoomTo(float newZoomFactor, float rate = -1.) override; + + void setExposureCompensation(float ev) override; + + int isoSensitivity() const override; + void setManualIsoSensitivity(int value) override; + void setManualExposureTime(float seconds) override; + float exposureTime() const override; + + bool isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const override; + void setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) override; + void setColorTemperature(int temperature) override; + + void setOutputUrl(const QUrl &url); + void setMediaEncoderSettings(const QMediaEncoderSettings &settings); + + bool startVideoRecording(); + + using VideoFrameCallback = std::function<void(const QVideoFrame&)>; + void requestVideoFrame(VideoFrameCallback cb); + +private: + void updateCameraFeatures(); + void setColorTemperatureInternal(unsigned temp); + + bool isVideoEncodingSupported() const; + + void onFrameAvailable(); + + QQnxMediaCaptureSession *m_session = nullptr; + QQnxVideoSink *m_videoSink = nullptr; + + QCameraDevice m_cameraDevice; + + QUrl m_outputUrl; + + QMediaEncoderSettings m_encoderSettings; + + uint32_t m_minColorTemperature = 0; + uint32_t m_maxColorTemperature = 0; + + QMutex m_currentFrameMutex; + + std::unique_ptr<QQnxCamera> m_qnxCamera; + std::unique_ptr<QQnxCameraFrameBuffer> m_currentFrame; + + std::deque<VideoFrameCallback> m_videoFrameRequests; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder.cpp b/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder.cpp index e5e1f6849..00a20bbd7 100644 --- a/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder.cpp +++ b/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxaudiorecorder_p.h" #include "qqnxmediaeventthread_p.h" @@ -48,8 +12,6 @@ #include <sys/stat.h> #include <sys/strm.h> -static int idCounter = 0; - static QByteArray buildDevicePath(const QByteArray &deviceId, const QMediaEncoderSettings &settings) { QByteArray devicePath = QByteArrayLiteral("snd:/dev/snd/") + deviceId + QByteArrayLiteral("?"); @@ -79,6 +41,8 @@ QQnxAudioRecorder::~QQnxAudioRecorder() void QQnxAudioRecorder::openConnection() { + static int idCounter = 0; + m_connection = ConnectionUniquePtr { mmr_connect(nullptr) }; if (!m_connection) { @@ -316,3 +280,5 @@ void QQnxAudioRecorder::handleMmEventError(const mmr_event_t *event) } QT_END_NAMESPACE + +#include "moc_qqnxaudiorecorder_p.cpp" diff --git a/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder_p.h b/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder_p.h index 4b29e0ff9..f343cee14 100644 --- a/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder_p.h +++ b/src/plugins/multimedia/qnx/capture/qqnxaudiorecorder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXAUDIORECORDER_H #define QQNXAUDIORECORDER_H @@ -56,7 +20,7 @@ #include <QUrl> #include <QtCore/qobject.h> -#include <QtCore/qtnamespacemacros.h> +#include <QtCore/qtconfigmacros.h> #include <QtMultimedia/qmediarecorder.h> diff --git a/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession.cpp b/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession.cpp index c38dcfd80..d73ca7e54 100644 --- a/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession.cpp +++ b/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession.cpp @@ -1,45 +1,9 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxmediacapturesession_p.h" #include "qqnxaudioinput_p.h" -#include "qqnxcamera_p.h" +#include "qqnxplatformcamera_p.h" #include "qqnximagecapture_p.h" #include "qqnxmediarecorder_p.h" #include "qqnxvideosink_p.h" @@ -65,7 +29,15 @@ void QQnxMediaCaptureSession::setCamera(QPlatformCamera *camera) { if (camera == m_camera) return; - m_camera = static_cast<QQnxCamera *>(camera); + + if (m_camera) + m_camera->setCaptureSession(nullptr); + + m_camera = static_cast<QQnxPlatformCamera *>(camera); + + if (m_camera) + m_camera->setCaptureSession(this); + emit cameraChanged(); } @@ -78,7 +50,15 @@ void QQnxMediaCaptureSession::setImageCapture(QPlatformImageCapture *imageCaptur { if (m_imageCapture == imageCapture) return; + + if (m_imageCapture) + m_imageCapture->setCaptureSession(nullptr); + m_imageCapture = static_cast<QQnxImageCapture *>(imageCapture); + + if (m_imageCapture) + m_imageCapture->setCaptureSession(this); + emit imageCaptureChanged(); } @@ -137,3 +117,5 @@ QQnxVideoSink * QQnxMediaCaptureSession::videoSink() const } QT_END_NAMESPACE + +#include "moc_qqnxmediacapturesession_p.cpp" diff --git a/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession_p.h b/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession_p.h index ec15d80ba..551416a61 100644 --- a/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession_p.h +++ b/src/plugins/multimedia/qnx/capture/qqnxmediacapturesession_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXMEDIACAPTURESESSION_H #define QQNXMEDIACAPTURESESSION_H @@ -57,7 +21,7 @@ QT_BEGIN_NAMESPACE class QQnxAudioInput; -class QQnxCamera; +class QQnxPlatformCamera; class QQnxImageCapture; class QQnxMediaRecorder; class QQnxVideoSink; @@ -90,7 +54,7 @@ public: QQnxVideoSink *videoSink() const; private: - QQnxCamera *m_camera = nullptr; + QQnxPlatformCamera *m_camera = nullptr; QQnxImageCapture *m_imageCapture = nullptr; QQnxMediaRecorder *m_mediaRecorder = nullptr; QQnxAudioInput *m_audioInput = nullptr; diff --git a/src/plugins/multimedia/qnx/capture/qqnxmediarecorder.cpp b/src/plugins/multimedia/qnx/capture/qqnxmediarecorder.cpp index 6aa0c686b..62ac030db 100644 --- a/src/plugins/multimedia/qnx/capture/qqnxmediarecorder.cpp +++ b/src/plugins/multimedia/qnx/capture/qqnxmediarecorder.cpp @@ -1,44 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#undef QT_NO_CONTEXTLESS_CONNECT // Remove after porting connect() calls + #include "qqnxmediarecorder_p.h" +#include "qqnxplatformcamera_p.h" #include "qqnxaudioinput_p.h" #include "qqnxcamera_p.h" #include "qqnxmediacapturesession_p.h" @@ -118,11 +86,13 @@ void QQnxMediaRecorder::startVideoRecording(QMediaEncoderSettings &settings) if (!hasCamera()) return; - auto *camera = static_cast<QQnxCamera*>(m_session->camera()); + auto *camera = static_cast<QQnxPlatformCamera*>(m_session->camera()); camera->setMediaEncoderSettings(settings); camera->setOutputUrl(outputLocation()); - camera->start(); + + if (camera->startVideoRecording()) + stateChanged(QMediaRecorder::RecordingState); } void QQnxMediaRecorder::stopVideoRecording() @@ -130,10 +100,11 @@ void QQnxMediaRecorder::stopVideoRecording() if (!hasCamera()) return; - auto *camera = static_cast<QQnxCamera*>(m_session->camera()); + auto *camera = static_cast<QQnxPlatformCamera*>(m_session->camera()); camera->stop(); + stateChanged(QMediaRecorder::StoppedState); } bool QQnxMediaRecorder::hasCamera() const diff --git a/src/plugins/multimedia/qnx/capture/qqnxmediarecorder_p.h b/src/plugins/multimedia/qnx/capture/qqnxmediarecorder_p.h index 20444488c..8b3ea21d3 100644 --- a/src/plugins/multimedia/qnx/capture/qqnxmediarecorder_p.h +++ b/src/plugins/multimedia/qnx/capture/qqnxmediarecorder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXMEDIARECORDER_H #define QQNXMEDIARECORDER_H diff --git a/src/plugins/multimedia/qnx/common/mmrenderertypes.h b/src/plugins/multimedia/qnx/common/mmrenderertypes.h index f08154cc6..f1d498388 100644 --- a/src/plugins/multimedia/qnx/common/mmrenderertypes.h +++ b/src/plugins/multimedia/qnx/common/mmrenderertypes.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MMRENDERERTYPES_H #define MMRENDERERTYPES_H diff --git a/src/plugins/multimedia/qnx/common/qqnxaudioinput.cpp b/src/plugins/multimedia/qnx/common/qqnxaudioinput.cpp index 2aca652ab..fff3cf1eb 100644 --- a/src/plugins/multimedia/qnx/common/qqnxaudioinput.cpp +++ b/src/plugins/multimedia/qnx/common/qqnxaudioinput.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxaudioinput_p.h" diff --git a/src/plugins/multimedia/qnx/common/qqnxaudioinput_p.h b/src/plugins/multimedia/qnx/common/qqnxaudioinput_p.h index 2a115f2f0..62a573cc1 100644 --- a/src/plugins/multimedia/qnx/common/qqnxaudioinput_p.h +++ b/src/plugins/multimedia/qnx/common/qqnxaudioinput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXAUDIOINPUT_P_H #define QQNXAUDIOINPUT_P_H diff --git a/src/plugins/multimedia/qnx/common/qqnxaudiooutput.cpp b/src/plugins/multimedia/qnx/common/qqnxaudiooutput.cpp index 598f55026..76f8fbafd 100644 --- a/src/plugins/multimedia/qnx/common/qqnxaudiooutput.cpp +++ b/src/plugins/multimedia/qnx/common/qqnxaudiooutput.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxaudiooutput_p.h" @@ -46,7 +10,7 @@ #include <QtCore/qloggingcategory.h> -Q_LOGGING_CATEGORY(qLcMediaAudioOutput, "qt.multimedia.audiooutput") +static Q_LOGGING_CATEGORY(qLcMediaAudioOutput, "qt.multimedia.audiooutput") QT_BEGIN_NAMESPACE diff --git a/src/plugins/multimedia/qnx/common/qqnxaudiooutput_p.h b/src/plugins/multimedia/qnx/common/qqnxaudiooutput_p.h index db186689d..2ae5844e6 100644 --- a/src/plugins/multimedia/qnx/common/qqnxaudiooutput_p.h +++ b/src/plugins/multimedia/qnx/common/qqnxaudiooutput_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXAUDIOOUTPUT_P_H #define QQNXAUDIOOUTPUT_P_H diff --git a/src/plugins/multimedia/qnx/common/qqnxmediaeventthread.cpp b/src/plugins/multimedia/qnx/common/qqnxmediaeventthread.cpp index 1487ebc98..f0cc9b1c0 100644 --- a/src/plugins/multimedia/qnx/common/qqnxmediaeventthread.cpp +++ b/src/plugins/multimedia/qnx/common/qqnxmediaeventthread.cpp @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2017 QNX Software Systems. All rights reserved. -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2017 QNX Software Systems. All rights reserved. +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxmediaeventthread_p.h" @@ -52,6 +16,8 @@ int mmr_event_arm(mmr_context_t *ctxt, struct sigevent const *sev); } +QT_BEGIN_NAMESPACE + static const int c_mmrCode = _PULSE_CODE_MINAVAIL + 0; static const int c_readCode = _PULSE_CODE_MINAVAIL + 1; static const int c_quitCode = _PULSE_CODE_MINAVAIL + 2; @@ -126,3 +92,7 @@ void QQnxMediaEventThread::shutdown() // block until thread terminates wait(); } + +QT_END_NAMESPACE + +#include "moc_qqnxmediaeventthread_p.cpp" diff --git a/src/plugins/multimedia/qnx/common/qqnxmediaeventthread_p.h b/src/plugins/multimedia/qnx/common/qqnxmediaeventthread_p.h index febc154ca..a622fcb62 100644 --- a/src/plugins/multimedia/qnx/common/qqnxmediaeventthread_p.h +++ b/src/plugins/multimedia/qnx/common/qqnxmediaeventthread_p.h @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2017 QNX Software Systems. All rights reserved. -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2017 QNX Software Systems. All rights reserved. +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXMEDIAEVENTTHREAD_P_H #define QQNXMEDIAEVENTTHREAD_P_H diff --git a/src/plugins/multimedia/qnx/common/qqnxwindowgrabber.cpp b/src/plugins/multimedia/qnx/common/qqnxwindowgrabber.cpp index 76231c0c3..28f16b70a 100644 --- a/src/plugins/multimedia/qnx/common/qqnxwindowgrabber.cpp +++ b/src/plugins/multimedia/qnx/common/qqnxwindowgrabber.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxwindowgrabber_p.h" @@ -49,8 +13,7 @@ #include <QOpenGLContext> #include <QOpenGLFunctions> -#include <QtGui/private/qrhi_p.h> -#include <QtGui/private/qrhigles2_p.h> +#include <rhi/qrhi.h> #include <cstring> @@ -442,8 +405,12 @@ QQnxWindowGrabber::BufferView QQnxWindowGrabberImage::getBuffer( GLuint QQnxWindowGrabberImage::getTexture(screen_window_t window, const QSize &size) { if (size != m_size) { - if (!m_glTexture) - glGenTextures(1, &m_glTexture); + // create a brand new texture to be the KHR image sibling, as + // previously used textures cannot be reused with new KHR image + // sources - note that glDeleteTextures handles nullptr gracefully + glDeleteTextures(1, &m_glTexture); + glGenTextures(1, &m_glTexture); + glBindTexture(GL_TEXTURE_2D, m_glTexture); if (m_eglImage) { glEGLImageTargetTexture2DOES(GL_TEXTURE_2D, 0); @@ -464,3 +431,5 @@ GLuint QQnxWindowGrabberImage::getTexture(screen_window_t window, const QSize &s } QT_END_NAMESPACE + +#include "moc_qqnxwindowgrabber_p.cpp" diff --git a/src/plugins/multimedia/qnx/common/qqnxwindowgrabber_p.h b/src/plugins/multimedia/qnx/common/qqnxwindowgrabber_p.h index 05143217b..1ffd96b63 100644 --- a/src/plugins/multimedia/qnx/common/qqnxwindowgrabber_p.h +++ b/src/plugins/multimedia/qnx/common/qqnxwindowgrabber_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQnxWindowGrabber_H #define QQnxWindowGrabber_H diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata.cpp b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata.cpp index f5ede6c4e..fcd535814 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata.cpp +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxmediametadata_p.h" #include <QtCore/qdebug.h> diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata_p.h b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata_p.h index 7037d96a2..db7639dc5 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata_p.h +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediametadata_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQnxMediaMetaData_H #define QQnxMediaMetaData_H diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer.cpp b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer.cpp index 802b5bc59..14b190836 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer.cpp +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer.cpp @@ -1,49 +1,14 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxmediaplayer_p.h" #include "qqnxvideosink_p.h" #include "qqnxmediautil_p.h" #include "qqnxmediaeventthread_p.h" #include "qqnxwindowgrabber_p.h" -#include <private/qabstractvideobuffer_p.h> +#include <private/qhwvideobuffer_p.h> +#include <private/qvideoframe_p.h> #include <QtCore/qabstracteventdispatcher.h> #include <QtCore/qcoreapplication.h> @@ -98,29 +63,24 @@ static std::tuple<int, int, bool> parseBufferLevel(const QString &value) return { level, capacity, true }; } -class QnxTextureBuffer : public QAbstractVideoBuffer +class QnxTextureBuffer : public QHwVideoBuffer { public: QnxTextureBuffer(QQnxWindowGrabber *QQnxWindowGrabber) - : QAbstractVideoBuffer(QVideoFrame::RhiTextureHandle) + : QHwVideoBuffer(QVideoFrame::RhiTextureHandle) { m_windowGrabber = QQnxWindowGrabber; m_handle = 0; } - QVideoFrame::MapMode mapMode() const override - { - return QVideoFrame::ReadWrite; - } - void unmap() override {} - MapData map(QVideoFrame::MapMode /*mode*/) override + MapData map(QtVideo::MapMode /*mode*/) override { return {}; } - quint64 textureHandle(int plane) const override + quint64 textureHandle(QRhi *, int plane) const override { if (plane != 0) return 0; @@ -138,19 +98,13 @@ private: class QnxRasterBuffer : public QAbstractVideoBuffer { public: - QnxRasterBuffer(QQnxWindowGrabber *windowGrabber) - : QAbstractVideoBuffer(QVideoFrame::NoHandle) - { - m_windowGrabber = windowGrabber; - } + QnxRasterBuffer(QQnxWindowGrabber *windowGrabber) { m_windowGrabber = windowGrabber; } - QVideoFrame::MapMode mapMode() const override + MapData map(QtVideo::MapMode mode) override { - return QVideoFrame::ReadOnly; - } + if (mode != QtVideo::MapMode::ReadOnly) + return {}; - MapData map(QVideoFrame::MapMode /*mode*/) override - { if (buffer.data) { qWarning("QnxRasterBuffer: need to unmap before mapping"); return {}; @@ -159,10 +113,10 @@ public: buffer = m_windowGrabber->getNextBuffer(); return { - .nPlanes = 1, + .planeCount = 1, .bytesPerLine = { buffer.stride }, .data = { buffer.data }, - .size = { buffer.width * buffer.height * buffer.pixelSize } + .dataSize = { buffer.width * buffer.height * buffer.pixelSize } }; } @@ -171,13 +125,13 @@ public: buffer = {}; } + QVideoFrameFormat format() const override { return {}; } + private: QQnxWindowGrabber *m_windowGrabber; QQnxWindowGrabber::BufferView buffer; }; -static int idCounter = 0; - QT_BEGIN_NAMESPACE QQnxMediaPlayer::QQnxMediaPlayer(QMediaPlayer *parent) @@ -204,6 +158,8 @@ QQnxMediaPlayer::~QQnxMediaPlayer() void QQnxMediaPlayer::openConnection() { + static int idCounter = 0; + m_connection = mmr_connect(nullptr); if (!m_connection) { emitPError(QString::fromLatin1("Unable to connect to the multimedia renderer")); @@ -553,12 +509,13 @@ void QQnxMediaPlayer::updateScene(const QSize &size) if (!m_platformVideoSink) return; - auto *buffer = m_windowGrabber->isEglImageSupported() - ? static_cast<QAbstractVideoBuffer*>(new QnxTextureBuffer(m_windowGrabber)) - : static_cast<QAbstractVideoBuffer*>(new QnxRasterBuffer(m_windowGrabber)); + QVideoFrameFormat format(size, QVideoFrameFormat::Format_BGRX8888); - const QVideoFrame actualFrame(buffer, - QVideoFrameFormat(size, QVideoFrameFormat::Format_BGRX8888)); + const QVideoFrame actualFrame = m_windowGrabber->isEglImageSupported() + ? QVideoFramePrivate::createFrame(std::make_unique<QnxTextureBuffer>(m_windowGrabber), + std::move(format)) + : QVideoFramePrivate::createFrame(std::make_unique<QnxRasterBuffer>(m_windowGrabber), + std::move(format)); m_platformVideoSink->setVideoFrame(actualFrame); } @@ -926,3 +883,5 @@ void QQnxMediaPlayer::readEvents() } QT_END_NAMESPACE + +#include "moc_qqnxmediaplayer_p.cpp" diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer_p.h b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer_p.h index 321c13cc8..c570a6334 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer_p.h +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediaplayer_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQnxMediaPlayer_H #define QQnxMediaPlayer_H diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil.cpp b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil.cpp index 3aeadb51b..074989642 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil.cpp +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxmediautil_p.h" #include <QDebug> diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil_p.h b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil_p.h index e1f972840..7b709142f 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil_p.h +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxmediautil_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MMRENDERERUTIL_H #define MMRENDERERUTIL_H diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink.cpp b/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink.cpp index e2f682dd1..18d4d1828 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink.cpp +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink.cpp @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxvideosink_p.h" @@ -58,3 +22,5 @@ QRhi *QQnxVideoSink::rhi() const } QT_END_NAMESPACE + +#include "moc_qqnxvideosink_p.cpp" diff --git a/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink_p.h b/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink_p.h index 6064c40bc..2cc7990db 100644 --- a/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink_p.h +++ b/src/plugins/multimedia/qnx/mediaplayer/qqnxvideosink_p.h @@ -1,42 +1,6 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Research In Motion -** Copyright (C) 2021 The Qt Company -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Research In Motion +// Copyright (C) 2021 The Qt Company +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXVIDFEOSINK_P_H #define QQNXVIDFEOSINK_P_H diff --git a/src/plugins/multimedia/qnx/qqnxformatinfo.cpp b/src/plugins/multimedia/qnx/qqnxformatinfo.cpp index 9579e98b3..77492e80d 100644 --- a/src/plugins/multimedia/qnx/qqnxformatinfo.cpp +++ b/src/plugins/multimedia/qnx/qqnxformatinfo.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxformatinfo_p.h" diff --git a/src/plugins/multimedia/qnx/qqnxformatinfo_p.h b/src/plugins/multimedia/qnx/qqnxformatinfo_p.h index 8874c2550..aae3a026a 100644 --- a/src/plugins/multimedia/qnx/qqnxformatinfo_p.h +++ b/src/plugins/multimedia/qnx/qqnxformatinfo_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXFORMATINFO_H #define QQNXFORMATINFO_H diff --git a/src/plugins/multimedia/qnx/qqnxmediaintegration.cpp b/src/plugins/multimedia/qnx/qqnxmediaintegration.cpp index 14d403b64..8567a69fd 100644 --- a/src/plugins/multimedia/qnx/qqnxmediaintegration.cpp +++ b/src/plugins/multimedia/qnx/qqnxmediaintegration.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxmediaintegration_p.h" #include "qqnxmediacapturesession_p.h" @@ -44,6 +8,8 @@ #include "qqnxvideodevices_p.h" #include "qqnxvideosink_p.h" #include "qqnxmediaplayer_p.h" +#include "qqnximagecapture_p.h" +#include "qqnxplatformcamera_p.h" #include <QtMultimedia/private/qplatformmediaplugin_p.h> QT_BEGIN_NAMESPACE @@ -60,49 +26,54 @@ public: QPlatformMediaIntegration* create(const QString &name) override { - if (name == QLatin1String("qnx")) + if (name == u"qnx") return new QQnxMediaIntegration; return nullptr; } }; -QQnxMediaIntegration::QQnxMediaIntegration() -{ - m_videoDevices = new QQnxVideoDevices(this); -} +QQnxMediaIntegration::QQnxMediaIntegration() : QPlatformMediaIntegration(QLatin1String("qnx")) { } -QQnxMediaIntegration::~QQnxMediaIntegration() +QPlatformMediaFormatInfo *QQnxMediaIntegration::createFormatInfo() { - delete m_formatInfo; + return new QQnxFormatInfo; } -QPlatformMediaFormatInfo *QQnxMediaIntegration::formatInfo() +QPlatformVideoDevices *QQnxMediaIntegration::createVideoDevices() { - if (!m_formatInfo) - m_formatInfo = new QQnxFormatInfo(); - return m_formatInfo; + return new QQnxVideoDevices(this); } -QPlatformVideoSink *QQnxMediaIntegration::createVideoSink(QVideoSink *sink) +QMaybe<QPlatformVideoSink *> QQnxMediaIntegration::createVideoSink(QVideoSink *sink) { return new QQnxVideoSink(sink); } -QPlatformMediaPlayer *QQnxMediaIntegration::createPlayer(QMediaPlayer *parent) +QMaybe<QPlatformMediaPlayer *> QQnxMediaIntegration::createPlayer(QMediaPlayer *parent) { return new QQnxMediaPlayer(parent); } -QPlatformMediaCaptureSession *QQnxMediaIntegration::createCaptureSession() +QMaybe<QPlatformMediaCaptureSession *> QQnxMediaIntegration::createCaptureSession() { return new QQnxMediaCaptureSession(); } -QPlatformMediaRecorder *QQnxMediaIntegration::createRecorder(QMediaRecorder *parent) +QMaybe<QPlatformMediaRecorder *> QQnxMediaIntegration::createRecorder(QMediaRecorder *parent) { return new QQnxMediaRecorder(parent); } +QMaybe<QPlatformCamera *> QQnxMediaIntegration::createCamera(QCamera *parent) +{ + return new QQnxPlatformCamera(parent); +} + +QMaybe<QPlatformImageCapture *> QQnxMediaIntegration::createImageCapture(QImageCapture *parent) +{ + return new QQnxImageCapture(parent); +} + QT_END_NAMESPACE #include "qqnxmediaintegration.moc" diff --git a/src/plugins/multimedia/qnx/qqnxmediaintegration_p.h b/src/plugins/multimedia/qnx/qqnxmediaintegration_p.h index a480841a5..60fafc246 100644 --- a/src/plugins/multimedia/qnx/qqnxmediaintegration_p.h +++ b/src/plugins/multimedia/qnx/qqnxmediaintegration_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQnxMediaIntegration_H #define QQnxMediaIntegration_H @@ -62,19 +26,23 @@ class QQnxMediaIntegration : public QPlatformMediaIntegration { public: QQnxMediaIntegration(); - ~QQnxMediaIntegration(); - QPlatformMediaFormatInfo *formatInfo() override; + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *sink) override; - QPlatformVideoSink *createVideoSink(QVideoSink *sink) override; + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *parent) override; - QPlatformMediaPlayer *createPlayer(QMediaPlayer *parent) override; + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; - QPlatformMediaCaptureSession *createCaptureSession() override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *parent) override; - QPlatformMediaRecorder *createRecorder(QMediaRecorder *parent) override; + QMaybe<QPlatformCamera *> createCamera(QCamera *parent) override; - QQnxFormatInfo *m_formatInfo = nullptr; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *parent) override; + +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; + + QPlatformVideoDevices *createVideoDevices() override; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/qnx/qqnxvideodevices.cpp b/src/plugins/multimedia/qnx/qqnxvideodevices.cpp index 5414990f9..ea0cfd956 100644 --- a/src/plugins/multimedia/qnx/qqnxvideodevices.cpp +++ b/src/plugins/multimedia/qnx/qqnxvideodevices.cpp @@ -1,51 +1,16 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qqnxvideodevices_p.h" +#include "qqnxcamera_p.h" #include "private/qcameradevice_p.h" #include "qcameradevice.h" -#include <camera/camera_api.h> - #include <qdir.h> #include <qdebug.h> +#include <optional> + QT_BEGIN_NAMESPACE static QVideoFrameFormat::PixelFormat fromCameraFrametype(camera_frametype_t type) @@ -71,77 +36,51 @@ static QVideoFrameFormat::PixelFormat fromCameraFrametype(camera_frametype_t typ } } -static QList<QCameraDevice> enumerateCameras() +static std::optional<QCameraDevice> createCameraDevice(camera_unit_t unit, bool isDefault) { + const QQnxCamera camera(unit); - camera_unit_t cameraUnits[64]; - - unsigned int knownCameras = 0; - const camera_error_t result = camera_get_supported_cameras(64, &knownCameras, cameraUnits); - if (result != CAMERA_EOK) { - qWarning() << "Unable to retrieve supported camera types:" << result; + if (!camera.isValid()) { + qWarning() << "Invalid camera unit:" << unit; return {}; } - QList<QCameraDevice> cameras; - for (unsigned int i = 0; i < knownCameras; ++i) { - QCameraDevicePrivate *p = new QCameraDevicePrivate; - p->id = QByteArray::number(cameraUnits[i]); - - char name[CAMERA_LOCATION_NAMELEN]; - camera_get_location_property(cameraUnits[i], CAMERA_LOCATION_NAME, &name, CAMERA_LOCATION_END); - p->description = QString::fromUtf8(name); - - if (i == 0) - p->isDefault = true; - - camera_handle_t handle; - if (camera_open(cameraUnits[i], CAMERA_MODE_PREAD, &handle) == CAMERA_EOK) { - // query camera properties - - uint32_t nResolutions = 0; - camera_get_supported_vf_resolutions(handle, 0, &nResolutions, nullptr); - QVarLengthArray<camera_res_t> resolutions(nResolutions); - camera_get_supported_vf_resolutions(handle, nResolutions, &nResolutions, resolutions.data()); - - uint32_t nFrameTypes; - camera_get_supported_vf_frame_types(handle, 0, &nFrameTypes, nullptr); - QVarLengthArray<camera_frametype_t> frameTypes(nFrameTypes); - camera_get_supported_vf_frame_types(handle, nFrameTypes, &nFrameTypes, frameTypes.data()); - - for (auto res : resolutions) { - QSize resolution(res.width, res.height); - p->photoResolutions.append(resolution); - - for (auto frameType : frameTypes) { - auto pixelFormat = fromCameraFrametype(frameType); - if (pixelFormat == QVideoFrameFormat::Format_Invalid) - continue; - - uint32_t nFrameRates; - camera_get_specified_vf_framerates(handle, frameType, res, 0, &nFrameRates, nullptr, nullptr); - QVarLengthArray<double> frameRates(nFrameRates); - bool continuous = false; - camera_get_specified_vf_framerates(handle, frameType, res, nFrameRates, &nFrameRates, frameRates.data(), &continuous); - - QCameraFormatPrivate *f = new QCameraFormatPrivate; - f->resolution = resolution; - f->pixelFormat = pixelFormat; - f->minFrameRate = 1.e10; - for (auto fr : frameRates) { - if (fr < f->minFrameRate) - f->minFrameRate = fr; - if (fr > f->maxFrameRate) - f->maxFrameRate = fr; - } - p->videoFormats.append(f->create()); - } + auto *p = new QCameraDevicePrivate; + + p->id = QByteArray::number(camera.unit()); + p->description = camera.name(); + p->isDefault = isDefault; + + const QList<camera_frametype_t> frameTypes = camera.supportedVfFrameTypes(); + + for (camera_res_t res : camera.supportedVfResolutions()) { + const QSize resolution(res.width, res.height); + + p->photoResolutions.append(resolution); + + for (camera_frametype_t frameType : camera.supportedVfFrameTypes()) { + const QVideoFrameFormat::PixelFormat pixelFormat = fromCameraFrametype(frameType); + + if (pixelFormat == QVideoFrameFormat::Format_Invalid) + continue; + + auto *f = new QCameraFormatPrivate; + p->videoFormats.append(f->create()); + + f->resolution = resolution; + f->pixelFormat = pixelFormat; + f->minFrameRate = 1.e10; + + for (double fr : camera.specifiedVfFrameRates(frameType, res)) { + if (fr < f->minFrameRate) + f->minFrameRate = fr; + if (fr > f->maxFrameRate) + f->maxFrameRate = fr; } } - - cameras.append(p->create()); } - return cameras; + + return p->create(); } QQnxVideoDevices::QQnxVideoDevices(QPlatformMediaIntegration *integration) @@ -151,10 +90,21 @@ QQnxVideoDevices::QQnxVideoDevices(QPlatformMediaIntegration *integration) QList<QCameraDevice> QQnxVideoDevices::videoDevices() const { - if (!camerasChecked) { - camerasChecked = true; - cameras = enumerateCameras(); + QList<QCameraDevice> cameras; + + bool isDefault = true; + + for (const camera_unit_t cameraUnit : QQnxCamera::supportedUnits()) { + const std::optional<QCameraDevice> cameraDevice = createCameraDevice(cameraUnit, isDefault); + + if (!cameraDevice) + continue; + + cameras.append(*cameraDevice); + + isDefault = false; } + return cameras; } diff --git a/src/plugins/multimedia/qnx/qqnxvideodevices_p.h b/src/plugins/multimedia/qnx/qqnxvideodevices_p.h index 8900319c1..cc2284e57 100644 --- a/src/plugins/multimedia/qnx/qqnxvideodevices_p.h +++ b/src/plugins/multimedia/qnx/qqnxvideodevices_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QQNXVIDEODEVICES_H #define QQNXVIDEODEVICES_H @@ -52,20 +16,15 @@ // #include <private/qplatformvideodevices_p.h> -#include <qcameradevice.h> QT_BEGIN_NAMESPACE class QQnxVideoDevices : public QPlatformVideoDevices { public: - QQnxVideoDevices(QPlatformMediaIntegration *integration); + explicit QQnxVideoDevices(QPlatformMediaIntegration *integration); QList<QCameraDevice> videoDevices() const override; - -private: - mutable bool camerasChecked = false; - mutable QList<QCameraDevice> cameras; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/wasm/CMakeLists.txt b/src/plugins/multimedia/wasm/CMakeLists.txt new file mode 100644 index 000000000..21f3e3472 --- /dev/null +++ b/src/plugins/multimedia/wasm/CMakeLists.txt @@ -0,0 +1,25 @@ +qt_internal_add_plugin(QWasmMediaPlugin + OUTPUT_NAME wasmmediaplugin + PLUGIN_TYPE multimedia + SOURCES + qwasmmediaintegration.cpp qwasmmediaintegration_p.h + mediaplayer/qwasmmediaplayer.cpp mediaplayer/qwasmmediaplayer_p.h + mediaplayer/qwasmvideosink.cpp mediaplayer/qwasmvideosink_p.h + common/qwasmvideooutput.cpp common/qwasmvideooutput_p.h + common/qwasmaudiooutput.cpp common/qwasmaudiooutput_p.h + common/qwasmaudioinput.cpp common/qwasmaudioinput_p.h + mediacapture/qwasmmediacapturesession.cpp mediacapture/qwasmmediacapturesession_p.h + mediacapture/qwasmmediarecorder.cpp mediacapture/qwasmmediarecorder_p.h + mediacapture/qwasmcamera.cpp mediacapture/qwasmcamera_p.h + mediacapture/qwasmimagecapture.cpp mediacapture/qwasmimagecapture_p.h + INCLUDE_DIRECTORIES + common + mediaplayer + mediacapture + LIBRARIES + Qt::MultimediaPrivate + Qt::CorePrivate + openal +) + +target_link_libraries(QWasmMediaPlugin PUBLIC embind) diff --git a/src/plugins/multimedia/wasm/common/qwasmaudioinput.cpp b/src/plugins/multimedia/wasm/common/qwasmaudioinput.cpp new file mode 100644 index 000000000..a0418c5c2 --- /dev/null +++ b/src/plugins/multimedia/wasm/common/qwasmaudioinput.cpp @@ -0,0 +1,107 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmaudioinput_p.h" + +#include <qaudioinput.h> +#include <private/qstdweb_p.h> + +QT_BEGIN_NAMESPACE + +Q_LOGGING_CATEGORY(qWasmAudioInput, "qt.multimedia.wasm.audioinput") + +QWasmAudioInput::QWasmAudioInput(QAudioInput *parent) + : QObject(parent), QPlatformAudioInput(parent) +{ + m_wasMuted = false; + setDeviceSourceStream(""); +} + +QWasmAudioInput::~QWasmAudioInput() +{ +} + +void QWasmAudioInput::setMuted(bool muted) +{ + qCDebug(qWasmAudioInput) << Q_FUNC_INFO << muted; + if (muted == m_wasMuted) + return; + if (m_mediaStream.isNull() || m_mediaStream.isUndefined()) + return; + emscripten::val audioTracks = m_mediaStream.call<emscripten::val>("getAudioTracks"); + if (audioTracks.isNull() || audioTracks.isUndefined()) + return; + if (audioTracks["length"].as<int>() < 1) + return; + audioTracks[0].set("muted", muted); + + emit mutedChanged(muted); + m_wasMuted = muted; + +} + +bool QWasmAudioInput::isMuted() const +{ + return m_wasMuted; +} + +void QWasmAudioInput::setAudioDevice(const QAudioDevice &audioDevice) +{ + if (device == audioDevice) + return; + + device = audioDevice; + setDeviceSourceStream(device.id().toStdString()); +} + +void QWasmAudioInput::setVolume(float volume) +{ + Q_UNUSED(volume) + // TODO seems no easy way to set input volume +} + +void QWasmAudioInput::setDeviceSourceStream(const std::string &id) +{ + qCDebug(qWasmAudioInput) << Q_FUNC_INFO << id; + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + if (mediaDevices.isNull() || mediaDevices.isUndefined()) { + qWarning() << "No media devices found"; + return; + } + + qstdweb::PromiseCallbacks getUserMediaCallback{ + // default + .thenFunc = + [this](emscripten::val stream) { + qCDebug(qWasmAudioInput) << "getUserMediaSuccess"; + m_mediaStream = stream; + }, + .catchFunc = + [](emscripten::val error) { + qCDebug(qWasmAudioInput) + << "addCameraSourceElement getUserMedia fail" + << QString::fromStdString(error["name"].as<std::string>()) + << QString::fromStdString(error["message"].as<std::string>()); + } + }; + + emscripten::val constraints = emscripten::val::object(); + constraints.set("audio", true); + if (!id.empty()) + constraints.set("deviceId", id); + + // we do it this way as this prompts user for mic permissions + qstdweb::Promise::make(mediaDevices, QStringLiteral("getUserMedia"), + std::move(getUserMediaCallback), constraints); +} + +emscripten::val QWasmAudioInput::mediaStream() +{ + return m_mediaStream; +} + +QT_END_NAMESPACE + +#include "moc_qwasmaudioinput_p.cpp" diff --git a/src/plugins/multimedia/wasm/common/qwasmaudioinput_p.h b/src/plugins/multimedia/wasm/common/qwasmaudioinput_p.h new file mode 100644 index 000000000..c772ee956 --- /dev/null +++ b/src/plugins/multimedia/wasm/common/qwasmaudioinput_p.h @@ -0,0 +1,57 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMAUDIOINPUT_H +#define QWASMAUDIOINPUT_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtCore/qobject.h> +#include <QtCore/qloggingcategory.h> + +#include <private/qtmultimediaglobal_p.h> +#include <private/qplatformaudioinput_p.h> + +#include <emscripten.h> +#include <emscripten/val.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmAudioInput) + +class QWasmAudioInput : public QObject, public QPlatformAudioInput +{ + Q_OBJECT +public: + explicit QWasmAudioInput(QAudioInput *parent); + ~QWasmAudioInput(); + + void setMuted(bool muted) override; + void setAudioDevice(const QAudioDevice & device) final; + + bool isMuted() const; + void setVolume(float volume) final; + emscripten::val mediaStream(); + + +Q_SIGNALS: + void mutedChanged(bool muted); + +private: + bool m_wasMuted = false; + void setDeviceSourceStream(const std::string &id); + emscripten::val m_mediaStream; +}; + +QT_END_NAMESPACE + +#endif // QWASMAUDIOINPUT_H diff --git a/src/plugins/multimedia/wasm/common/qwasmaudiooutput.cpp b/src/plugins/multimedia/wasm/common/qwasmaudiooutput.cpp new file mode 100644 index 000000000..a9a644140 --- /dev/null +++ b/src/plugins/multimedia/wasm/common/qwasmaudiooutput.cpp @@ -0,0 +1,378 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <qaudiodevice.h> +#include <qaudiooutput.h> +#include <qwasmaudiooutput_p.h> + +#include <QMimeDatabase> +#include <QtCore/qloggingcategory.h> +#include <QMediaDevices> +#include <QUrl> +#include <QFile> +#include <QMimeDatabase> +#include <QFileInfo> + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qWasmMediaAudioOutput, "qt.multimedia.wasm.audiooutput") + +QWasmAudioOutput::QWasmAudioOutput(QAudioOutput *parent) + : QPlatformAudioOutput(parent) +{ +} + +QWasmAudioOutput::~QWasmAudioOutput() = default; + +void QWasmAudioOutput::setAudioDevice(const QAudioDevice &audioDevice) +{ + qCDebug(qWasmMediaAudioOutput) << Q_FUNC_INFO << device.id(); + device = audioDevice; +} + +void QWasmAudioOutput::setVideoElement(emscripten::val videoElement) +{ + m_videoElement = videoElement; +} + +emscripten::val QWasmAudioOutput::videoElement() +{ + return m_videoElement; +} + +void QWasmAudioOutput::setMuted(bool muted) +{ + emscripten::val realElement = videoElement(); + if (!realElement.isUndefined()) { + realElement.set("muted", muted); + return; + } + if (m_audio.isUndefined() || m_audio.isNull()) { + qCDebug(qWasmMediaAudioOutput) << "Error" + << "Audio element could not be created"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("Media file could not be opened")); + return; + } + m_audio.set("mute", muted); +} + +void QWasmAudioOutput::setVolume(float volume) +{ + volume = qBound(qreal(0.0), volume, qreal(1.0)); + emscripten::val realElement = videoElement(); + if (!realElement.isUndefined()) { + realElement.set("volume", volume); + return; + } + if (m_audio.isUndefined() || m_audio.isNull()) { + qCDebug(qWasmMediaAudioOutput) << "Error" + << "Audio element not available"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("Media file could not be opened")); + return; + } + + m_audio.set("volume", volume); +} + +void QWasmAudioOutput::setSource(const QUrl &url) +{ + qCDebug(qWasmMediaAudioOutput) << Q_FUNC_INFO << url; + if (url.isEmpty()) { + stop(); + return; + } + + createAudioElement(device.id().toStdString()); + + if (m_audio.isUndefined() || m_audio.isNull()) { + qCDebug(qWasmMediaAudioOutput) << "Error" + << "Audio element could not be created"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("Audio element could not be created")); + return; + } + + emscripten::val document = emscripten::val::global("document"); + emscripten::val body = document["body"]; + + m_audio.set("id", device.id().toStdString()); + + body.call<void>("appendChild", m_audio); + + + if (url.isLocalFile()) { // is localfile + qCDebug(qWasmMediaAudioOutput) << "is localfile"; + m_source = url.toLocalFile(); + + QFile mediaFile(m_source); + if (!mediaFile.open(QIODevice::ReadOnly)) { + qCDebug(qWasmMediaAudioOutput) << "Error" + << "Media file could not be opened"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("Media file could not be opened")); + return; + } + + // local files are relatively small due to browser filesystem being restricted + QByteArray content = mediaFile.readAll(); + + QMimeDatabase db; + qCDebug(qWasmMediaAudioOutput) << db.mimeTypeForData(content).name(); + + qstdweb::Blob contentBlob = qstdweb::Blob::copyFrom(content.constData(), content.size()); + emscripten::val contentUrl = + qstdweb::window()["URL"].call<emscripten::val>("createObjectURL", contentBlob.val()); + + emscripten::val audioSourceElement = + document.call<emscripten::val>("createElement", std::string("source")); + + audioSourceElement.set("src", contentUrl); + + // work around Safari not being able to read audio from blob URLs. + QFileInfo info(m_source); + QMimeType mimeType = db.mimeTypeForFile(info); + + audioSourceElement.set("type", mimeType.name().toStdString()); + m_audio.call<void>("appendChild", audioSourceElement); + + m_audio.call<void>("setAttribute", emscripten::val("srcObject"), contentUrl); + + } else { + m_source = url.toString(); + m_audio.set("src", m_source.toStdString()); + } + m_audio.set("id", device.id().toStdString()); + + body.call<void>("appendChild", m_audio); + qCDebug(qWasmMediaAudioOutput) << Q_FUNC_INFO << device.id(); + + doElementCallbacks(); +} + +void QWasmAudioOutput::setSource(QIODevice *stream) +{ + m_audioIODevice = stream; +} + +void QWasmAudioOutput::start() +{ + if (m_audio.isNull() || m_audio.isUndefined()) { + qCDebug(qWasmMediaAudioOutput) << "audio failed to start"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("Audio element resource error")); + return; + } + + m_audio.call<void>("play"); +} + +void QWasmAudioOutput::stop() +{ + if (m_audio.isNull() || m_audio.isUndefined()) { + qCDebug(qWasmMediaAudioOutput) << "audio failed to start"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("Audio element resource error")); + return; + } + if (!m_source.isEmpty()) { + pause(); + m_audio.set("currentTime", emscripten::val(0)); + } + if (m_audioIODevice) { + m_audioIODevice->close(); + delete m_audioIODevice; + m_audioIODevice = 0; + } +} + +void QWasmAudioOutput::pause() +{ + if (m_audio.isNull() || m_audio.isUndefined()) { + qCDebug(qWasmMediaAudioOutput) << "audio failed to start"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("Audio element resource error")); + return; + } + m_audio.call<emscripten::val>("pause"); +} + +void QWasmAudioOutput::createAudioElement(const std::string &id) +{ + emscripten::val document = emscripten::val::global("document"); + m_audio = document.call<emscripten::val>("createElement", std::string("audio")); + + // only works in chrome and firefox. + // Firefox this feature is behind media.setsinkid.enabled preferences + // allows user to choose audio output device + + if (!m_audio.hasOwnProperty("sinkId") || m_audio["sinkId"].isUndefined()) { + return; + } + + std::string usableId = id; + if (usableId.empty()) + usableId = QMediaDevices::defaultAudioOutput().id(); + + qstdweb::PromiseCallbacks sinkIdCallbacks{ + .thenFunc = [](emscripten::val) { qCWarning(qWasmMediaAudioOutput) << "setSinkId ok"; }, + .catchFunc = + [](emscripten::val) { + qCWarning(qWasmMediaAudioOutput) << "Error while trying to setSinkId"; + } + }; + qstdweb::Promise::make(m_audio, "setSinkId", std::move(sinkIdCallbacks), std::move(usableId)); + + m_audio.set("id", usableId.c_str()); +} + +void QWasmAudioOutput::doElementCallbacks() +{ + // error + auto errorCallback = [&](emscripten::val event) { + qCDebug(qWasmMediaAudioOutput) << "error"; + if (event.isUndefined() || event.isNull()) + return; + emit errorOccured(m_audio["error"]["code"].as<int>(), + QString::fromStdString(m_audio["error"]["message"].as<std::string>())); + + QString errorMessage = + QString::fromStdString(m_audio["error"]["message"].as<std::string>()); + if (errorMessage.isEmpty()) { + switch (m_audio["error"]["code"].as<int>()) { + case AudioElementError::MEDIA_ERR_ABORTED: + errorMessage = QStringLiteral("aborted by the user agent at the user's request."); + break; + case AudioElementError::MEDIA_ERR_NETWORK: + errorMessage = QStringLiteral("network error."); + break; + case AudioElementError::MEDIA_ERR_DECODE: + errorMessage = QStringLiteral("decoding error."); + break; + case AudioElementError::MEDIA_ERR_SRC_NOT_SUPPORTED: + errorMessage = QStringLiteral("src attribute not suitable."); + break; + }; + } + qCDebug(qWasmMediaAudioOutput) << m_audio["error"]["code"].as<int>() << errorMessage; + + emit errorOccured(m_audio["error"]["code"].as<int>(), errorMessage); + }; + m_errorChangeEvent.reset(new qstdweb::EventCallback(m_audio, "error", errorCallback)); + + // loadeddata + auto loadedDataCallback = [&](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaAudioOutput) << "loaded data"; + qstdweb::window()["URL"].call<emscripten::val>("revokeObjectURL", m_audio["src"]); + }; + m_loadedDataEvent.reset(new qstdweb::EventCallback(m_audio, "loadeddata", loadedDataCallback)); + + // canplay + auto canPlayCallback = [&](emscripten::val event) { + if (event.isUndefined() || event.isNull()) + return; + qCDebug(qWasmMediaAudioOutput) << "can play"; + emit readyChanged(true); + emit stateChanged(QWasmMediaPlayer::Preparing); + }; + m_canPlayChangeEvent.reset(new qstdweb::EventCallback(m_audio, "canplay", canPlayCallback)); + + // canplaythrough + auto canPlayThroughCallback = [&](emscripten::val event) { + Q_UNUSED(event) + emit stateChanged(QWasmMediaPlayer::Prepared); + }; + m_canPlayThroughChangeEvent.reset( + new qstdweb::EventCallback(m_audio, "canplaythrough", canPlayThroughCallback)); + + // play + auto playCallback = [&](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaAudioOutput) << "play"; + emit stateChanged(QWasmMediaPlayer::Started); + }; + m_playEvent.reset(new qstdweb::EventCallback(m_audio, "play", playCallback)); + + // durationchange + auto durationChangeCallback = [&](emscripten::val event) { + qCDebug(qWasmMediaAudioOutput) << "durationChange"; + + // duration in ms + emit durationChanged(event["target"]["duration"].as<double>() * 1000); + }; + m_durationChangeEvent.reset( + new qstdweb::EventCallback(m_audio, "durationchange", durationChangeCallback)); + + // ended + auto endedCallback = [&](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaAudioOutput) << "ended"; + m_currentMediaStatus = QMediaPlayer::EndOfMedia; + emit statusChanged(m_currentMediaStatus); + }; + m_endedEvent.reset(new qstdweb::EventCallback(m_audio, "ended", endedCallback)); + + // progress (buffering progress) + auto progesssCallback = [&](emscripten::val event) { + if (event.isUndefined() || event.isNull()) + return; + qCDebug(qWasmMediaAudioOutput) << "progress"; + float duration = event["target"]["duration"].as<int>(); + if (duration < 0) // track not exactly ready yet + return; + + emscripten::val timeRanges = event["target"]["buffered"]; + + if ((!timeRanges.isNull() || !timeRanges.isUndefined()) + && timeRanges["length"].as<int>() == 1) { + emscripten::val dVal = timeRanges.call<emscripten::val>("end", 0); + + if (!dVal.isNull() || !dVal.isUndefined()) { + double bufferedEnd = dVal.as<double>(); + + if (duration > 0 && bufferedEnd > 0) { + float bufferedValue = (bufferedEnd / duration * 100); + qCDebug(qWasmMediaAudioOutput) << "progress buffered" << bufferedValue; + + emit bufferingChanged(m_currentBufferedValue); + if (bufferedEnd == duration) + m_currentMediaStatus = QMediaPlayer::BufferedMedia; + else + m_currentMediaStatus = QMediaPlayer::BufferingMedia; + + emit statusChanged(m_currentMediaStatus); + } + } + } + }; + m_progressChangeEvent.reset(new qstdweb::EventCallback(m_audio, "progress", progesssCallback)); + + // timupdate + auto timeUpdateCallback = [&](emscripten::val event) { + qCDebug(qWasmMediaAudioOutput) + << "timeupdate" << (event["target"]["currentTime"].as<double>() * 1000); + + // qt progress is ms + emit progressChanged(event["target"]["currentTime"].as<double>() * 1000); + }; + m_timeUpdateEvent.reset(new qstdweb::EventCallback(m_audio, "timeupdate", timeUpdateCallback)); + + // pause + auto pauseCallback = [&](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaAudioOutput) << "pause"; + + int currentTime = m_audio["currentTime"].as<int>(); // in seconds + int duration = m_audio["duration"].as<int>(); // in seconds + if ((currentTime > 0 && currentTime < duration)) { + emit stateChanged(QWasmMediaPlayer::Paused); + } else { + emit stateChanged(QWasmMediaPlayer::Stopped); + } + }; + m_pauseChangeEvent.reset(new qstdweb::EventCallback(m_audio, "pause", pauseCallback)); +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/wasm/common/qwasmaudiooutput_p.h b/src/plugins/multimedia/wasm/common/qwasmaudiooutput_p.h new file mode 100644 index 000000000..69fda120b --- /dev/null +++ b/src/plugins/multimedia/wasm/common/qwasmaudiooutput_p.h @@ -0,0 +1,97 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMAUDIOOUTPUT_H +#define QWASMAUDIOOUTPUT_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformaudiooutput_p.h> +#include "qwasmmediaplayer_p.h" + +#include <emscripten/val.h> +#include <private/qstdweb_p.h> +#include <private/qwasmaudiosink_p.h> +#include <QIODevice> +#include <QObject> + +QT_BEGIN_NAMESPACE + +class Q_MULTIMEDIA_EXPORT QWasmAudioOutput : public QObject, public QPlatformAudioOutput +{ + Q_OBJECT + +public: + QWasmAudioOutput(QAudioOutput *qq); + ~QWasmAudioOutput(); + + enum AudioElementError { + MEDIA_ERR_ABORTED = 1, + MEDIA_ERR_NETWORK, + MEDIA_ERR_DECODE, + MEDIA_ERR_SRC_NOT_SUPPORTED + }; + + void setAudioDevice(const QAudioDevice &device) final; + void setMuted(bool muted) override; + void setVolume(float volume) override; + + void start(); + void stop(); + void pause(); + + void setSource(const QUrl &url); + void setSource(QIODevice *stream); + void setVideoElement(emscripten::val videoElement); + +Q_SIGNALS: + void readyChanged(bool); + void bufferingChanged(qint32 percent); + void errorOccured(qint32 code, const QString &message); + void stateChanged(QWasmMediaPlayer::QWasmMediaPlayerState newState); + void progressChanged(qint32 position); + void durationChanged(qint64 duration); + void statusChanged(QMediaPlayer::MediaStatus status); + void sizeChange(qint32 width, qint32 height); + void metaDataLoaded(); + +private: + void doElementCallbacks(); + void createAudioElement(const std::string &id); + + emscripten::val videoElement(); + + QScopedPointer<QWasmAudioSink> m_sink; + QScopedPointer<qstdweb::EventCallback> m_playEvent; + QScopedPointer<qstdweb::EventCallback> m_endedEvent; + QScopedPointer<qstdweb::EventCallback> m_durationChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_errorChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_canPlayChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_canPlayThroughChangeEvent; + + QScopedPointer<qstdweb::EventCallback> m_playingChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_progressChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_pauseChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_timeUpdateEvent; + QScopedPointer<qstdweb::EventCallback> m_loadedDataEvent; + + QString m_source; + QIODevice *m_audioIODevice = nullptr; + emscripten::val m_audio = emscripten::val::undefined(); + emscripten::val m_videoElement = emscripten::val::undefined(); + QMediaPlayer::MediaStatus m_currentMediaStatus; + qreal m_currentBufferedValue; +}; + +QT_END_NAMESPACE + +#endif // QWASMAUDIOOUTPUT_H diff --git a/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp b/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp new file mode 100644 index 000000000..84d325635 --- /dev/null +++ b/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp @@ -0,0 +1,1071 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include <QDebug> +#include <QUrl> +#include <QPoint> +#include <QRect> +#include <QMediaPlayer> +#include <QVideoFrame> +#include <QFile> +#include <QBuffer> +#include <QMimeDatabase> +#include "qwasmvideooutput_p.h" + +#include <qvideosink.h> +#include <private/qplatformvideosink_p.h> +#include <private/qmemoryvideobuffer_p.h> +#include <private/qvideotexturehelper_p.h> +#include <private/qvideoframe_p.h> +#include <private/qstdweb_p.h> +#include <QTimer> + +#include <emscripten/bind.h> +#include <emscripten/html5.h> +#include <emscripten/val.h> + + +QT_BEGIN_NAMESPACE + + +using namespace emscripten; + +Q_LOGGING_CATEGORY(qWasmMediaVideoOutput, "qt.multimedia.wasm.videooutput") + +// TODO unique videosurface ? +static std::string m_videoSurfaceId; + +void qtVideoBeforeUnload(emscripten::val event) +{ + Q_UNUSED(event) + // large videos will leave the unloading window + // in a frozen state, so remove the video element first + emscripten::val document = emscripten::val::global("document"); + emscripten::val videoElement = + document.call<emscripten::val>("getElementById", std::string(m_videoSurfaceId)); + videoElement.call<void>("removeAttribute", emscripten::val("src")); + videoElement.call<void>("load"); +} + +EMSCRIPTEN_BINDINGS(video_module) +{ + emscripten::function("mbeforeUnload", qtVideoBeforeUnload); +} + +static bool checkForVideoFrame() +{ + emscripten::val videoFrame = emscripten::val::global("VideoFrame"); + return (!videoFrame.isNull() && !videoFrame.isUndefined()); +} + +QWasmVideoOutput::QWasmVideoOutput(QObject *parent) : QObject{ parent } +{ + m_hasVideoFrame = checkForVideoFrame(); +} + +void QWasmVideoOutput::setVideoSize(const QSize &newSize) +{ + if (m_pendingVideoSize == newSize) + return; + + m_pendingVideoSize = newSize; + updateVideoElementGeometry(QRect(0, 0, m_pendingVideoSize.width(), m_pendingVideoSize.height())); +} + +void QWasmVideoOutput::setVideoMode(QWasmVideoOutput::WasmVideoMode mode) +{ + m_currentVideoMode = mode; +} + +void QWasmVideoOutput::start() +{ + if (m_video.isUndefined() || m_video.isNull() + || !m_wasmSink) { + // error + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + return; + } + + switch (m_currentVideoMode) { + case QWasmVideoOutput::VideoOutput: { + emscripten::val sourceObj = m_video["src"]; + if ((sourceObj.isUndefined() || sourceObj.isNull()) && !m_source.isEmpty()) { + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "calling load" << m_source; + m_video.set("src", m_source); + m_video.call<void>("load"); + } + } break; + case QWasmVideoOutput::Camera: { + if (!m_cameraIsReady) { + m_shouldBeStarted = true; + } + + emscripten::val stream = m_video["srcObject"]; + if (stream.isNull() || stream.isUndefined()) { // camera device + qCDebug(qWasmMediaVideoOutput) << "ERROR"; + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + return; + } else { + emscripten::val videoTracks = stream.call<emscripten::val>("getVideoTracks"); + if (videoTracks.isNull() || videoTracks.isUndefined()) { + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "videoTracks is null"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("video surface error")); + return; + } + if (videoTracks["length"].as<int>() == 0) { + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << "videoTracks count is 0"; + emit errorOccured(QMediaPlayer::ResourceError, + QStringLiteral("video surface error")); + return; + } + emscripten::val videoSettings = videoTracks[0].call<emscripten::val>("getSettings"); + if (!videoSettings.isNull() || !videoSettings.isUndefined()) { + // double fRate = videoSettings["frameRate"].as<double>(); TODO + const int width = videoSettings["width"].as<int>(); + const int height = videoSettings["height"].as<int>(); + + qCDebug(qWasmMediaVideoOutput) + << "width" << width << "height" << height; + + updateVideoElementGeometry(QRect(0, 0, width, height)); + } + } + } break; + }; + + m_shouldStop = false; + m_toBePaused = false; + m_video.call<void>("play"); + + if (m_currentVideoMode == QWasmVideoOutput::Camera) { + if (m_hasVideoFrame) { + m_video.call<emscripten::val>("requestVideoFrameCallback", + emscripten::val::module_property("qtVideoFrameTimerCallback")); + } else { + videoFrameTimerCallback(); + } + } +} + +void QWasmVideoOutput::stop() +{ + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO; + + if (m_video.isUndefined() || m_video.isNull()) { + // error + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + m_shouldStop = true; + if (m_toBePaused) { + // we are stopped , need to reset + m_toBePaused = false; + m_video.call<void>("load"); + } else { + m_video.call<void>("pause"); + } +} + +void QWasmVideoOutput::pause() +{ + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO; + + if (m_video.isUndefined() || m_video.isNull()) { + // error + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + return; + } + m_shouldStop = false; + m_toBePaused = true; + m_video.call<void>("pause"); +} + +void QWasmVideoOutput::reset() +{ + // flush pending frame + if (m_wasmSink) + m_wasmSink->platformVideoSink()->setVideoFrame(QVideoFrame()); + + m_source = ""; + m_video.set("currentTime", emscripten::val(0)); + m_video.call<void>("load"); +} + +emscripten::val QWasmVideoOutput::surfaceElement() +{ + return m_video; +} + +void QWasmVideoOutput::setSurface(QVideoSink *surface) +{ + if (!surface || surface == m_wasmSink) { + qWarning() << "Surface not ready"; + return; + } + + m_wasmSink = surface; +} + +bool QWasmVideoOutput::isReady() const +{ + if (m_video.isUndefined() || m_video.isNull()) { + // error + return false; + } + + constexpr int hasCurrentData = 2; + if (!m_video.isUndefined() || !m_video.isNull()) + return m_video["readyState"].as<int>() >= hasCurrentData; + else + return true; +} + +void QWasmVideoOutput::setSource(const QUrl &url) +{ + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << url; + + if (m_video.isUndefined() || m_video.isNull()) { + return; + } + + m_source = url.toString(); + if (url.isEmpty()) { + stop(); + return; + } + if (url.isLocalFile()) { + QFile localFile(url.toLocalFile()); + if (localFile.open(QIODevice::ReadOnly)) { + QDataStream buffer(&localFile); // we will serialize the data into the file + setSource(buffer.device()); + } else { + qWarning() << "Failed to open file"; + } + return; + } + + updateVideoElementSource(m_source); +} + +void QWasmVideoOutput::updateVideoElementSource(const QString &src) +{ + m_video.set("src", src.toStdString()); + m_video.call<void>("load"); +} + +void QWasmVideoOutput::addCameraSourceElement(const std::string &id) +{ + m_cameraIsReady = false; + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + if (mediaDevices.isNull() || mediaDevices.isUndefined()) { + qWarning() << "No media devices found"; + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("Resource error")); + return; + } + + qstdweb::PromiseCallbacks getUserMediaCallback{ + .thenFunc = + [this](emscripten::val stream) { + qCDebug(qWasmMediaVideoOutput) << "getUserMediaSuccess"; + + m_video.set("srcObject", stream); + m_cameraIsReady = true; + if (m_shouldBeStarted) { + start(); + m_shouldBeStarted = false; + } + }, + .catchFunc = + [](emscripten::val error) { + qCDebug(qWasmMediaVideoOutput) + << "getUserMedia fail" + << QString::fromStdString(error["name"].as<std::string>()) + << QString::fromStdString(error["message"].as<std::string>()); + } + }; + + emscripten::val constraints = emscripten::val::object(); + + constraints.set("audio", m_hasAudio); + + emscripten::val videoContraints = emscripten::val::object(); + videoContraints.set("exact", id); + videoContraints.set("deviceId", id); + constraints.set("video", videoContraints); + + // we do it this way as this prompts user for mic/camera permissions + qstdweb::Promise::make(mediaDevices, QStringLiteral("getUserMedia"), + std::move(getUserMediaCallback), constraints); +} + +void QWasmVideoOutput::setSource(QIODevice *stream) +{ + if (stream->bytesAvailable() == 0) { + qWarning() << "data not available"; + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("data not available")); + return; + } + if (m_video.isUndefined() || m_video.isNull()) { + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + return; + } + + QMimeDatabase db; + QMimeType mime = db.mimeTypeForData(stream); + + QByteArray buffer = stream->readAll(); + + qstdweb::Blob contentBlob = qstdweb::Blob::copyFrom(buffer.data(), buffer.size(), mime.name().toStdString()); + + emscripten::val window = qstdweb::window(); + + if (window["safari"].isUndefined()) { + emscripten::val contentUrl = window["URL"].call<emscripten::val>("createObjectURL", contentBlob.val()); + m_video.set("src", contentUrl); + m_source = QString::fromStdString(contentUrl.as<std::string>()); + } else { + // only Safari currently supports Blob with srcObject + m_video.set("srcObject", contentBlob.val()); + } +} + +void QWasmVideoOutput::setVolume(qreal volume) +{ // between 0 - 1 + volume = qBound(qreal(0.0), volume, qreal(1.0)); + m_video.set("volume", volume); +} + +void QWasmVideoOutput::setMuted(bool muted) +{ + if (m_video.isUndefined() || m_video.isNull()) { + // error + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + return; + } + m_video.set("muted", muted); +} + +qint64 QWasmVideoOutput::getCurrentPosition() +{ + return (!m_video.isUndefined() || !m_video.isNull()) + ? (m_video["currentTime"].as<double>() * 1000) + : 0; +} + +void QWasmVideoOutput::seekTo(qint64 positionMSecs) +{ + if (isVideoSeekable()) { + float positionToSetInSeconds = float(positionMSecs) / 1000; + emscripten::val seekableTimeRange = m_video["seekable"]; + if (!seekableTimeRange.isNull() || !seekableTimeRange.isUndefined()) { + // range user can seek + if (seekableTimeRange["length"].as<int>() < 1) + return; + if (positionToSetInSeconds + >= seekableTimeRange.call<emscripten::val>("start", 0).as<double>() + && positionToSetInSeconds + <= seekableTimeRange.call<emscripten::val>("end", 0).as<double>()) { + m_requestedPosition = positionToSetInSeconds; + + m_video.set("currentTime", m_requestedPosition); + } + } + } + qCDebug(qWasmMediaVideoOutput) << "m_requestedPosition" << m_requestedPosition; +} + +bool QWasmVideoOutput::isVideoSeekable() +{ + if (m_video.isUndefined() || m_video.isNull()) { + // error + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("video surface error")); + return false; + } + + emscripten::val seekableTimeRange = m_video["seekable"]; + if (seekableTimeRange["length"].as<int>() < 1) + return false; + if (!seekableTimeRange.isNull() || !seekableTimeRange.isUndefined()) { + bool isit = !qFuzzyCompare(seekableTimeRange.call<emscripten::val>("start", 0).as<double>(), + seekableTimeRange.call<emscripten::val>("end", 0).as<double>()); + return isit; + } + return false; +} + +void QWasmVideoOutput::createVideoElement(const std::string &id) +{ + // TODO: there can be more than one element !! + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << id; + // Create <video> element and add it to the page body + emscripten::val document = emscripten::val::global("document"); + emscripten::val body = document["body"]; + + emscripten::val oldVideo = document.call<emscripten::val>("getElementsByClassName", + (m_currentVideoMode == QWasmVideoOutput::Camera + ? std::string("Camera") + : std::string("Video"))); + + // we don't provide alternate tracks + // but need to remove stale track + if (oldVideo["length"].as<int>() > 0) + oldVideo[0].call<void>("remove"); + + m_videoSurfaceId = id; + m_video = document.call<emscripten::val>("createElement", std::string("video")); + + m_video.set("id", m_videoSurfaceId.c_str()); + m_video.call<void>("setAttribute", std::string("class"), + (m_currentVideoMode == QWasmVideoOutput::Camera ? std::string("Camera") + : std::string("Video"))); + m_video.set("data-qvideocontext", + emscripten::val(quintptr(reinterpret_cast<void *>(this)))); + + // if video + m_video.set("preload", "metadata"); + + // Uncaught DOMException: Failed to execute 'getImageData' on + // 'OffscreenCanvasRenderingContext2D': The canvas has been tainted by + // cross-origin data. + // TODO figure out somehow to let user choose between these + std::string originString = "anonymous"; // requires server Access-Control-Allow-Origin * + // std::string originString = "use-credentials"; // must not + // Access-Control-Allow-Origin * + + m_video.call<void>("setAttribute", std::string("crossorigin"), originString); + body.call<void>("appendChild", m_video); + + // Create/add video source + document.call<emscripten::val>("createElement", + std::string("source")).set("src", m_source.toStdString()); + + // Set position:absolute, which makes it possible to position the video + // element using x,y. coordinates, relative to its parent (the page's <body> + // element) + emscripten::val style = m_video["style"]; + style.set("position", "absolute"); + style.set("display", "none"); // hide + + if (!m_source.isEmpty()) + updateVideoElementSource(m_source); +} + +void QWasmVideoOutput::createOffscreenElement(const QSize &offscreenSize) +{ + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO; + + if (m_hasVideoFrame) // VideoFrame does not require offscreen canvas/context + return; + + // create offscreen element for grabbing frames + // OffscreenCanvas - no safari :( + // https://developer.mozilla.org/en-US/docs/Web/API/OffscreenCanvas + + emscripten::val document = emscripten::val::global("document"); + + // TODO use correct frameBytesAllocationSize? + // offscreen render buffer + m_offscreen = emscripten::val::global("OffscreenCanvas"); + + if (m_offscreen.isUndefined()) { + // Safari OffscreenCanvas not supported, try old skool way + m_offscreen = document.call<emscripten::val>("createElement", std::string("canvas")); + + m_offscreen.set("style", + "position:absolute;left:-1000px;top:-1000px"); // offscreen + m_offscreen.set("width", offscreenSize.width()); + m_offscreen.set("height", offscreenSize.height()); + m_offscreenContext = m_offscreen.call<emscripten::val>("getContext", std::string("2d")); + } else { + m_offscreen = emscripten::val::global("OffscreenCanvas") + .new_(offscreenSize.width(), offscreenSize.height()); + emscripten::val offscreenAttributes = emscripten::val::array(); + offscreenAttributes.set("willReadFrequently", true); + m_offscreenContext = m_offscreen.call<emscripten::val>("getContext", std::string("2d"), + offscreenAttributes); + } + std::string offscreenId = m_videoSurfaceId + "_offscreenOutputSurface"; + m_offscreen.set("id", offscreenId.c_str()); +} + +void QWasmVideoOutput::doElementCallbacks() +{ + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO; + + // event callbacks + // timupdate + auto timeUpdateCallback = [=](emscripten::val event) { + qCDebug(qWasmMediaVideoOutput) << "timeupdate"; + + // qt progress is ms + emit progressChanged(event["target"]["currentTime"].as<double>() * 1000); + }; + m_timeUpdateEvent.reset(new qstdweb::EventCallback(m_video, "timeupdate", timeUpdateCallback)); + + // play + auto playCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "play" << m_video["src"].as<std::string>(); + if (!m_isSeeking) + emit stateChanged(QWasmMediaPlayer::Preparing); + }; + m_playEvent.reset(new qstdweb::EventCallback(m_video, "play", playCallback)); + + // ended + auto endedCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "ended"; + m_currentMediaStatus = QMediaPlayer::EndOfMedia; + emit statusChanged(m_currentMediaStatus); + m_shouldStop = true; + stop(); + }; + m_endedEvent.reset(new qstdweb::EventCallback(m_video, "ended", endedCallback)); + + // durationchange + auto durationChangeCallback = [=](emscripten::val event) { + qCDebug(qWasmMediaVideoOutput) << "durationChange"; + + // qt duration is in milliseconds. + qint64 dur = event["target"]["duration"].as<double>() * 1000; + emit durationChanged(dur); + }; + m_durationChangeEvent.reset( + new qstdweb::EventCallback(m_video, "durationchange", durationChangeCallback)); + + // loadeddata + auto loadedDataCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "loaded data"; + + emit stateChanged(QWasmMediaPlayer::Prepared); + }; + m_loadedDataEvent.reset(new qstdweb::EventCallback(m_video, "loadeddata", loadedDataCallback)); + + // error + auto errorCallback = [=](emscripten::val event) { + qCDebug(qWasmMediaVideoOutput) << "error"; + if (event.isUndefined() || event.isNull()) + return; + emit errorOccured(m_video["error"]["code"].as<int>(), + QString::fromStdString(m_video["error"]["message"].as<std::string>())); + }; + m_errorChangeEvent.reset(new qstdweb::EventCallback(m_video, "error", errorCallback)); + + // resize + auto resizeCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "resize"; + + updateVideoElementGeometry( + QRect(0, 0, m_video["videoWidth"].as<int>(), m_video["videoHeight"].as<int>())); + emit sizeChange(m_video["videoWidth"].as<int>(), m_video["videoHeight"].as<int>()); + + }; + m_resizeChangeEvent.reset(new qstdweb::EventCallback(m_video, "resize", resizeCallback)); + + // loadedmetadata + auto loadedMetadataCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "loaded meta data"; + + emit metaDataLoaded(); + }; + m_loadedMetadataChangeEvent.reset( + new qstdweb::EventCallback(m_video, "loadedmetadata", loadedMetadataCallback)); + + // loadstart + auto loadStartCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "load started"; + m_currentMediaStatus = QMediaPlayer::LoadingMedia; + emit statusChanged(m_currentMediaStatus); + m_shouldStop = false; + }; + m_loadStartChangeEvent.reset(new qstdweb::EventCallback(m_video, "loadstart", loadStartCallback)); + + // canplay + + auto canPlayCallback = [=](emscripten::val event) { + if (event.isUndefined() || event.isNull()) + return; + qCDebug(qWasmMediaVideoOutput) << "can play" + << "m_requestedPosition" << m_requestedPosition; + + if (!m_shouldStop) + emit readyChanged(true); // sets video available + }; + m_canPlayChangeEvent.reset(new qstdweb::EventCallback(m_video, "canplay", canPlayCallback)); + + // canplaythrough + auto canPlayThroughCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "can play through" + << "m_shouldStop" << m_shouldStop; + + if (m_currentMediaStatus == QMediaPlayer::EndOfMedia) + return; + if (!m_isSeeking && !m_shouldStop) { + emscripten::val timeRanges = m_video["buffered"]; + if ((!timeRanges.isNull() || !timeRanges.isUndefined()) + && timeRanges["length"].as<int>() == 1) { + double buffered = m_video["buffered"].call<emscripten::val>("end", 0).as<double>(); + const double duration = m_video["duration"].as<double>(); + + if (duration == buffered) { + m_currentBufferedValue = 100; + emit bufferingChanged(m_currentBufferedValue); + } + } + m_currentMediaStatus = QMediaPlayer::LoadedMedia; + emit statusChanged(m_currentMediaStatus); + if (m_hasVideoFrame) { + m_video.call<emscripten::val>("requestVideoFrameCallback", + emscripten::val::module_property("qtVideoFrameTimerCallback")); + } else { + videoFrameTimerCallback(); + } + } else { + m_shouldStop = false; + } + }; + m_canPlayThroughChangeEvent.reset( + new qstdweb::EventCallback(m_video, "canplaythrough", canPlayThroughCallback)); + + // seeking + auto seekingCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) + << "seeking started" << (m_video["currentTime"].as<double>() * 1000); + m_isSeeking = true; + }; + m_seekingChangeEvent.reset(new qstdweb::EventCallback(m_video, "seeking", seekingCallback)); + + // seeked + auto seekedCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "seeked" << (m_video["currentTime"].as<double>() * 1000); + emit progressChanged(m_video["currentTime"].as<double>() * 1000); + m_isSeeking = false; + }; + m_seekedChangeEvent.reset(new qstdweb::EventCallback(m_video, "seeked", seekedCallback)); + + // emptied + auto emptiedCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "emptied"; + emit readyChanged(false); + m_currentMediaStatus = QMediaPlayer::EndOfMedia; + emit statusChanged(m_currentMediaStatus); + }; + m_emptiedChangeEvent.reset(new qstdweb::EventCallback(m_video, "emptied", emptiedCallback)); + + // stalled + auto stalledCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "stalled"; + m_currentMediaStatus = QMediaPlayer::StalledMedia; + emit statusChanged(m_currentMediaStatus); + }; + m_stalledChangeEvent.reset(new qstdweb::EventCallback(m_video, "stalled", stalledCallback)); + + // waiting + auto waitingCallback = [=](emscripten::val event) { + Q_UNUSED(event) + + qCDebug(qWasmMediaVideoOutput) << "waiting"; + // check buffer + }; + m_waitingChangeEvent.reset(new qstdweb::EventCallback(m_video, "waiting", waitingCallback)); + + // suspend + + // playing + auto playingCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "playing"; + if (m_isSeeking) + return; + emit stateChanged(QWasmMediaPlayer::Started); + if (m_toBePaused || !m_shouldStop) { // paused + m_toBePaused = false; + + if (m_hasVideoFrame) { + m_video.call<emscripten::val>("requestVideoFrameCallback", + emscripten::val::module_property("qtVideoFrameTimerCallback")); + } else { + videoFrameTimerCallback(); // get the ball rolling + } + } + }; + m_playingChangeEvent.reset(new qstdweb::EventCallback(m_video, "playing", playingCallback)); + + // progress (buffering progress) + auto progesssCallback = [=](emscripten::val event) { + if (event.isUndefined() || event.isNull()) + return; + + const double duration = event["target"]["duration"].as<double>(); + if (duration < 0) // track not exactly ready yet + return; + + emscripten::val timeRanges = event["target"]["buffered"]; + + if ((!timeRanges.isNull() || !timeRanges.isUndefined()) + && timeRanges["length"].as<int>() == 1) { + emscripten::val dVal = timeRanges.call<emscripten::val>("end", 0); + if (!dVal.isNull() || !dVal.isUndefined()) { + double bufferedEnd = dVal.as<double>(); + + if (duration > 0 && bufferedEnd > 0) { + const double bufferedValue = (bufferedEnd / duration * 100); + qCDebug(qWasmMediaVideoOutput) << "progress buffered"; + m_currentBufferedValue = bufferedValue; + emit bufferingChanged(m_currentBufferedValue); + if (bufferedEnd == duration) + m_currentMediaStatus = QMediaPlayer::BufferedMedia; + else + m_currentMediaStatus = QMediaPlayer::BufferingMedia; + emit statusChanged(m_currentMediaStatus); + } + } + } + }; + m_progressChangeEvent.reset(new qstdweb::EventCallback(m_video, "progress", progesssCallback)); + + // pause + auto pauseCallback = [=](emscripten::val event) { + Q_UNUSED(event) + qCDebug(qWasmMediaVideoOutput) << "pause"; + + const double currentTime = m_video["currentTime"].as<double>(); // in seconds + const double duration = m_video["duration"].as<double>(); // in seconds + if ((currentTime > 0 && currentTime < duration) && (!m_shouldStop && m_toBePaused)) { + emit stateChanged(QWasmMediaPlayer::Paused); + } else { + // stop this crazy thing! + m_video.set("currentTime", emscripten::val(0)); + emit stateChanged(QWasmMediaPlayer::Stopped); + } + }; + m_pauseChangeEvent.reset(new qstdweb::EventCallback(m_video, "pause", pauseCallback)); + + // onunload + // we use lower level events here as to avert a crash on activate using the + // qtdweb see _qt_beforeUnload + emscripten::val window = emscripten::val::global("window"); + window.call<void>("addEventListener", std::string("beforeunload"), + emscripten::val::module_property("mbeforeUnload")); +} + +void QWasmVideoOutput::updateVideoElementGeometry(const QRect &windowGeometry) +{ + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO << windowGeometry; + QRect m_videoElementSource(windowGeometry.topLeft(), windowGeometry.size()); + + emscripten::val style = m_video["style"]; + style.set("left", QString("%1px").arg(m_videoElementSource.left()).toStdString()); + style.set("top", QString("%1px").arg(m_videoElementSource.top()).toStdString()); + style.set("width", QString("%1px").arg(m_videoElementSource.width()).toStdString()); + style.set("height", QString("%1px").arg(m_videoElementSource.height()).toStdString()); + style.set("z-index", "999"); + + if (!m_hasVideoFrame) { + // offscreen + m_offscreen.set("width", m_videoElementSource.width()); + m_offscreen.set("height", m_videoElementSource.height()); + } +} + +qint64 QWasmVideoOutput::getDuration() +{ + // qt duration is in ms + // js is sec + + if (m_video.isUndefined() || m_video.isNull()) + return 0; + return m_video["duration"].as<double>() * 1000; +} + +void QWasmVideoOutput::newFrame(const QVideoFrame &frame) +{ + m_wasmSink->setVideoFrame(frame); +} + +void QWasmVideoOutput::setPlaybackRate(qreal rate) +{ + m_video.set("playbackRate", emscripten::val(rate)); +} + +qreal QWasmVideoOutput::playbackRate() +{ + return (m_video.isUndefined() || m_video.isNull()) ? 0 : m_video["playbackRate"].as<float>(); +} + +void QWasmVideoOutput::checkNetworkState() +{ + int netState = m_video["networkState"].as<int>(); + + qCDebug(qWasmMediaVideoOutput) << netState; + + switch (netState) { + case QWasmMediaPlayer::QWasmMediaNetworkState::NetworkEmpty: // no data + break; + case QWasmMediaPlayer::QWasmMediaNetworkState::NetworkIdle: + break; + case QWasmMediaPlayer::QWasmMediaNetworkState::NetworkLoading: + break; + case QWasmMediaPlayer::QWasmMediaNetworkState::NetworkNoSource: // no source + emit errorOccured(netState, QStringLiteral("No media source found")); + break; + }; +} + +void QWasmVideoOutput::videoComputeFrame(void *context) +{ + if (m_offscreenContext.isUndefined() || m_offscreenContext.isNull()) { + qCDebug(qWasmMediaVideoOutput) << "offscreen canvas context could not be found"; + return; + } + emscripten::val document = emscripten::val::global("document"); + + emscripten::val videoElement = + document.call<emscripten::val>("getElementById", std::string(m_videoSurfaceId)); + + if (videoElement.isUndefined() || videoElement.isNull()) { + qCDebug(qWasmMediaVideoOutput) << "video element could not be found"; + return; + } + + const int videoWidth = videoElement["videoWidth"].as<int>(); + const int videoHeight = videoElement["videoHeight"].as<int>(); + + if (videoWidth == 0 || videoHeight == 0) + return; + + m_offscreenContext.call<void>("drawImage", videoElement, 0, 0, videoWidth, videoHeight); + + emscripten::val frame = // one frame, Uint8ClampedArray + m_offscreenContext.call<emscripten::val>("getImageData", 0, 0, videoWidth, videoHeight); + + const QSize frameBytesAllocationSize(videoWidth, videoHeight); + + // this seems to work ok, even though getImageData returns a Uint8ClampedArray + QByteArray frameBytes = qstdweb::Uint8Array(frame["data"]).copyToQByteArray(); + + QVideoFrameFormat frameFormat = + QVideoFrameFormat(frameBytesAllocationSize, QVideoFrameFormat::Format_RGBA8888); + + auto *textureDescription = QVideoTextureHelper::textureDescription(frameFormat.pixelFormat()); + + QVideoFrame vFrame = QVideoFramePrivate::createFrame( + std::make_unique<QMemoryVideoBuffer>( + std::move(frameBytes), + textureDescription->strideForWidth(frameFormat.frameWidth())), + frameFormat); + QWasmVideoOutput *wasmVideoOutput = reinterpret_cast<QWasmVideoOutput *>(context); + + if (!wasmVideoOutput->m_wasmSink) { + qWarning() << "ERROR ALERT!! video sink not set"; + } + wasmVideoOutput->m_wasmSink->setVideoFrame(vFrame); +} + + +void QWasmVideoOutput::videoFrameCallback(emscripten::val now, emscripten::val metadata) +{ + Q_UNUSED(now) + Q_UNUSED(metadata) + + emscripten::val videoElement = + emscripten::val::global("document"). + call<emscripten::val>("getElementById", + std::string(m_videoSurfaceId)); + + emscripten::val oneVideoFrame = val::global("VideoFrame").new_(videoElement); + + if (oneVideoFrame.isNull() || oneVideoFrame.isUndefined()) { + qCDebug(qWasmMediaVideoOutput) << Q_FUNC_INFO + << "ERROR" << "failed to construct VideoFrame"; + return; + } + emscripten::val frameBytesAllocationSize = oneVideoFrame.call<emscripten::val>("allocationSize"); + + emscripten::val frameBuffer = + emscripten::val::global("Uint8Array").new_(frameBytesAllocationSize); + QWasmVideoOutput *wasmVideoOutput = + reinterpret_cast<QWasmVideoOutput*>(videoElement["data-qvideocontext"].as<quintptr>()); + + qstdweb::PromiseCallbacks copyToCallback; + copyToCallback.thenFunc = [wasmVideoOutput, oneVideoFrame, frameBuffer, videoElement] + (emscripten::val frameLayout) + { + if (frameLayout.isNull() || frameLayout.isUndefined()) { + qCDebug(qWasmMediaVideoOutput) << "theres no frameLayout"; + return; + } + + // frameBuffer now has a new frame, send to Qt + const QSize frameSize(oneVideoFrame["displayWidth"].as<int>(), + oneVideoFrame["displayHeight"].as<int>()); + + + QByteArray frameBytes = QByteArray::fromEcmaUint8Array(frameBuffer); + + QVideoFrameFormat::PixelFormat pixelFormat = fromJsPixelFormat(oneVideoFrame["format"].as<std::string>()); + if (pixelFormat == QVideoFrameFormat::Format_Invalid) { + qWarning() << "Invalid pixel format"; + return; + } + QVideoFrameFormat frameFormat = QVideoFrameFormat(frameSize, pixelFormat); + + auto *textureDescription = QVideoTextureHelper::textureDescription(frameFormat.pixelFormat()); + + auto buffer = std::make_unique<QMemoryVideoBuffer>( + std::move(frameBytes), + textureDescription->strideForWidth(frameFormat.frameWidth())); + + QVideoFrame vFrame = + QVideoFramePrivate::createFrame(std::move(buffer), std::move(frameFormat)); + + if (!wasmVideoOutput) { + qCDebug(qWasmMediaVideoOutput) << "ERROR:" + << "data-qvideocontext not found"; + return; + } + if (!wasmVideoOutput->m_wasmSink) { + qWarning() << "ERROR ALERT!! video sink not set"; + return; + } + wasmVideoOutput->m_wasmSink->setVideoFrame(vFrame); + oneVideoFrame.call<emscripten::val>("close"); + }; + copyToCallback.catchFunc = [&, wasmVideoOutput, oneVideoFrame, videoElement](emscripten::val error) + { + qCDebug(qWasmMediaVideoOutput) << "Error" + << QString::fromStdString(error["name"].as<std::string>()) + << QString::fromStdString(error["message"].as<std::string>()) ; + + oneVideoFrame.call<emscripten::val>("close"); + wasmVideoOutput->stop(); + return; + }; + + qstdweb::Promise::make(oneVideoFrame, "copyTo", std::move(copyToCallback), frameBuffer); + + videoElement.call<emscripten::val>("requestVideoFrameCallback", + emscripten::val::module_property("qtVideoFrameTimerCallback")); + +} + +void QWasmVideoOutput::videoFrameTimerCallback() +{ + static auto frame = [](double frameTime, void *context) -> int { + Q_UNUSED(frameTime); + QWasmVideoOutput *videoOutput = reinterpret_cast<QWasmVideoOutput *>(context); + + emscripten::val document = emscripten::val::global("document"); + emscripten::val videoElement = + document.call<emscripten::val>("getElementById", std::string(m_videoSurfaceId)); + + if (videoElement["paused"].as<bool>() || videoElement["ended"].as<bool>()) + return false; + + videoOutput->videoComputeFrame(context); + + return true; + }; + + emscripten_request_animation_frame_loop(frame, this); + // about 60 fps +} + + +QVideoFrameFormat::PixelFormat QWasmVideoOutput::fromJsPixelFormat(std::string videoFormat) +{ + if (videoFormat == "I420") + return QVideoFrameFormat::Format_YUV420P; + // no equivalent pixel format + // else if (videoFormat == "I420A") + else if (videoFormat == "I422") + return QVideoFrameFormat::Format_YUV422P; + // no equivalent pixel format + // else if (videoFormat == "I444") + else if (videoFormat == "NV12") + return QVideoFrameFormat::Format_NV12; + else if (videoFormat == "RGBA") + return QVideoFrameFormat::Format_RGBA8888; + else if (videoFormat == "I420") + return QVideoFrameFormat::Format_YUV420P; + else if (videoFormat == "RGBX") + return QVideoFrameFormat::Format_RGBX8888; + else if (videoFormat == "BGRA") + return QVideoFrameFormat::Format_BGRA8888; + else if (videoFormat == "BGRX") + return QVideoFrameFormat::Format_BGRX8888; + + return QVideoFrameFormat::Format_Invalid; +} + + +emscripten::val QWasmVideoOutput::getDeviceCapabilities() +{ + emscripten::val stream = m_video["srcObject"]; + if (!stream.isUndefined() || !stream["getVideoTracks"].isUndefined()) { + emscripten::val tracks = stream.call<emscripten::val>("getVideoTracks"); + if (!tracks.isUndefined()) { + if (tracks["length"].as<int>() == 0) + return emscripten::val::undefined(); + + emscripten::val track = tracks[0]; + if (!track.isUndefined()) { + emscripten::val trackCaps = emscripten::val::undefined(); + if (!track["getCapabilities"].isUndefined()) + trackCaps = track.call<emscripten::val>("getCapabilities"); + else // firefox does not support getCapabilities + trackCaps = track.call<emscripten::val>("getSettings"); + + if (!trackCaps.isUndefined()) + return trackCaps; + } + } + } else { + // camera not started track capabilities not available + emit errorOccured(QMediaPlayer::ResourceError, QStringLiteral("capabilities not available")); + } + + return emscripten::val::undefined(); +} + +bool QWasmVideoOutput::setDeviceSetting(const std::string &key, emscripten::val value) +{ + emscripten::val stream = m_video["srcObject"]; + if (stream.isNull() || stream.isUndefined() + || stream["getVideoTracks"].isUndefined()) + return false; + + emscripten::val tracks = stream.call<emscripten::val>("getVideoTracks"); + if (!tracks.isNull() || !tracks.isUndefined()) { + if (tracks["length"].as<int>() == 0) + return false; + + emscripten::val track = tracks[0]; + emscripten::val contraint = emscripten::val::object(); + contraint.set(std::move(key), value); + track.call<emscripten::val>("applyConstraints", contraint); + return true; + } + + return false; +} + +EMSCRIPTEN_BINDINGS(qtwasmvideooutput) { + emscripten::function("qtVideoFrameTimerCallback", &QWasmVideoOutput::videoFrameCallback); +} + +QT_END_NAMESPACE + +#include "moc_qwasmvideooutput_p.cpp" diff --git a/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h b/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h new file mode 100644 index 000000000..f078ffb44 --- /dev/null +++ b/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h @@ -0,0 +1,153 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#ifndef QWASMVIDEOOUTPUT_H +#define QWASMVIDEOOUTPUT_H + +#include <QObject> + +#include <emscripten/val.h> +#include <QMediaPlayer> +#include <QVideoFrame> + +#include "qwasmmediaplayer_p.h" +#include <QtCore/qloggingcategory.h> + +#include <private/qstdweb_p.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmMediaVideoOutput) + +class QVideoSink; + +class QWasmVideoOutput : public QObject +{ + Q_OBJECT +public: + enum WasmVideoMode { VideoOutput, Camera }; + Q_ENUM(WasmVideoMode) + + explicit QWasmVideoOutput(QObject *parent = nullptr); + + void setVideoSize(const QSize &); + void start(); + void stop(); + void reset(); + void pause(); + + void setSurface(QVideoSink *surface); + emscripten::val surfaceElement(); + + bool isReady() const; + + void setSource(const QUrl &url); + void setSource(QIODevice *stream); + void setVolume(qreal volume); + void setMuted(bool muted); + + qint64 getCurrentPosition(); + void seekTo(qint64 position); + bool isVideoSeekable(); + void setPlaybackRate(qreal rate); + qreal playbackRate(); + + qint64 getDuration(); + void newFrame(const QVideoFrame &newFrame); + + void createVideoElement(const std::string &id); + void createOffscreenElement(const QSize &offscreenSize); + void doElementCallbacks(); + void updateVideoElementGeometry(const QRect &windowGeometry); + void updateVideoElementSource(const QString &src); + void addCameraSourceElement(const std::string &id); + void removeSourceElement(); + void setVideoMode(QWasmVideoOutput::WasmVideoMode mode); + + void setHasAudio(bool needsAudio) { m_hasAudio = needsAudio; } + + bool hasCapability(const QString &cap); + emscripten::val getDeviceCapabilities(); + bool setDeviceSetting(const std::string &key, emscripten::val value); + bool isCameraReady() { return m_cameraIsReady; } + bool m_hasVideoFrame = false; + + static void videoFrameCallback(emscripten::val now, emscripten::val metadata); + void videoFrameTimerCallback(); + // mediacapturesession has the videosink + QVideoSink *m_wasmSink = nullptr; + + emscripten::val currentVideoElement() { return m_video; } + +Q_SIGNALS: + void readyChanged(bool); + void bufferingChanged(qint32 percent); + void errorOccured(qint32 code, const QString &message); + void stateChanged(QWasmMediaPlayer::QWasmMediaPlayerState newState); + void progressChanged(qint32 position); + void durationChanged(qint64 duration); + void statusChanged(QMediaPlayer::MediaStatus status); + void sizeChange(qint32 width, qint32 height); + void metaDataLoaded(); + +private: + void checkNetworkState(); + void videoComputeFrame(void *context); + void getDeviceSettings(); + + static QVideoFrameFormat::PixelFormat fromJsPixelFormat(std::string videoFormat); + + emscripten::val m_video = emscripten::val::undefined(); + emscripten::val m_videoElementSource = emscripten::val::undefined(); + + QString m_source; + float m_requestedPosition = 0.0; + emscripten::val m_offscreen = emscripten::val::undefined(); + + bool m_shouldStop = false; + bool m_toBePaused = false; + bool m_isSeeking = false; + bool m_hasAudio = false; + bool m_cameraIsReady = false; + bool m_shouldBeStarted = false; + + emscripten::val m_offscreenContext = emscripten::val::undefined(); + QSize m_pendingVideoSize; + QWasmVideoOutput::WasmVideoMode m_currentVideoMode = QWasmVideoOutput::VideoOutput; + QMediaPlayer::MediaStatus m_currentMediaStatus; + qreal m_currentBufferedValue; + + QScopedPointer<qstdweb::EventCallback> m_timeUpdateEvent; + QScopedPointer<qstdweb::EventCallback> m_playEvent; + QScopedPointer<qstdweb::EventCallback> m_endedEvent; + QScopedPointer<qstdweb::EventCallback> m_durationChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_loadedDataEvent; + QScopedPointer<qstdweb::EventCallback> m_errorChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_resizeChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_loadedMetadataChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_loadStartChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_canPlayChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_canPlayThroughChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_seekingChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_seekedChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_emptiedChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_stalledChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_waitingChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_playingChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_progressChangeEvent; + QScopedPointer<qstdweb::EventCallback> m_pauseChangeEvent; +}; + +QT_END_NAMESPACE +#endif // QWASMVIDEOOUTPUT_H diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmcamera.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera.cpp new file mode 100644 index 000000000..fbc5cf262 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera.cpp @@ -0,0 +1,478 @@ +// Copyright (C) 2022 The Qt Company Ltd and/or its subsidiary(-ies). +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmcamera_p.h" +#include "qmediadevices.h" +#include <qcameradevice.h> +#include "private/qplatformvideosink_p.h" +#include <private/qmemoryvideobuffer_p.h> +#include <private/qvideotexturehelper_p.h> +#include <private/qwasmmediadevices_p.h> + +#include "qwasmmediacapturesession_p.h" +#include <common/qwasmvideooutput_p.h> + +#include <emscripten/val.h> +#include <emscripten/bind.h> +#include <emscripten/html5.h> +#include <QUuid> +#include <QTimer> + +#include <private/qstdweb_p.h> + +Q_LOGGING_CATEGORY(qWasmCamera, "qt.multimedia.wasm.camera") + +QWasmCamera::QWasmCamera(QCamera *camera) + : QPlatformCamera(camera), + m_cameraOutput(new QWasmVideoOutput), + m_cameraIsReady(false) +{ + QWasmMediaDevices *wasmMediaDevices = + static_cast<QWasmMediaDevices *>(QPlatformMediaIntegration::instance()->mediaDevices()); + + connect(wasmMediaDevices, &QWasmMediaDevices::videoInputsChanged,this, [this]() { + const QList<QCameraDevice> cameras = QMediaDevices::videoInputs(); + + if (!cameras.isEmpty()) { + if (m_cameraDev.id().isEmpty()) + setCamera(cameras.at(0)); // default camera + else + setCamera(m_cameraDev); + return; + } + }); + + connect(this, &QWasmCamera::cameraIsReady, this, [this]() { + m_cameraIsReady = true; + if (m_cameraShouldStartActive) { + QTimer::singleShot(50, this, [this]() { + setActive(true); + }); + } + }); +} + +QWasmCamera::~QWasmCamera() = default; + +bool QWasmCamera::isActive() const +{ + return m_cameraActive; +} + +void QWasmCamera::setActive(bool active) +{ + + if (!m_CaptureSession) { + updateError(QCamera::CameraError, QStringLiteral("video surface error")); + m_shouldBeActive = true; + return; + } + + if (!m_cameraIsReady) { + m_cameraShouldStartActive = true; + return; + } + + QVideoSink *sink = m_CaptureSession->videoSink(); + if (!sink) { + qWarning() << Q_FUNC_INFO << "sink not ready"; + return; + } + + m_cameraOutput->setSurface(m_CaptureSession->videoSink()); + m_cameraActive = active; + m_shouldBeActive = false; + + if (m_cameraActive) + m_cameraOutput->start(); + else + m_cameraOutput->pause(); + + updateCameraFeatures(); + emit activeChanged(active); +} + +void QWasmCamera::setCamera(const QCameraDevice &camera) +{ + if (!m_cameraDev.id().isEmpty()) + return; + + m_cameraOutput->setVideoMode(QWasmVideoOutput::Camera); + + constexpr QSize initialSize(0, 0); + constexpr QRect initialRect(QPoint(0, 0), initialSize); + m_cameraOutput->createVideoElement(camera.id().toStdString()); // videoElementId + m_cameraOutput->createOffscreenElement(initialSize); + m_cameraOutput->updateVideoElementGeometry(initialRect); + + const auto cameras = QMediaDevices::videoInputs(); + + if (std::find(cameras.begin(), cameras.end(), camera) != cameras.end()) { + m_cameraDev = camera; + createCamera(m_cameraDev); + emit cameraIsReady(); + return; + } + + if (cameras.count() > 0) { + m_cameraDev = camera; + createCamera(m_cameraDev); + emit cameraIsReady(); + } else { + updateError(QCamera::CameraError, QStringLiteral("Failed to find a camera")); + } +} + +bool QWasmCamera::setCameraFormat(const QCameraFormat &format) +{ + m_cameraFormat = format; + + return true; +} + +void QWasmCamera::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + QWasmMediaCaptureSession *captureSession = static_cast<QWasmMediaCaptureSession *>(session); + if (m_CaptureSession == captureSession) + return; + + m_CaptureSession = captureSession; + + if (m_shouldBeActive) + setActive(true); +} + +void QWasmCamera::setFocusMode(QCamera::FocusMode mode) +{ + if (!isFocusModeSupported(mode)) + return; + + static constexpr std::string_view focusModeString = "focusMode"; + if (mode == QCamera::FocusModeManual) + m_cameraOutput->setDeviceSetting(focusModeString.data(), emscripten::val("manual")); + if (mode == QCamera::FocusModeAuto) + m_cameraOutput->setDeviceSetting(focusModeString.data(), emscripten::val("continuous")); + focusModeChanged(mode); +} + +bool QWasmCamera::isFocusModeSupported(QCamera::FocusMode mode) const +{ + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val focusMode = caps["focusMode"]; + if (focusMode.isUndefined()) + return false; + + std::vector<std::string> focalModes; + + for (int i = 0; i < focusMode["length"].as<int>(); i++) + focalModes.push_back(focusMode[i].as<std::string>()); + + // Do we need to take into account focusDistance + // it is not always available, and what distance + // would be far/near + + bool found = false; + switch (mode) { + case QCamera::FocusModeAuto: + return std::find(focalModes.begin(), focalModes.end(), "continuous") != focalModes.end() + || std::find(focalModes.begin(), focalModes.end(), "single-shot") + != focalModes.end(); + case QCamera::FocusModeAutoNear: + case QCamera::FocusModeAutoFar: + case QCamera::FocusModeHyperfocal: + case QCamera::FocusModeInfinity: + break; + case QCamera::FocusModeManual: + found = std::find(focalModes.begin(), focalModes.end(), "manual") != focalModes.end(); + }; + return found; +} + +void QWasmCamera::setTorchMode(QCamera::TorchMode mode) +{ + if (!isTorchModeSupported(mode)) + return; + + if (m_wasmTorchMode == mode) + return; + + static constexpr std::string_view torchModeString = "torchMode"; + bool hasChanged = false; + switch (mode) { + case QCamera::TorchOff: + m_cameraOutput->setDeviceSetting(torchModeString.data(), emscripten::val(false)); + hasChanged = true; + break; + case QCamera::TorchOn: + m_cameraOutput->setDeviceSetting(torchModeString.data(), emscripten::val(true)); + hasChanged = true; + break; + case QCamera::TorchAuto: + break; + }; + m_wasmTorchMode = mode; + if (hasChanged) + torchModeChanged(m_wasmTorchMode); +} + +bool QWasmCamera::isTorchModeSupported(QCamera::TorchMode mode) const +{ + if (!m_cameraIsReady) + return false; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val exposureMode = caps["torch"]; + if (exposureMode.isUndefined()) + return false; + + return (mode != QCamera::TorchAuto); +} + +void QWasmCamera::setExposureMode(QCamera::ExposureMode mode) +{ + // TODO manually come up with exposureTime values ? + if (!isExposureModeSupported(mode)) + return; + + if (m_wasmExposureMode == mode) + return; + + bool hasChanged = false; + static constexpr std::string_view exposureModeString = "exposureMode"; + switch (mode) { + case QCamera::ExposureManual: + m_cameraOutput->setDeviceSetting(exposureModeString.data(), emscripten::val("manual")); + hasChanged = true; + break; + case QCamera::ExposureAuto: + m_cameraOutput->setDeviceSetting(exposureModeString.data(), emscripten::val("continuous")); + hasChanged = true; + break; + default: + break; + }; + + if (hasChanged) { + m_wasmExposureMode = mode; + exposureModeChanged(m_wasmExposureMode); + } +} + +bool QWasmCamera::isExposureModeSupported(QCamera::ExposureMode mode) const +{ + if (!m_cameraIsReady) + return false; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val exposureMode = caps["exposureMode"]; + if (exposureMode.isUndefined()) + return false; + + std::vector<std::string> exposureModes; + + for (int i = 0; i < exposureMode["length"].as<int>(); i++) + exposureModes.push_back(exposureMode[i].as<std::string>()); + + bool found = false; + switch (mode) { + case QCamera::ExposureAuto: + found = std::find(exposureModes.begin(), exposureModes.end(), "continuous") + != exposureModes.end(); + break; + case QCamera::ExposureManual: + found = std::find(exposureModes.begin(), exposureModes.end(), "manual") + != exposureModes.end(); + break; + default: + break; + }; + + return found; +} + +void QWasmCamera::setExposureCompensation(float bias) +{ + if (!m_cameraIsReady) + return; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + emscripten::val exposureComp = caps["exposureCompensation"]; + if (exposureComp.isUndefined()) + return; + if (m_wasmExposureCompensation == bias) + return; + + static constexpr std::string_view exposureCompensationModeString = "exposureCompensation"; + m_cameraOutput->setDeviceSetting(exposureCompensationModeString.data(), emscripten::val(bias)); + m_wasmExposureCompensation = bias; + emit exposureCompensationChanged(m_wasmExposureCompensation); +} + +void QWasmCamera::setManualExposureTime(float secs) +{ + if (m_wasmExposureTime == secs) + return; + + if (!m_cameraIsReady) + return; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + emscripten::val exposureTime = caps["exposureTime"]; + if (exposureTime.isUndefined()) + return; + static constexpr std::string_view exposureTimeString = "exposureTime"; + m_cameraOutput->setDeviceSetting(exposureTimeString.data(), emscripten::val(secs)); + m_wasmExposureTime = secs; + emit exposureTimeChanged(m_wasmExposureTime); +} + +int QWasmCamera::isoSensitivity() const +{ + if (!m_cameraIsReady) + return 0; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val isoSpeed = caps["iso"]; + if (isoSpeed.isUndefined()) + return 0; + + return isoSpeed.as<double>(); +} + +void QWasmCamera::setManualIsoSensitivity(int sens) +{ + if (!m_cameraIsReady) + return; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + emscripten::val isoSpeed = caps["iso"]; + if (isoSpeed.isUndefined()) + return; + if (m_wasmIsoSensitivity == sens) + return; + static constexpr std::string_view isoString = "iso"; + m_cameraOutput->setDeviceSetting(isoString.data(), emscripten::val(sens)); + m_wasmIsoSensitivity = sens; + emit isoSensitivityChanged(m_wasmIsoSensitivity); +} + +bool QWasmCamera::isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const +{ + if (!m_cameraIsReady) + return false; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return false; + + emscripten::val whiteBalanceMode = caps["whiteBalanceMode"]; + if (whiteBalanceMode.isUndefined()) + return false; + + if (mode == QCamera::WhiteBalanceAuto || mode == QCamera::WhiteBalanceManual) + return true; + + return false; +} + +void QWasmCamera::setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) +{ + if (!isWhiteBalanceModeSupported(mode)) + return; + + if (m_wasmWhiteBalanceMode == mode) + return; + + bool hasChanged = false; + static constexpr std::string_view whiteBalanceModeString = "whiteBalanceMode"; + switch (mode) { + case QCamera::WhiteBalanceAuto: + m_cameraOutput->setDeviceSetting(whiteBalanceModeString.data(), emscripten::val("auto")); + hasChanged = true; + break; + case QCamera::WhiteBalanceManual: + m_cameraOutput->setDeviceSetting(whiteBalanceModeString.data(), emscripten::val("manual")); + hasChanged = true; + break; + default: + break; + }; + + if (hasChanged) { + m_wasmWhiteBalanceMode = mode; + emit whiteBalanceModeChanged(m_wasmWhiteBalanceMode); + } +} + +void QWasmCamera::setColorTemperature(int temperature) +{ + if (!m_cameraIsReady) + return; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + emscripten::val whiteBalanceMode = caps["colorTemperature"]; + if (whiteBalanceMode.isUndefined()) + return; + if(m_wasmColorTemperature == temperature) + return; + + static constexpr std::string_view colorBalanceString = "colorTemperature"; + m_cameraOutput->setDeviceSetting(colorBalanceString.data(), emscripten::val(temperature)); + m_wasmColorTemperature = temperature; + colorTemperatureChanged(m_wasmColorTemperature); +} + +void QWasmCamera::createCamera(const QCameraDevice &camera) +{ + m_cameraOutput->addCameraSourceElement(camera.id().toStdString()); +} + +void QWasmCamera::updateCameraFeatures() +{ + if (!m_cameraIsReady) + return; + + emscripten::val caps = m_cameraOutput->getDeviceCapabilities(); + if (caps.isUndefined()) + return; + + QCamera::Features cameraFeatures; + + if (!caps["colorTemperature"].isUndefined()) + cameraFeatures |= QCamera::Feature::ColorTemperature; + + if (!caps["exposureCompensation"].isUndefined()) + cameraFeatures |= QCamera::Feature::ExposureCompensation; + + if (!caps["iso"].isUndefined()) + cameraFeatures |= QCamera::Feature::IsoSensitivity; + + if (!caps["exposureTime"].isUndefined()) + cameraFeatures |= QCamera::Feature::ManualExposureTime; + + if (!caps["focusDistance"].isUndefined()) + cameraFeatures |= QCamera::Feature::FocusDistance; + + supportedFeaturesChanged(cameraFeatures); +} diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmcamera_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera_p.h new file mode 100644 index 000000000..7bb6d02d7 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmcamera_p.h @@ -0,0 +1,99 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMCAMERA_H +#define QWASMCAMERA_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformcamera_p.h> +#include <private/qplatformvideodevices_p.h> +#include <common/qwasmvideooutput_p.h> + +#include <QCameraDevice> +#include <QtCore/qloggingcategory.h> + +#include <emscripten/val.h> +#include <emscripten/bind.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmCamera) + +class QWasmMediaCaptureSession; + +class QWasmCamera : public QPlatformCamera +{ + Q_OBJECT + +public: + explicit QWasmCamera(QCamera *camera); + ~QWasmCamera(); + + bool isActive() const override; + void setActive(bool active) override; + + void setCamera(const QCameraDevice &camera) override; + bool setCameraFormat(const QCameraFormat &format) override; + + void setCaptureSession(QPlatformMediaCaptureSession *session) override; + + void setFocusMode(QCamera::FocusMode mode) override; + bool isFocusModeSupported(QCamera::FocusMode mode) const override; + + void setTorchMode(QCamera::TorchMode mode) override; + bool isTorchModeSupported(QCamera::TorchMode mode) const override; + + void setExposureMode(QCamera::ExposureMode mode) override; + bool isExposureModeSupported(QCamera::ExposureMode mode) const override; + void setExposureCompensation(float bias) override; + + void setManualExposureTime(float) override; + int isoSensitivity() const override; + void setManualIsoSensitivity(int) override; + + bool isWhiteBalanceModeSupported(QCamera::WhiteBalanceMode mode) const override; + void setWhiteBalanceMode(QCamera::WhiteBalanceMode mode) override; + + void setColorTemperature(int temperature) override; + + QWasmVideoOutput *cameraOutput() { return m_cameraOutput.data(); } +Q_SIGNALS: + void cameraIsReady(); +private: + void createCamera(const QCameraDevice &camera); + void updateCameraFeatures(); + + QCameraDevice m_cameraDev; + QWasmMediaCaptureSession *m_CaptureSession = nullptr; + bool m_cameraActive = false; + QScopedPointer <QWasmVideoOutput> m_cameraOutput; + + emscripten::val supportedCapabilities = emscripten::val::object(); // browser + emscripten::val currentCapabilities = emscripten::val::object(); // camera track + emscripten::val currentSettings = emscripten::val::object(); // camera track + + QCamera::TorchMode m_wasmTorchMode; + QCamera::ExposureMode m_wasmExposureMode; + float m_wasmExposureTime; + float m_wasmExposureCompensation; + int m_wasmIsoSensitivity; + QCamera::WhiteBalanceMode m_wasmWhiteBalanceMode; + int m_wasmColorTemperature; + bool m_cameraIsReady = false; + bool m_cameraShouldStartActive = false; + bool m_shouldBeActive = false; +}; + +QT_END_NAMESPACE + +#endif // QWASMCAMERA_H diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture.cpp new file mode 100644 index 000000000..f62d6f1a6 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture.cpp @@ -0,0 +1,130 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmimagecapture_p.h" +#include <qimagewriter.h> +#include "qwasmmediacapturesession_p.h" +#include "qwasmcamera_p.h" +#include "qwasmvideosink_p.h" + +QT_BEGIN_NAMESPACE + +Q_LOGGING_CATEGORY(qWasmImageCapture, "qt.multimedia.wasm.imagecapture") +/* TODO +signals: +imageExposed +*/ +QWasmImageCapture::QWasmImageCapture(QImageCapture *parent) : QPlatformImageCapture(parent) { } + +QWasmImageCapture::~QWasmImageCapture() = default; + +int QWasmImageCapture::capture(const QString &fileName) +{ + if (!isReadyForCapture()) { + emit error(m_lastId, QImageCapture::NotReadyError, msgCameraNotReady()); + return -1; + } + + // TODO if fileName.isEmpty() we choose filename and location + + QImage image = takePicture(); + if (image.isNull()) + return -1; + + QImageWriter writer(fileName); + // TODO + // writer.setQuality(quality); + // writer.setFormat("png"); + + if (writer.write(image)) { + qCDebug(qWasmImageCapture) << Q_FUNC_INFO << "image saved"; + emit imageSaved(m_lastId, fileName); + } else { + QImageCapture::Error err = (writer.error() == QImageWriter::UnsupportedFormatError) + ? QImageCapture::FormatError + : QImageCapture::ResourceError; + + emit error(m_lastId, err, writer.errorString()); + } + + return m_lastId; +} + +int QWasmImageCapture::captureToBuffer() +{ + if (!isReadyForCapture()) { + emit error(m_lastId, QImageCapture::NotReadyError, msgCameraNotReady()); + return -1; + } + + QImage image = takePicture(); + if (image.isNull()) + return -1; + + emit imageCaptured(m_lastId, image); + return m_lastId; +} + +QImage QWasmImageCapture::takePicture() +{ + QVideoFrame thisFrame = m_captureSession->videoSink()->videoFrame(); + if (!thisFrame.isValid()) + return QImage(); + + m_lastId++; + emit imageAvailable(m_lastId, thisFrame); + + QImage image = thisFrame.toImage(); + if (image.isNull()) { + qCDebug(qWasmImageCapture) << Q_FUNC_INFO << "image is null"; + emit error(m_lastId, QImageCapture::ResourceError, QStringLiteral("Resource error")); + return QImage(); + } + + emit imageCaptured(m_lastId, image); + if (m_settings.resolution().isValid() && m_settings.resolution() != image.size()) + image = image.scaled(m_settings.resolution()); + + return image; +} + +bool QWasmImageCapture::isReadyForCapture() const +{ + return m_isReadyForCapture; +} + +QImageEncoderSettings QWasmImageCapture::imageSettings() const +{ + return m_settings; +} + +void QWasmImageCapture::setImageSettings(const QImageEncoderSettings &settings) +{ + m_settings = settings; +} + +void QWasmImageCapture::setReadyForCapture(bool isReady) +{ + if (m_isReadyForCapture != isReady) { + m_isReadyForCapture = isReady; + emit readyForCaptureChanged(m_isReadyForCapture); + } +} + +void QWasmImageCapture::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + QWasmMediaCaptureSession *captureSession = static_cast<QWasmMediaCaptureSession *>(session); + // nullptr clears + if (m_captureSession == captureSession) + return; + + m_isReadyForCapture = captureSession; + if (captureSession) { + m_lastId = 0; + m_captureSession = captureSession; + } + emit readyForCaptureChanged(m_isReadyForCapture); + m_captureSession = captureSession; +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture_p.h new file mode 100644 index 000000000..2e9e9b227 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmimagecapture_p.h @@ -0,0 +1,58 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMIMAGECAPTURE_H +#define QWASMIMAGECAPTURE_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QObject> +#include <private/qplatformimagecapture_p.h> +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmImageCapture) + +class QWasmMediaCaptureSession; + +class QWasmImageCapture : public QPlatformImageCapture +{ + Q_OBJECT +public: + explicit QWasmImageCapture(QImageCapture *parent = nullptr); + ~QWasmImageCapture(); + + bool isReadyForCapture() const override; + + int capture(const QString &fileName) override; + int captureToBuffer() override; + + QImageEncoderSettings imageSettings() const override; + void setImageSettings(const QImageEncoderSettings &settings) override; + + void setReadyForCapture(bool isReady); + + void setCaptureSession(QPlatformMediaCaptureSession *session); + +private: + QImage takePicture(); + + // weak + QWasmMediaCaptureSession *m_captureSession = nullptr; + QImageEncoderSettings m_settings; + bool m_isReadyForCapture = false; + int m_lastId = 0; +}; + +QT_END_NAMESPACE +#endif // QWASMIMAGECAPTURE_H diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession.cpp new file mode 100644 index 000000000..826650570 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession.cpp @@ -0,0 +1,111 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmmediacapturesession_p.h" +#include "mediacapture/qwasmimagecapture_p.h" + +#include "qwasmcamera_p.h" +#include <private/qplatformmediadevices_p.h> +#include <private/qplatformmediaintegration_p.h> +#include <private/qwasmmediadevices_p.h> + +Q_LOGGING_CATEGORY(qWasmMediaCaptureSession, "qt.multimedia.wasm.capturesession") + +QWasmMediaCaptureSession::QWasmMediaCaptureSession() +{ + QWasmMediaDevices *wasmMediaDevices = static_cast<QWasmMediaDevices *>(QPlatformMediaIntegration::instance()->mediaDevices()); + wasmMediaDevices->initDevices(); +} + +QWasmMediaCaptureSession::~QWasmMediaCaptureSession() = default; + +QPlatformCamera *QWasmMediaCaptureSession::camera() +{ + return m_camera.data(); +} + +void QWasmMediaCaptureSession::setCamera(QPlatformCamera *camera) +{ + if (!camera) { + if (m_camera == nullptr) + return; + m_camera.reset(nullptr); + } else { + QWasmCamera *wasmCamera = static_cast<QWasmCamera *>(camera); + if (m_camera.data() == wasmCamera) + return; + m_camera.reset(wasmCamera); + m_camera->setCaptureSession(this); + } + emit cameraChanged(); +} + +QPlatformImageCapture *QWasmMediaCaptureSession::imageCapture() +{ + return m_imageCapture; +} + +void QWasmMediaCaptureSession::setImageCapture(QPlatformImageCapture *imageCapture) +{ + if (m_imageCapture == imageCapture) + return; + + if (m_imageCapture) + m_imageCapture->setCaptureSession(nullptr); + + m_imageCapture = static_cast<QWasmImageCapture *>(imageCapture); + + if (m_imageCapture) { + m_imageCapture->setCaptureSession(this); + + m_imageCapture->setReadyForCapture(true); + emit imageCaptureChanged(); + } +} + +QPlatformMediaRecorder *QWasmMediaCaptureSession::mediaRecorder() +{ + return m_mediaRecorder; +} + +void QWasmMediaCaptureSession::setMediaRecorder(QPlatformMediaRecorder *mediaRecorder) +{ + if (m_mediaRecorder == mediaRecorder) + return; + + if (m_mediaRecorder) + m_mediaRecorder->setCaptureSession(nullptr); + + m_mediaRecorder = static_cast<QWasmMediaRecorder *>(mediaRecorder); + + if (m_mediaRecorder) + m_mediaRecorder->setCaptureSession(this); +} + +void QWasmMediaCaptureSession::setAudioInput(QPlatformAudioInput *input) +{ + if (m_audioInput == input) + return; + + m_needsAudio = (bool)input; + m_audioInput = input; +} + +bool QWasmMediaCaptureSession::hasAudio() +{ + return m_needsAudio; +} + +void QWasmMediaCaptureSession::setVideoPreview(QVideoSink *sink) +{ + if (m_wasmSink == sink) + return; + m_wasmSink = sink; +} + +void QWasmMediaCaptureSession::setAudioOutput(QPlatformAudioOutput *output) +{ + if (m_audioOutput == output) + return; + m_audioOutput = output; +} diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession_p.h new file mode 100644 index 000000000..817580c90 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediacapturesession_p.h @@ -0,0 +1,71 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMMEDIACAPTURESESSION_H +#define QWASMMEDIACAPTURESESSION_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include "qwasmimagecapture_p.h" + +#include <private/qplatformmediacapture_p.h> +#include <private/qplatformmediaintegration_p.h> +#include "qwasmmediarecorder_p.h" +#include <QScopedPointer> +#include <QtCore/qloggingcategory.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmMediaCaptureSession) + +class QAudioInput; +class QWasmCamera; + +class QWasmMediaCaptureSession : public QPlatformMediaCaptureSession +{ +public: + explicit QWasmMediaCaptureSession(); + ~QWasmMediaCaptureSession() override; + + QPlatformCamera *camera() override; + void setCamera(QPlatformCamera *camera) override; + + QPlatformImageCapture *imageCapture() override; + void setImageCapture(QPlatformImageCapture *imageCapture) override; + + QPlatformMediaRecorder *mediaRecorder() override; + void setMediaRecorder(QPlatformMediaRecorder *recorder) override; + + void setAudioInput(QPlatformAudioInput *input) override; + QPlatformAudioInput * audioInput() const { return m_audioInput; } + void setVideoPreview(QVideoSink *sink) override; + void setAudioOutput(QPlatformAudioOutput *output) override; + + bool hasAudio(); + QVideoSink *videoSink() { return m_wasmSink; } + +private: + QWasmMediaRecorder *m_mediaRecorder = nullptr; + + QScopedPointer <QWasmCamera> m_camera; + + QWasmImageCapture *m_imageCapture = nullptr; + + QPlatformAudioInput *m_audioInput = nullptr; + QPlatformAudioOutput *m_audioOutput = nullptr; + bool m_needsAudio = false; + QVideoSink *m_wasmSink = nullptr; +}; + +QT_END_NAMESPACE + +#endif // QWASMMEDIACAPTURESESSION_H diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp new file mode 100644 index 000000000..98f04616a --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder.cpp @@ -0,0 +1,520 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmmediarecorder_p.h" +#include "qwasmmediacapturesession_p.h" +#include <private/qplatformmediadevices_p.h> +#include <private/qplatformmediaintegration_p.h> +#include "qwasmcamera_p.h" +#include "qwasmaudioinput_p.h" + +#include <private/qstdweb_p.h> +#include <QtCore/QIODevice> +#include <QFile> +#include <QTimer> +#include <QDebug> + +QT_BEGIN_NAMESPACE + +Q_LOGGING_CATEGORY(qWasmMediaRecorder, "qt.multimedia.wasm.mediarecorder") + +QWasmMediaRecorder::QWasmMediaRecorder(QMediaRecorder *parent) + : QPlatformMediaRecorder(parent) +{ + m_durationTimer.reset(new QElapsedTimer()); + QPlatformMediaIntegration::instance()->mediaDevices(); // initialize getUserMedia +} + +QWasmMediaRecorder::~QWasmMediaRecorder() +{ + if (m_outputTarget->isOpen()) + m_outputTarget->close(); + + if (!m_mediaRecorder.isNull()) { + m_mediaStreamDataAvailable.reset(nullptr); + m_mediaStreamStopped.reset(nullptr); + m_mediaStreamError.reset(nullptr); + m_mediaStreamStart.reset(nullptr); + } +} + +bool QWasmMediaRecorder::isLocationWritable(const QUrl &location) const +{ + return location.isValid() && (location.isLocalFile() || location.isRelative()); +} + +QMediaRecorder::RecorderState QWasmMediaRecorder::state() const +{ + QMediaRecorder::RecorderState recordingState = QMediaRecorder::StoppedState; + + if (!m_mediaRecorder.isUndefined()) { + std::string state = m_mediaRecorder["state"].as<std::string>(); + if (state == "recording") + recordingState = QMediaRecorder::RecordingState; + else if (state == "paused") + recordingState = QMediaRecorder::PausedState; + } + return recordingState; +} + +qint64 QWasmMediaRecorder::duration() const +{ // milliseconds + return m_durationMs; +} + +void QWasmMediaRecorder::record(QMediaEncoderSettings &settings) +{ + if (!m_session) + return; + + m_mediaSettings = settings; + initUserMedia(); +} + +void QWasmMediaRecorder::pause() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + m_mediaRecorder.call<void>("pause"); + emit stateChanged(state()); +} + +void QWasmMediaRecorder::resume() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + + m_mediaRecorder.call<void>("resume"); + emit stateChanged(state()); +} + +void QWasmMediaRecorder::stop() +{ + if (!m_session || (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull())) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaRecorder"; + return; + } + if (m_mediaRecorder["state"].as<std::string>() == "recording") + m_mediaRecorder.call<void>("stop"); +} + +void QWasmMediaRecorder::setCaptureSession(QPlatformMediaCaptureSession *session) +{ + m_session = static_cast<QWasmMediaCaptureSession *>(session); +} + +bool QWasmMediaRecorder::hasCamera() const +{ + return m_session && m_session->camera(); +} + +void QWasmMediaRecorder::initUserMedia() +{ + setUpFileSink(); + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + if (mediaDevices.isNull() || mediaDevices.isUndefined()) { + qCDebug(qWasmMediaRecorder) << "MediaDevices are undefined or null"; + return; + } + + if (!m_session) + return; + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << m_session; + + emscripten::val stream = emscripten::val::undefined(); + if (hasCamera()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "has camera"; + QWasmCamera *wasmCamera = reinterpret_cast<QWasmCamera *>(m_session->camera()); + + if (wasmCamera) { + emscripten::val m_video = wasmCamera->cameraOutput()->surfaceElement(); + if (m_video.isNull() || m_video.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "video element not found"; + return; + } + + stream = m_video["srcObject"]; + if (stream.isNull() || stream.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "Video input stream not found"; + return; + } + } + } else { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "has audio"; + stream = static_cast<QWasmAudioInput *>(m_session->audioInput())->mediaStream(); + + if (stream.isNull() || stream.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "Audio input stream not found"; + return; + } + } + if (stream.isNull() || stream.isUndefined()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "No input stream found"; + return; + } + + setStream(stream); +} + +void QWasmMediaRecorder::startAudioRecording() +{ + startStream(); +} + +void QWasmMediaRecorder::setStream(emscripten::val stream) +{ + emscripten::val emMediaSettings = emscripten::val::object(); + QMediaFormat::VideoCodec videoCodec = m_mediaSettings.videoCodec(); + QMediaFormat::AudioCodec audioCodec = m_mediaSettings.audioCodec(); + QMediaFormat::FileFormat fileFormat = m_mediaSettings.fileFormat(); + + // mime and codecs + QString mimeCodec; + if (!m_mediaSettings.mimeType().name().isEmpty()) { + mimeCodec = m_mediaSettings.mimeType().name(); + + if (videoCodec != QMediaFormat::VideoCodec::Unspecified) + mimeCodec += QStringLiteral(": codecs="); + + if (audioCodec != QMediaFormat::AudioCodec::Unspecified) { + // TODO + } + + if (fileFormat != QMediaFormat::UnspecifiedFormat) + mimeCodec += QMediaFormat::fileFormatName(m_mediaSettings.fileFormat()); + + emMediaSettings.set("mimeType", mimeCodec.toStdString()); + } + + if (m_mediaSettings.audioBitRate() > 0) + emMediaSettings.set("audioBitsPerSecond", emscripten::val(m_mediaSettings.audioBitRate())); + + if (m_mediaSettings.videoBitRate() > 0) + emMediaSettings.set("videoBitsPerSecond", emscripten::val(m_mediaSettings.videoBitRate())); + + // create the MediaRecorder, and set up data callback + m_mediaRecorder = emscripten::val::global("MediaRecorder").new_(stream, emMediaSettings); + + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "m_mediaRecorder state:" + << QString::fromStdString(m_mediaRecorder["state"].as<std::string>()); + + if (m_mediaRecorder.isNull() || m_mediaRecorder.isUndefined()) { + qWarning() << "MediaRecorder could not be found"; + return; + } + m_mediaRecorder.set("data-mediarecordercontext", + emscripten::val(quintptr(reinterpret_cast<void *>(this)))); + + if (!m_mediaStreamDataAvailable.isNull()) { + m_mediaStreamDataAvailable.reset(); + m_mediaStreamStopped.reset(); + m_mediaStreamError.reset(); + m_mediaStreamStart.reset(); + m_mediaStreamPause.reset(); + m_mediaStreamResume.reset(); + } + + // dataavailable + auto callback = [](emscripten::val blob) { + if (blob.isUndefined() || blob.isNull()) { + qCDebug(qWasmMediaRecorder) << "blob is null"; + return; + } + if (blob["target"].isUndefined() || blob["target"].isNull()) + return; + if (blob["data"].isUndefined() || blob["data"].isNull()) + return; + if (blob["target"]["data-mediarecordercontext"].isUndefined() + || blob["target"]["data-mediarecordercontext"].isNull()) + return; + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + blob["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + const double timeCode = + blob.hasOwnProperty("timecode") ? blob["timecode"].as<double>() : 0; + recorder->audioDataAvailable(blob["data"], timeCode); + } + }; + + m_mediaStreamDataAvailable.reset( + new qstdweb::EventCallback(m_mediaRecorder, "dataavailable", callback)); + + // stopped + auto stoppedCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + qCDebug(qWasmMediaRecorder) + << "STOPPED: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = false; + recorder->m_durationTimer->invalidate(); + + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamStopped.reset( + new qstdweb::EventCallback(m_mediaRecorder, "stop", stoppedCallback)); + + // error + auto errorCallback = [](emscripten::val theError) { + if (theError.isUndefined() || theError.isNull()) { + qCDebug(qWasmMediaRecorder) << "error is null"; + return; + } + qCDebug(qWasmMediaRecorder) + << theError["code"].as<int>() + << QString::fromStdString(theError["message"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + theError["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->updateError(QMediaRecorder::ResourceError, + QString::fromStdString(theError["message"].as<std::string>())); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamError.reset(new qstdweb::EventCallback(m_mediaRecorder, "error", errorCallback)); + + // start + auto startCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + + qCDebug(qWasmMediaRecorder) + << "START: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = true; + recorder->m_durationTimer->start(); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamStart.reset(new qstdweb::EventCallback(m_mediaRecorder, "start", startCallback)); + + // pause + auto pauseCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + + qCDebug(qWasmMediaRecorder) + << "pause: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = true; + recorder->m_durationTimer->start(); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamPause.reset(new qstdweb::EventCallback(m_mediaRecorder, "pause", pauseCallback)); + + // resume + auto resumeCallback = [](emscripten::val event) { + if (event.isUndefined() || event.isNull()) { + qCDebug(qWasmMediaRecorder) << "event is null"; + return; + } + + qCDebug(qWasmMediaRecorder) + << "resume: state changed" + << QString::fromStdString(event["target"]["state"].as<std::string>()); + + QWasmMediaRecorder *recorder = reinterpret_cast<QWasmMediaRecorder *>( + event["target"]["data-mediarecordercontext"].as<quintptr>()); + + if (recorder) { + recorder->m_isRecording = true; + recorder->m_durationTimer->start(); + emit recorder->stateChanged(recorder->state()); + } + }; + + m_mediaStreamResume.reset( + new qstdweb::EventCallback(m_mediaRecorder, "resume", resumeCallback)); + + // set up what options we can + if (hasCamera()) + setTrackContraints(m_mediaSettings, stream); + else + startStream(); +} + +void QWasmMediaRecorder::audioDataAvailable(emscripten::val blob, double timeCodeDifference) +{ + Q_UNUSED(timeCodeDifference) + if (blob.isUndefined() || blob.isNull()) { + qCDebug(qWasmMediaRecorder) << "blob is null"; + return; + } + + auto fileReader = std::make_shared<qstdweb::FileReader>(); + + fileReader->onError([=](emscripten::val theError) { + updateError(QMediaRecorder::ResourceError, + QString::fromStdString(theError["message"].as<std::string>())); + }); + + fileReader->onAbort([=](emscripten::val) { + updateError(QMediaRecorder::ResourceError, QStringLiteral("File read aborted")); + }); + + fileReader->onLoad([=](emscripten::val) { + if (fileReader->val().isNull() || fileReader->val().isUndefined()) + return; + qstdweb::ArrayBuffer result = fileReader->result(); + if (result.val().isNull() || result.val().isUndefined()) + return; + QByteArray fileContent = qstdweb::Uint8Array(result).copyToQByteArray(); + + if (m_isRecording && !fileContent.isEmpty()) { + m_durationMs = m_durationTimer->elapsed(); + if (m_outputTarget->isOpen()) + m_outputTarget->write(fileContent, fileContent.length()); + // we've read everything + emit durationChanged(m_durationMs); + qCDebug(qWasmMediaRecorder) << "duration changed" << m_durationMs; + } + }); + + fileReader->readAsArrayBuffer(qstdweb::Blob(blob)); +} + +// constraints are suggestions, as not all hardware supports all settings +void QWasmMediaRecorder::setTrackContraints(QMediaEncoderSettings &settings, emscripten::val stream) +{ + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << settings.audioSampleRate(); + + if (stream.isUndefined() || stream.isNull()) { + qCDebug(qWasmMediaRecorder)<< Q_FUNC_INFO << "could not find MediaStream"; + return; + } + + emscripten::val navigator = emscripten::val::global("navigator"); + emscripten::val mediaDevices = navigator["mediaDevices"]; + + // check which ones are supported + // emscripten::val allConstraints = mediaDevices.call<emscripten::val>("getSupportedConstraints"); + // browsers only support some settings + + emscripten::val videoParams = emscripten::val::object(); + emscripten::val constraints = emscripten::val::object(); + + if (hasCamera()) { + if (settings.videoFrameRate() > 0) + videoParams.set("frameRate", emscripten::val(settings.videoFrameRate())); + if (settings.videoResolution().height() > 0) + videoParams.set("height", + emscripten::val(settings.videoResolution().height())); // viewportHeight? + if (settings.videoResolution().width() > 0) + videoParams.set("width", emscripten::val(settings.videoResolution().width())); + + constraints.set("video", videoParams); // only video here + } + + emscripten::val audioParams = emscripten::val::object(); + if (settings.audioSampleRate() > 0) + audioParams.set("sampleRate", emscripten::val(settings.audioSampleRate())); // may not work + if (settings.audioBitRate() > 0) + audioParams.set("sampleSize", emscripten::val(settings.audioBitRate())); // may not work + if (settings.audioChannelCount() > 0) + audioParams.set("channelCount", emscripten::val(settings.audioChannelCount())); + + constraints.set("audio", audioParams); // only audio here + + if (hasCamera() && stream["active"].as<bool>()) { + emscripten::val videoTracks = emscripten::val::undefined(); + videoTracks = stream.call<emscripten::val>("getVideoTracks"); + if (videoTracks.isNull() || videoTracks.isUndefined()) { + qCDebug(qWasmMediaRecorder) << "no video tracks"; + return; + } + if (videoTracks["length"].as<int>() > 0) { + // try to apply the video options + qstdweb::Promise::make(videoTracks[0], QStringLiteral("applyConstraints"), + { .thenFunc = + [this](emscripten::val result) { + Q_UNUSED(result) + startStream(); + }, + .catchFunc = + [this](emscripten::val theError) { + qWarning() << "setting video params failed error"; + qCDebug(qWasmMediaRecorder) + << theError["code"].as<int>() + << QString::fromStdString(theError["message"].as<std::string>()); + updateError(QMediaRecorder::ResourceError, + QString::fromStdString(theError["message"].as<std::string>())); + } }, + constraints); + } + } +} + +// this starts the recording stream +void QWasmMediaRecorder::startStream() +{ + if (m_mediaRecorder.isUndefined() || m_mediaRecorder.isNull()) { + qCDebug(qWasmMediaRecorder) << Q_FUNC_INFO << "could not find MediaStream"; + return; + } + qCDebug(qWasmMediaRecorder) << "m_mediaRecorder state:" << + QString::fromStdString(m_mediaRecorder["state"].as<std::string>()); + + constexpr int sliceSizeInMs = 250; // TODO find what size is best + m_mediaRecorder.call<void>("start", emscripten::val(sliceSizeInMs)); + + /* this method can optionally be passed a timeslice argument with a value in milliseconds. + * If this is specified, the media will be captured in separate chunks of that duration, + * rather than the default behavior of recording the media in a single large chunk.*/ + + emit stateChanged(state()); +} + +void QWasmMediaRecorder::setUpFileSink() +{ + QString m_targetFileName = outputLocation().toLocalFile(); + QString suffix = m_mediaSettings.mimeType().preferredSuffix(); + if (m_targetFileName.isEmpty()) { + m_targetFileName = "/home/web_user/tmp." + suffix; + QPlatformMediaRecorder::setOutputLocation(m_targetFileName); + } + + m_outputTarget = new QFile(m_targetFileName, this); + if (!m_outputTarget->open(QIODevice::WriteOnly)) { + qWarning() << "target file is not writable"; + return; + } +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder_p.h b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder_p.h new file mode 100644 index 000000000..c325e411b --- /dev/null +++ b/src/plugins/multimedia/wasm/mediacapture/qwasmmediarecorder_p.h @@ -0,0 +1,89 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMMEDIARECORDER_H +#define QWASMMEDIARECORDER_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformmediarecorder_p.h> +#include <private/qplatformmediacapture_p.h> +#include <QtCore/qglobal.h> +#include <QtCore/qloggingcategory.h> +#include <QElapsedTimer> + +#include <emscripten.h> +#include <emscripten/val.h> +#include <emscripten/bind.h> +#include <private/qstdweb_p.h> + +QT_BEGIN_NAMESPACE + +Q_DECLARE_LOGGING_CATEGORY(qWasmMediaRecorder) + +class QWasmMediaCaptureSession; +class QIODevice; + +class QWasmMediaRecorder final : public QObject, public QPlatformMediaRecorder +{ + Q_OBJECT +public: + explicit QWasmMediaRecorder(QMediaRecorder *parent); + ~QWasmMediaRecorder() final; + + bool isLocationWritable(const QUrl &location) const override; + QMediaRecorder::RecorderState state() const override; + qint64 duration() const override; + void record(QMediaEncoderSettings &settings) override; + void pause() override; + void resume() override; + void stop() override; + + void setCaptureSession(QPlatformMediaCaptureSession *session); + +private: + + bool hasCamera() const; + void startAudioRecording(); + void setStream(emscripten::val stream); + void streamCallback(emscripten::val event); + void exceptionCallback(emscripten::val event); + void dataAvailableCallback(emscripten::val dataEvent); + void startStream(); + void setTrackContraints(QMediaEncoderSettings &settings, emscripten::val stream); + void initUserMedia(); + void audioDataAvailable(emscripten::val Blob, double timeCodeDifference); + void setUpFileSink(); + + emscripten::val m_mediaRecorder = emscripten::val::undefined(); + emscripten::val m_mediaStream = emscripten::val::undefined(); + + QWasmMediaCaptureSession *m_session = nullptr; + QMediaEncoderSettings m_mediaSettings; + QIODevice *m_outputTarget; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamDataAvailable; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamStopped; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamError; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamStart; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamPause; + QScopedPointer<qstdweb::EventCallback> m_mediaStreamResume; + + qint64 m_durationMs = 0; + bool m_isRecording = false; + QScopedPointer <QElapsedTimer> m_durationTimer; + +private Q_SLOTS: +}; + +QT_END_NAMESPACE + +#endif // QWASMMEDIARECORDER_H diff --git a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp new file mode 100644 index 000000000..75886b7c2 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp @@ -0,0 +1,475 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmmediaplayer_p.h" +#include <common/qwasmvideooutput_p.h> +#include <common/qwasmaudiooutput_p.h> +#include "qaudiooutput.h" + +#include <QtCore/qloggingcategory.h> +#include <QUuid> +#include <QtGlobal> +#include <QMimeDatabase> +#include <QFileInfo> + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(lcMediaPlayer, "qt.multimedia.wasm.mediaplayer"); + +QWasmMediaPlayer::QWasmMediaPlayer(QMediaPlayer *parent) + : QPlatformMediaPlayer(parent), + m_videoOutput(new QWasmVideoOutput), + m_State(QWasmMediaPlayer::Idle) +{ + qCDebug(lcMediaPlayer) << Q_FUNC_INFO << this; + +} + +QWasmMediaPlayer::~QWasmMediaPlayer() +{ + delete m_videoOutput; +} + +void QWasmMediaPlayer::initVideo() +{ + m_videoOutput->setVideoMode(QWasmVideoOutput::VideoOutput); + QUuid videoElementId = QUuid::createUuid(); + qCDebug(lcMediaPlayer) << Q_FUNC_INFO << "videoElementId"<< videoElementId << this; + + m_videoOutput->createVideoElement(videoElementId.toString(QUuid::WithoutBraces).toStdString()); + m_videoOutput->doElementCallbacks(); + m_videoOutput->createOffscreenElement(QSize(1280, 720)); + m_videoOutput->updateVideoElementGeometry(QRect(0, 0, 1280, 720)); // initial size 720p standard + + connect(m_videoOutput, &QWasmVideoOutput::bufferingChanged, this, + &QWasmMediaPlayer::bufferingChanged); + connect(m_videoOutput, &QWasmVideoOutput::errorOccured, this, + &QWasmMediaPlayer::errorOccured); + connect(m_videoOutput, &QWasmVideoOutput::stateChanged, this, + &QWasmMediaPlayer::mediaStateChanged); + connect(m_videoOutput, &QWasmVideoOutput::progressChanged, this, + &QWasmMediaPlayer::setPositionChanged); + connect(m_videoOutput, &QWasmVideoOutput::durationChanged, this, + &QWasmMediaPlayer::setDurationChanged); + connect(m_videoOutput, &QWasmVideoOutput::sizeChange, this, + &QWasmMediaPlayer::videoSizeChanged); + connect(m_videoOutput, &QWasmVideoOutput::readyChanged, this, + &QWasmMediaPlayer::videoOutputReady); + connect(m_videoOutput, &QWasmVideoOutput::statusChanged, this, + &QWasmMediaPlayer::onMediaStatusChanged); + connect(m_videoOutput, &QWasmVideoOutput::metaDataLoaded, this, + &QWasmMediaPlayer::videoMetaDataChanged); + + setVideoAvailable(true); +} + +void QWasmMediaPlayer::initAudio() +{ + connect(m_audioOutput->q, &QAudioOutput::deviceChanged, + this, &QWasmMediaPlayer::updateAudioDevice); + connect(m_audioOutput->q, &QAudioOutput::volumeChanged, + this, &QWasmMediaPlayer::volumeChanged); + connect(m_audioOutput->q, &QAudioOutput::mutedChanged, + this, &QWasmMediaPlayer::mutedChanged); + + connect(m_audioOutput, &QWasmAudioOutput::bufferingChanged, this, + &QWasmMediaPlayer::bufferingChanged); + connect(m_audioOutput, &QWasmAudioOutput::errorOccured, this, + &QWasmMediaPlayer::errorOccured); + connect(m_audioOutput, &QWasmAudioOutput::progressChanged, this, + &QWasmMediaPlayer::setPositionChanged); + connect(m_audioOutput, &QWasmAudioOutput::durationChanged, this, + &QWasmMediaPlayer::setDurationChanged); + connect(m_audioOutput, &QWasmAudioOutput::statusChanged, this, + &QWasmMediaPlayer::onMediaStatusChanged); + connect(m_audioOutput, &QWasmAudioOutput::stateChanged, this, + &QWasmMediaPlayer::mediaStateChanged); + setAudioAvailable(true); +} + +qint64 QWasmMediaPlayer::duration() const +{ + return m_videoOutput->getDuration(); +} + +qint64 QWasmMediaPlayer::position() const +{ + if (mediaStatus() == QMediaPlayer::EndOfMedia) + return duration(); + + if (m_videoAvailable) + return m_videoOutput->getCurrentPosition(); + + return 0; +} + +void QWasmMediaPlayer::setPosition(qint64 position) +{ + if (!isSeekable()) + return; + + const int seekPosition = (position > INT_MAX) ? INT_MAX : position; + + if (seekPosition == this->position()) + return; + + if (mediaStatus() == QMediaPlayer::EndOfMedia) + setMediaStatus(QMediaPlayer::LoadedMedia); + + if (m_videoAvailable) + return m_videoOutput->seekTo(position); + + emit positionChanged(seekPosition); +} + +void QWasmMediaPlayer::volumeChanged(float gain) +{ + if (m_State != QWasmMediaPlayer::Started) + return; + + if (m_videoAvailable) + m_videoOutput->setVolume(gain); +} + +void QWasmMediaPlayer::mutedChanged(bool muted) +{ + if (m_State != QWasmMediaPlayer::Started) + return; + + if (m_videoAvailable) + m_videoOutput->setMuted(muted); +} + +float QWasmMediaPlayer::bufferProgress() const +{ + return qBound(0.0, (m_bufferPercent * .01), 1.0); +} + +bool QWasmMediaPlayer::isAudioAvailable() const +{ + return m_audioAvailable; +} + +bool QWasmMediaPlayer::isVideoAvailable() const +{ + return m_videoAvailable; +} + +QMediaTimeRange QWasmMediaPlayer::availablePlaybackRanges() const +{ + return m_availablePlaybackRange; +} + +void QWasmMediaPlayer::updateAvailablePlaybackRanges() +{ + if (m_buffering) { + const qint64 pos = position(); + const qint64 end = (duration() / 100) * m_bufferPercent; + m_availablePlaybackRange.addInterval(pos, end); + } else if (isSeekable()) { + m_availablePlaybackRange = QMediaTimeRange(0, duration()); + } else { + m_availablePlaybackRange = QMediaTimeRange(); + } +} + +qreal QWasmMediaPlayer::playbackRate() const +{ + if (m_State != QWasmMediaPlayer::Started) + return 0; + + if (isVideoAvailable()) + return m_videoOutput->playbackRate(); + return 0; +} + +void QWasmMediaPlayer::setPlaybackRate(qreal rate) +{ + if (m_State != QWasmMediaPlayer::Started || !isVideoAvailable()) + return; + + m_videoOutput->setPlaybackRate(rate); + emit playbackRateChanged(rate); +} + +QUrl QWasmMediaPlayer::media() const +{ + return m_mediaContent; +} + +const QIODevice *QWasmMediaPlayer::mediaStream() const +{ + return m_mediaStream; +} + +void QWasmMediaPlayer::setMedia(const QUrl &mediaContent, QIODevice *stream) +{ + qCDebug(lcMediaPlayer) << Q_FUNC_INFO << mediaContent << stream; + QMimeDatabase db; + + if (mediaContent.isEmpty()) { + if (stream) { + m_mediaStream = stream; + qCDebug(lcMediaPlayer) << db.mimeTypeForData(stream).name(); + if (db.mimeTypeForData(stream).name().contains("audio")) { + setAudioAvailable(true); + m_audioOutput->setSource(m_mediaStream); + } else { // treat octet-stream as video + setVideoAvailable(true); + m_videoOutput->setSource(m_mediaStream); + } + } else { + + setMediaStatus(QMediaPlayer::NoMedia); + } + } else { + QString sourceFile = mediaContent.toLocalFile(); + qCDebug(lcMediaPlayer) << db.mimeTypeForFile(QFileInfo(sourceFile)).name(); + if (db.mimeTypeForFile(QFileInfo(sourceFile)).name().contains("audio")) { + setAudioAvailable(true); + m_audioOutput->setSource(mediaContent); + } else { // treat octet-stream as video + setVideoAvailable(true); + m_videoOutput->setSource(mediaContent); + } + } + + resetBufferingProgress(); +} + +void QWasmMediaPlayer::setVideoSink(QVideoSink *sink) +{ + if (m_videoSink == sink) + return; + + m_videoSink = sink; + + if (!m_videoSink) + return; + + initVideo(); + m_videoOutput->setSurface(sink); + setVideoAvailable(true); + if (isAudioAvailable() && m_audioOutput) + m_audioOutput->setVideoElement(m_videoOutput->currentVideoElement()); +} + +void QWasmMediaPlayer::setAudioOutput(QPlatformAudioOutput *output) +{ + if (m_audioOutput == output) + return; + + if (m_audioOutput) + m_audioOutput->q->disconnect(this); + m_audioOutput = static_cast<QWasmAudioOutput *>(output); + setAudioAvailable(true); +} + +void QWasmMediaPlayer::updateAudioDevice() +{ + if (m_audioOutput) { + m_audioOutput->setAudioDevice(m_audioOutput->q->device()); + } +} + +void QWasmMediaPlayer::play() +{ + resetCurrentLoop(); + + if (isVideoAvailable()) { + m_videoOutput->start(); + m_playWhenReady = true; + } else { + initAudio(); + if (isAudioAvailable()) { + m_audioOutput->start(); + } + } + +#ifdef DEBUG_AUDIOENGINE + QAudioEnginePrivate::checkNoError("play"); +#endif +} + +void QWasmMediaPlayer::pause() +{ + if ((m_State + & (QWasmMediaPlayer::Started | QWasmMediaPlayer::Paused + | QWasmMediaPlayer::PlaybackCompleted)) == 0) { + return; + } + if (isVideoAvailable()) { + m_videoOutput->pause(); + } else { + m_audioOutput->pause(); + stateChanged(QMediaPlayer::PausedState); + } +} + +void QWasmMediaPlayer::stop() +{ + m_playWhenReady = false; + + if (m_State == QWasmMediaPlayer::Idle || m_State == QWasmMediaPlayer::PlaybackCompleted + || m_State == QWasmMediaPlayer::Stopped) { + qWarning() << Q_FUNC_INFO << __LINE__; + return; + } + + if (isVideoAvailable()) { + m_videoOutput->stop(); + } else { + m_audioOutput->stop(); + } + +} + +bool QWasmMediaPlayer::isSeekable() const +{ + return isVideoAvailable() && m_videoOutput->isVideoSeekable(); +} + +void QWasmMediaPlayer::errorOccured(qint32 code, const QString &message) +{ + QString errorString; + QMediaPlayer::Error error = QMediaPlayer::ResourceError; + + switch (code) { + case QWasmMediaNetworkState::NetworkEmpty: // no data + break; + case QWasmMediaNetworkState::NetworkIdle: + break; + case QWasmMediaNetworkState::NetworkLoading: + break; + case QWasmMediaNetworkState::NetworkNoSource: // no source + error = QMediaPlayer::ResourceError; + errorString = message; + break; + }; + + emit QPlatformMediaPlayer::error(error, errorString); +} + +void QWasmMediaPlayer::bufferingChanged(qint32 percent) +{ + m_buffering = percent != 100; + m_bufferPercent = percent; + + updateAvailablePlaybackRanges(); + emit bufferProgressChanged(bufferProgress()); +} + +void QWasmMediaPlayer::videoSizeChanged(qint32 width, qint32 height) +{ + QSize newSize(width, height); + + if (width == 0 || height == 0 || newSize == m_videoSize) + return; + + m_videoSize = newSize; +} + +void QWasmMediaPlayer::mediaStateChanged(QWasmMediaPlayer::QWasmMediaPlayerState state) +{ + m_State = state; + QMediaPlayer::PlaybackState m_mediaPlayerState; + switch (m_State) { + case QWasmMediaPlayer::Started: + m_mediaPlayerState = QMediaPlayer::PlayingState; + break; + case QWasmMediaPlayer::Paused: + m_mediaPlayerState = QMediaPlayer::PausedState; + break; + case QWasmMediaPlayer::Stopped: + m_mediaPlayerState = QMediaPlayer::StoppedState; + break; + default: + m_mediaPlayerState = QMediaPlayer::StoppedState; + break; + }; + + QPlatformMediaPlayer::stateChanged(m_mediaPlayerState); +} + +int QWasmMediaPlayer::trackCount(TrackType trackType) +{ + Q_UNUSED(trackType) + // TODO QTBUG-108517 + return 0; // tracks.count(); +} + +void QWasmMediaPlayer::setPositionChanged(qint64 position) +{ + QPlatformMediaPlayer::positionChanged(position); +} + +void QWasmMediaPlayer::setDurationChanged(qint64 duration) +{ + QPlatformMediaPlayer::durationChanged(duration); +} + +void QWasmMediaPlayer::videoOutputReady(bool ready) +{ + setVideoAvailable(ready); + + if (m_playWhenReady && m_videoOutput->isReady()) + play(); +} + +void QWasmMediaPlayer::setMediaStatus(QMediaPlayer::MediaStatus status) +{ + mediaStatusChanged(status); + + switch (status) { + case QMediaPlayer::NoMedia: + case QMediaPlayer::InvalidMedia: + emit durationChanged(0); + break; + case QMediaPlayer::EndOfMedia: + setPositionChanged(position()); + default: + break; + }; +} + +void QWasmMediaPlayer::setAudioAvailable(bool available) +{ + if (m_audioAvailable == available) + return; + + m_audioAvailable = available; + emit audioAvailableChanged(m_audioAvailable); +} + +void QWasmMediaPlayer::setVideoAvailable(bool available) +{ + if (m_videoAvailable == available) + return; + + if (!available) + m_videoSize = QSize(); + + m_videoAvailable = available; + emit videoAvailableChanged(m_videoAvailable); +} + +void QWasmMediaPlayer::resetBufferingProgress() +{ + m_buffering = false; + m_bufferPercent = 0; + m_availablePlaybackRange = QMediaTimeRange(); +} + +void QWasmMediaPlayer::onMediaStatusChanged(QMediaPlayer::MediaStatus status) +{ + setMediaStatus(status); +} + +void QWasmMediaPlayer::videoMetaDataChanged() +{ + metaDataChanged(); +} + +QT_END_NAMESPACE + +#include "moc_qwasmmediaplayer_p.cpp" diff --git a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer_p.h b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer_p.h new file mode 100644 index 000000000..9269ecdb6 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer_p.h @@ -0,0 +1,124 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMMEDIAPLAYER_H +#define QWASMMEDIAPLAYER_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qglobal.h> +#include <private/qplatformmediaplayer_p.h> +#include <qsize.h> +#include <qurl.h> +#include <QtCore/qpointer.h> + +QT_BEGIN_NAMESPACE + +class QWasmAudioOutput; +class QWasmVideoOutput; + +class QWasmMediaPlayer : public QObject, public QPlatformMediaPlayer +{ + Q_OBJECT + +public: + explicit QWasmMediaPlayer(QMediaPlayer *parent = 0); + ~QWasmMediaPlayer() override; + + enum QWasmMediaPlayerState { + Error, + Idle, + Uninitialized, + Preparing, + Prepared, + Started, + Paused, + Stopped, + PlaybackCompleted + }; + Q_ENUM(QWasmMediaPlayerState) + + enum QWasmMediaNetworkState { NetworkEmpty = 0, NetworkIdle, NetworkLoading, NetworkNoSource }; + Q_ENUM(QWasmMediaNetworkState) + + qint64 duration() const override; + qint64 position() const override; + float bufferProgress() const override; + bool isAudioAvailable() const override; + bool isVideoAvailable() const override; + QMediaTimeRange availablePlaybackRanges() const override; + qreal playbackRate() const override; + void setPlaybackRate(qreal rate) override; + QUrl media() const override; + const QIODevice *mediaStream() const override; + void setMedia(const QUrl &mediaContent, QIODevice *stream) override; + void setVideoSink(QVideoSink *surface) override; + void setAudioOutput(QPlatformAudioOutput *output) override; + void setPosition(qint64 position) override; + void play() override; + void pause() override; + void stop() override; + bool isSeekable() const override; + int trackCount(TrackType trackType) override; + + void updateAudioDevice(); + +private Q_SLOTS: + void volumeChanged(float volume); + void mutedChanged(bool muted); + void videoOutputReady(bool ready); + void errorOccured(qint32 code, const QString &message); + void bufferingChanged(qint32 percent); + void videoSizeChanged(qint32 width, qint32 height); + void mediaStateChanged(QWasmMediaPlayer::QWasmMediaPlayerState state); + void setPositionChanged(qint64 position); + void setDurationChanged(qint64 duration); + void videoMetaDataChanged(); + + void onMediaStatusChanged(QMediaPlayer::MediaStatus status); + +private: + void setMediaStatus(QMediaPlayer::MediaStatus status); + void setAudioAvailable(bool available); + void setVideoAvailable(bool available); + void updateAvailablePlaybackRanges(); + void resetBufferingProgress(); + + void setSubtitle(QString subtitle); + void disableTrack(TrackType trackType); + void initVideo(); + void initAudio(); + + friend class StateChangeNotifier; + + QPointer<QWasmVideoOutput> m_videoOutput; + QWasmAudioOutput *m_audioOutput = nullptr; + + QUrl m_mediaContent; + QIODevice *m_mediaStream = nullptr; + + QVideoSink *m_videoSink = nullptr; + int m_bufferPercent = -1; + bool m_audioAvailable = false; + bool m_videoAvailable = false; + QSize m_videoSize; + bool m_buffering = false; + QMediaTimeRange m_availablePlaybackRange; + int m_State; + + bool m_playWhenReady = false; + +}; + +QT_END_NAMESPACE + +#endif // QWASMMEDIAPLAYER_H diff --git a/src/plugins/multimedia/wasm/mediaplayer/qwasmvideosink.cpp b/src/plugins/multimedia/wasm/mediaplayer/qwasmvideosink.cpp new file mode 100644 index 000000000..b6fe0e8e0 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediaplayer/qwasmvideosink.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmvideosink_p.h" + +#include <QtGui/rhi/qrhi.h> + +QT_BEGIN_NAMESPACE + +QWasmVideoSink::QWasmVideoSink(QVideoSink *parent) + : QPlatformVideoSink(parent) +{ +} + +void QWasmVideoSink::setRhi(QRhi *rhi) +{ + if (rhi && rhi->backend() != QRhi::OpenGLES2) + rhi = nullptr; + if (m_rhi == rhi) + return; + m_rhi = rhi; +} + +QT_END_NAMESPACE + +#include "moc_qwasmvideosink_p.cpp" diff --git a/src/plugins/multimedia/wasm/mediaplayer/qwasmvideosink_p.h b/src/plugins/multimedia/wasm/mediaplayer/qwasmvideosink_p.h new file mode 100644 index 000000000..5f2885249 --- /dev/null +++ b/src/plugins/multimedia/wasm/mediaplayer/qwasmvideosink_p.h @@ -0,0 +1,40 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#ifndef QWASMVIDEOSINK_H +#define QWASMVIDEOSINK_H + +#include <private/qplatformvideosink_p.h> + +QT_BEGIN_NAMESPACE + +class QVideoSink; +class QRhi; + +class QWasmVideoSink : public QPlatformVideoSink +{ + Q_OBJECT + +public: + explicit QWasmVideoSink(QVideoSink *parent = 0); + + void setRhi(QRhi *) override; + +private: + QRhi *m_rhi = nullptr; +}; + +QT_END_NAMESPACE + +#endif // QWASMVIDEOSINK_H diff --git a/src/plugins/multimedia/wasm/qwasmmediaintegration.cpp b/src/plugins/multimedia/wasm/qwasmmediaintegration.cpp new file mode 100644 index 000000000..effc194a4 --- /dev/null +++ b/src/plugins/multimedia/wasm/qwasmmediaintegration.cpp @@ -0,0 +1,109 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwasmmediaintegration_p.h" +#include <QLoggingCategory> + +#include <QCamera> +#include <QCameraDevice> + +#include <private/qplatformmediaformatinfo_p.h> +#include <private/qplatformmediaplugin_p.h> +#include <private/qplatformmediadevices_p.h> +#include <private/qplatformvideodevices_p.h> + +#include "mediaplayer/qwasmmediaplayer_p.h" +#include "mediaplayer/qwasmvideosink_p.h" +#include "qwasmaudioinput_p.h" +#include "common/qwasmaudiooutput_p.h" + +#include "mediacapture/qwasmmediacapturesession_p.h" +#include "mediacapture/qwasmmediarecorder_p.h" +#include "mediacapture/qwasmcamera_p.h" +#include "mediacapture/qwasmmediacapturesession_p.h" +#include "mediacapture/qwasmimagecapture_p.h" + +QT_BEGIN_NAMESPACE + + +class QWasmMediaPlugin : public QPlatformMediaPlugin +{ + Q_OBJECT + Q_PLUGIN_METADATA(IID QPlatformMediaPlugin_iid FILE "wasm.json") + +public: + QWasmMediaPlugin() + : QPlatformMediaPlugin() + {} + + QPlatformMediaIntegration *create(const QString &name) override + { + if (name == u"wasm") + return new QWasmMediaIntegration; + return nullptr; + } +}; + +QWasmMediaIntegration::QWasmMediaIntegration() + : QPlatformMediaIntegration(QLatin1String("wasm")) { } + +QMaybe<QPlatformMediaPlayer *> QWasmMediaIntegration::createPlayer(QMediaPlayer *player) +{ + return new QWasmMediaPlayer(player); +} + +QMaybe<QPlatformVideoSink *> QWasmMediaIntegration::createVideoSink(QVideoSink *sink) +{ + return new QWasmVideoSink(sink); +} + +QMaybe<QPlatformAudioInput *> QWasmMediaIntegration::createAudioInput(QAudioInput *audioInput) +{ + return new QWasmAudioInput(audioInput); +} + +QMaybe<QPlatformAudioOutput *> QWasmMediaIntegration::createAudioOutput(QAudioOutput *q) +{ + return new QWasmAudioOutput(q); +} + +QPlatformMediaFormatInfo *QWasmMediaIntegration::createFormatInfo() +{ + // TODO: create custom implementation + return new QPlatformMediaFormatInfo; +} + +QPlatformVideoDevices *QWasmMediaIntegration::createVideoDevices() +{ + return new QWasmCameraDevices(this); +} + +QMaybe<QPlatformMediaCaptureSession *> QWasmMediaIntegration::createCaptureSession() +{ + return new QWasmMediaCaptureSession(); +} + +QMaybe<QPlatformMediaRecorder *> QWasmMediaIntegration::createRecorder(QMediaRecorder *recorder) +{ + return new QWasmMediaRecorder(recorder); +} + +QMaybe<QPlatformCamera *> QWasmMediaIntegration::createCamera(QCamera *camera) +{ + return new QWasmCamera(camera); +} + +QMaybe<QPlatformImageCapture *> +QWasmMediaIntegration::createImageCapture(QImageCapture *imageCapture) +{ + return new QWasmImageCapture(imageCapture); +} + +QList<QCameraDevice> QWasmMediaIntegration::videoInputs() +{ + return videoDevices()->videoDevices(); +} + +QT_END_NAMESPACE + +#include "qwasmmediaintegration.moc" diff --git a/src/plugins/multimedia/wasm/qwasmmediaintegration_p.h b/src/plugins/multimedia/wasm/qwasmmediaintegration_p.h new file mode 100644 index 000000000..d946c1854 --- /dev/null +++ b/src/plugins/multimedia/wasm/qwasmmediaintegration_p.h @@ -0,0 +1,50 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWASMMEDIAINTEGRATION_H +#define QWASMMEDIAINTEGRATION_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformmediaintegration_p.h> + +#include <private/qwasmmediadevices_p.h> + +QT_BEGIN_NAMESPACE + +class QWasmMediaDevices; + +class QWasmMediaIntegration : public QPlatformMediaIntegration +{ +public: + QWasmMediaIntegration(); + + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *player) override; + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *sink) override; + + QMaybe<QPlatformAudioInput *> createAudioInput(QAudioInput *audioInput) override; + QMaybe<QPlatformAudioOutput *> createAudioOutput(QAudioOutput *q) override; + + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; + QMaybe<QPlatformCamera *> createCamera(QCamera *camera) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *recorder) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *imageCapture) override; + QList<QCameraDevice> videoInputs() override; + +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; + QPlatformVideoDevices *createVideoDevices() override; +}; + +QT_END_NAMESPACE + +#endif // QWASMMEDIAINTEGRATION_H diff --git a/src/plugins/multimedia/wasm/wasm.json b/src/plugins/multimedia/wasm/wasm.json new file mode 100644 index 000000000..02335aebe --- /dev/null +++ b/src/plugins/multimedia/wasm/wasm.json @@ -0,0 +1,5 @@ +{ + "Keys": [ + "wasm" + ] +} diff --git a/src/plugins/multimedia/windows/CMakeLists.txt b/src/plugins/multimedia/windows/CMakeLists.txt index 4443e69e4..963081e0a 100644 --- a/src/plugins/multimedia/windows/CMakeLists.txt +++ b/src/plugins/multimedia/windows/CMakeLists.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + qt_internal_add_plugin(QWindowsMediaPlugin OUTPUT_NAME windowsmediaplugin PLUGIN_TYPE multimedia @@ -14,9 +17,7 @@ qt_internal_add_plugin(QWindowsMediaPlugin player/mfevrvideowindowcontrol.cpp player/mfevrvideowindowcontrol_p.h player/mfplayercontrol.cpp player/mfplayercontrol_p.h player/mfplayersession.cpp player/mfplayersession_p.h - player/mftvideo.cpp player/mftvideo_p.h player/mfvideorenderercontrol.cpp player/mfvideorenderercontrol_p.h - player/samplegrabber.cpp player/samplegrabber_p.h mediacapture/qwindowscamera.cpp mediacapture/qwindowscamera_p.h mediacapture/qwindowsimagecapture.cpp diff --git a/src/plugins/multimedia/windows/common/mfmetadata.cpp b/src/plugins/multimedia/windows/common/mfmetadata.cpp index 9f66dc64c..cc8c425e3 100644 --- a/src/plugins/multimedia/windows/common/mfmetadata.cpp +++ b/src/plugins/multimedia/windows/common/mfmetadata.cpp @@ -1,44 +1,9 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <qmediametadata.h> #include <qdatetime.h> +#include <qtimezone.h> #include <qimage.h> #include <quuid.h> @@ -77,7 +42,7 @@ static QVariant convertValue(const PROPVARIANT& var) value = QDateTime(QDate(t.wYear, t.wMonth, t.wDay), QTime(t.wHour, t.wMinute, t.wSecond, t.wMilliseconds), - Qt::UTC); + QTimeZone(QTimeZone::UTC)); break; case VT_STREAM: { diff --git a/src/plugins/multimedia/windows/common/mfmetadata_p.h b/src/plugins/multimedia/windows/common/mfmetadata_p.h index d1846e9c5..9ff196240 100644 --- a/src/plugins/multimedia/windows/common/mfmetadata_p.h +++ b/src/plugins/multimedia/windows/common/mfmetadata_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFMETADATACONTROL_H #define MFMETADATACONTROL_H @@ -52,7 +16,7 @@ // #include <qmediametadata.h> -#include "Mfidl.h" +#include "mfidl.h" QT_USE_NAMESPACE diff --git a/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol.cpp b/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol.cpp index 7cc162204..912ab5e94 100644 --- a/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol.cpp +++ b/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol.cpp @@ -1,55 +1,21 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <system_error> #include <mferror.h> #include <qglobal.h> -#include "Wmcodecdsp.h" +#include "wmcodecdsp.h" #include "mfaudiodecodercontrol_p.h" #include <private/qwindowsaudioutils_p.h> +QT_BEGIN_NAMESPACE + MFAudioDecoderControl::MFAudioDecoderControl(QAudioDecoder *parent) : QPlatformAudioDecoder(parent) , m_sourceResolver(new SourceResolver) { - connect(m_sourceResolver, SIGNAL(mediaSourceReady()), this, SLOT(handleMediaSourceReady())); - connect(m_sourceResolver, SIGNAL(error(long)), this, SLOT(handleMediaSourceError(long))); + connect(m_sourceResolver, &SourceResolver::mediaSourceReady, this, &MFAudioDecoderControl::handleMediaSourceReady); + connect(m_sourceResolver, &SourceResolver::error, this, &MFAudioDecoderControl::handleMediaSourceError); } MFAudioDecoderControl::~MFAudioDecoderControl() @@ -121,22 +87,22 @@ void MFAudioDecoderControl::startReadingSource(IMFMediaSource *source) { Q_ASSERT(source); - m_decoderSourceReader.reset(new MFDecoderSourceReader()); + m_decoderSourceReader = makeComObject<MFDecoderSourceReader>(); if (!m_decoderSourceReader) { error(QAudioDecoder::ResourceError, tr("Could not instantiate MFDecoderSourceReader")); return; } auto mediaType = m_decoderSourceReader->setSource(source, m_outputFormat.sampleFormat()); - QAudioFormat mediaFormat = QWindowsAudioUtils::mediaTypeToFormat(mediaType.get()); + QAudioFormat mediaFormat = QWindowsAudioUtils::mediaTypeToFormat(mediaType.Get()); if (!mediaFormat.isValid()) { error(QAudioDecoder::FormatError, tr("Invalid media format")); - m_decoderSourceReader.reset(); + m_decoderSourceReader.Reset(); return; } - QWindowsIUPointer<IMFPresentationDescriptor> pd; - if (SUCCEEDED(source->CreatePresentationDescriptor(pd.address()))) { + ComPtr<IMFPresentationDescriptor> pd; + if (SUCCEEDED(source->CreatePresentationDescriptor(pd.GetAddressOf()))) { UINT64 duration = 0; pd->GetUINT64(MF_PD_DURATION, &duration); duration /= 10000; @@ -145,12 +111,12 @@ void MFAudioDecoderControl::startReadingSource(IMFMediaSource *source) } if (!m_resampler.setup(mediaFormat, m_outputFormat.isValid() ? m_outputFormat : mediaFormat)) { - qWarning() << "Failed to setup resampler"; + qWarning() << "Failed to set up resampler"; return; } - connect(m_decoderSourceReader.get(), SIGNAL(finished()), this, SLOT(handleSourceFinished())); - connect(m_decoderSourceReader.get(), SIGNAL(newSample(QWindowsIUPointer<IMFSample>)), this, SLOT(handleNewSample(QWindowsIUPointer<IMFSample>))); + connect(m_decoderSourceReader.Get(), &MFDecoderSourceReader::finished, this, &MFAudioDecoderControl::handleSourceFinished); + connect(m_decoderSourceReader.Get(), &MFDecoderSourceReader::newSample, this, &MFAudioDecoderControl::handleNewSample); setIsDecoding(true); @@ -186,14 +152,14 @@ void MFAudioDecoderControl::stop() if (!isDecoding()) return; - disconnect(m_decoderSourceReader.get()); + disconnect(m_decoderSourceReader.Get()); m_decoderSourceReader->clearSource(); - m_decoderSourceReader.reset(); + m_decoderSourceReader.Reset(); if (bufferAvailable()) { QAudioBuffer buffer; m_audioBuffer.swap(buffer); - emit bufferAvailableChanged(false); + bufferAvailableChanged(false); } setIsDecoding(false); @@ -207,12 +173,12 @@ void MFAudioDecoderControl::stop() } } -void MFAudioDecoderControl::handleNewSample(QWindowsIUPointer<IMFSample> sample) +void MFAudioDecoderControl::handleNewSample(ComPtr<IMFSample> sample) { Q_ASSERT(sample); qint64 sampleStartTimeUs = m_resampler.outputFormat().durationForBytes(m_resampler.totalOutputBytes()); - QByteArray out = m_resampler.resample(sample.get()); + QByteArray out = m_resampler.resample(sample.Get()); if (out.isEmpty()) { error(QAudioDecoder::Error::ResourceError, tr("Failed processing a sample")); @@ -220,15 +186,15 @@ void MFAudioDecoderControl::handleNewSample(QWindowsIUPointer<IMFSample> sample) } else { m_audioBuffer = QAudioBuffer(out, m_resampler.outputFormat(), sampleStartTimeUs); - emit bufferAvailableChanged(true); - emit bufferReady(); + bufferAvailableChanged(true); + bufferReady(); } } void MFAudioDecoderControl::handleSourceFinished() { stop(); - emit finished(); + finished(); } void MFAudioDecoderControl::setAudioFormat(const QAudioFormat &format) @@ -236,7 +202,7 @@ void MFAudioDecoderControl::setAudioFormat(const QAudioFormat &format) if (m_outputFormat == format) return; m_outputFormat = format; - emit formatChanged(m_outputFormat); + formatChanged(m_outputFormat); } QAudioBuffer MFAudioDecoderControl::read() @@ -246,10 +212,14 @@ QAudioBuffer MFAudioDecoderControl::read() if (bufferAvailable()) { buffer.swap(m_audioBuffer); m_position = buffer.startTime() / 1000; - emit positionChanged(m_position); - emit bufferAvailableChanged(false); + positionChanged(m_position); + bufferAvailableChanged(false); m_decoderSourceReader->readNextSample(); } return buffer; } + +QT_END_NAMESPACE + +#include "moc_mfaudiodecodercontrol_p.cpp" diff --git a/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol_p.h b/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol_p.h index 0b1b42198..9bb2371ec 100644 --- a/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol_p.h +++ b/src/plugins/multimedia/windows/decoder/mfaudiodecodercontrol_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFAUDIODECODERCONTROL_H #define MFAUDIODECODERCONTROL_H @@ -54,10 +18,10 @@ #include "mfdecodersourcereader_p.h" #include <private/qplatformaudiodecoder_p.h> #include <sourceresolver_p.h> -#include <private/qwindowsiupointer_p.h> +#include <private/qcomptr_p.h> #include <private/qwindowsresampler_p.h> -QT_USE_NAMESPACE +QT_BEGIN_NAMESPACE class MFAudioDecoderControl : public QPlatformAudioDecoder { @@ -87,13 +51,13 @@ public: private Q_SLOTS: void handleMediaSourceReady(); void handleMediaSourceError(long hr); - void handleNewSample(QWindowsIUPointer<IMFSample>); + void handleNewSample(ComPtr<IMFSample>); void handleSourceFinished(); private: void startReadingSource(IMFMediaSource *source); - QWindowsIUPointer<MFDecoderSourceReader> m_decoderSourceReader; + ComPtr<MFDecoderSourceReader> m_decoderSourceReader; SourceResolver *m_sourceResolver; QWindowsResampler m_resampler; QUrl m_source; @@ -106,4 +70,6 @@ private: bool m_deferredStart = false; }; +QT_END_NAMESPACE + #endif//MFAUDIODECODERCONTROL_H diff --git a/src/plugins/multimedia/windows/decoder/mfdecodersourcereader.cpp b/src/plugins/multimedia/windows/decoder/mfdecodersourcereader.cpp index e0b54dcde..097f83437 100644 --- a/src/plugins/multimedia/windows/decoder/mfdecodersourcereader.cpp +++ b/src/plugins/multimedia/windows/decoder/mfdecodersourcereader.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <system_error> #include <mferror.h> @@ -43,24 +7,26 @@ #include <qdebug.h> #include "mfdecodersourcereader_p.h" -QWindowsIUPointer<IMFMediaType> MFDecoderSourceReader::setSource(IMFMediaSource *source, QAudioFormat::SampleFormat sampleFormat) +QT_BEGIN_NAMESPACE + +ComPtr<IMFMediaType> MFDecoderSourceReader::setSource(IMFMediaSource *source, QAudioFormat::SampleFormat sampleFormat) { - QWindowsIUPointer<IMFMediaType> mediaType; - m_sourceReader.reset(); + ComPtr<IMFMediaType> mediaType; + m_sourceReader.Reset(); if (!source) return mediaType; - QWindowsIUPointer<IMFAttributes> attr; - MFCreateAttributes(attr.address(), 1); + ComPtr<IMFAttributes> attr; + MFCreateAttributes(attr.GetAddressOf(), 1); if (FAILED(attr->SetUnknown(MF_SOURCE_READER_ASYNC_CALLBACK, this))) return mediaType; if (FAILED(attr->SetUINT32(MF_SOURCE_READER_DISCONNECT_MEDIASOURCE_ON_SHUTDOWN, TRUE))) return mediaType; - HRESULT hr = MFCreateSourceReaderFromMediaSource(source, attr.get(), m_sourceReader.address()); + HRESULT hr = MFCreateSourceReaderFromMediaSource(source, attr.Get(), m_sourceReader.GetAddressOf()); if (FAILED(hr)) { - qWarning() << "MFDecoderSourceReader: failed to setup source reader: " + qWarning() << "MFDecoderSourceReader: failed to set up source reader: " << std::system_category().message(hr).c_str(); return mediaType; } @@ -68,12 +34,12 @@ QWindowsIUPointer<IMFMediaType> MFDecoderSourceReader::setSource(IMFMediaSource m_sourceReader->SetStreamSelection(DWORD(MF_SOURCE_READER_ALL_STREAMS), FALSE); m_sourceReader->SetStreamSelection(DWORD(MF_SOURCE_READER_FIRST_AUDIO_STREAM), TRUE); - QWindowsIUPointer<IMFMediaType> pPartialType; - MFCreateMediaType(pPartialType.address()); + ComPtr<IMFMediaType> pPartialType; + MFCreateMediaType(pPartialType.GetAddressOf()); pPartialType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); pPartialType->SetGUID(MF_MT_SUBTYPE, sampleFormat == QAudioFormat::Float ? MFAudioFormat_Float : MFAudioFormat_PCM); - m_sourceReader->SetCurrentMediaType(DWORD(MF_SOURCE_READER_FIRST_AUDIO_STREAM), nullptr, pPartialType.get()); - m_sourceReader->GetCurrentMediaType(DWORD(MF_SOURCE_READER_FIRST_AUDIO_STREAM), mediaType.address()); + m_sourceReader->SetCurrentMediaType(DWORD(MF_SOURCE_READER_FIRST_AUDIO_STREAM), nullptr, pPartialType.Get()); + m_sourceReader->GetCurrentMediaType(DWORD(MF_SOURCE_READER_FIRST_AUDIO_STREAM), mediaType.GetAddressOf()); // Ensure the stream is selected. m_sourceReader->SetStreamSelection(DWORD(MF_SOURCE_READER_FIRST_AUDIO_STREAM), TRUE); @@ -125,10 +91,13 @@ STDMETHODIMP MFDecoderSourceReader::OnReadSample(HRESULT hrStatus, DWORD dwStrea Q_UNUSED(dwStreamIndex); Q_UNUSED(llTimestamp); if (pSample) { - pSample->AddRef(); - emit newSample(QWindowsIUPointer{pSample}); + emit newSample(ComPtr<IMFSample>{pSample}); } else if ((dwStreamFlags & MF_SOURCE_READERF_ENDOFSTREAM) == MF_SOURCE_READERF_ENDOFSTREAM) { emit finished(); } return S_OK; } + +QT_END_NAMESPACE + +#include "moc_mfdecodersourcereader_p.cpp" diff --git a/src/plugins/multimedia/windows/decoder/mfdecodersourcereader_p.h b/src/plugins/multimedia/windows/decoder/mfdecodersourcereader_p.h index da35f466e..dee6f8bf5 100644 --- a/src/plugins/multimedia/windows/decoder/mfdecodersourcereader_p.h +++ b/src/plugins/multimedia/windows/decoder/mfdecodersourcereader_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFDECODERSOURCEREADER_H #define MFDECODERSOURCEREADER_H @@ -57,9 +21,9 @@ #include <QtCore/qobject.h> #include "qaudioformat.h" -#include <private/qwindowsiupointer_p.h> +#include <private/qcomptr_p.h> -QT_USE_NAMESPACE +QT_BEGIN_NAMESPACE class MFDecoderSourceReader : public QObject, public IMFSourceReaderCallback { @@ -68,8 +32,8 @@ public: MFDecoderSourceReader() {} ~MFDecoderSourceReader() override {} - void clearSource() { m_sourceReader.reset(); } - QWindowsIUPointer<IMFMediaType> setSource(IMFMediaSource *source, QAudioFormat::SampleFormat); + void clearSource() { m_sourceReader.Reset(); } + ComPtr<IMFMediaType> setSource(IMFMediaSource *source, QAudioFormat::SampleFormat); void readNextSample(); @@ -85,12 +49,15 @@ public: STDMETHODIMP OnEvent(DWORD, IMFMediaEvent *) override { return S_OK; } Q_SIGNALS: - void newSample(QWindowsIUPointer<IMFSample>); + void newSample(ComPtr<IMFSample>); void finished(); private: long m_cRef = 1; - QWindowsIUPointer<IMFSourceReader> m_sourceReader; + ComPtr<IMFSourceReader> m_sourceReader; }; + +QT_END_NAMESPACE + #endif//MFDECODERSOURCEREADER_H diff --git a/src/plugins/multimedia/windows/evr/evrcustompresenter.cpp b/src/plugins/multimedia/windows/evr/evrcustompresenter.cpp index e8b99a09f..2a3433f4d 100644 --- a/src/plugins/multimedia/windows/evr/evrcustompresenter.cpp +++ b/src/plugins/multimedia/windows/evr/evrcustompresenter.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "evrcustompresenter_p.h" @@ -45,7 +9,7 @@ #include <private/qplatformvideosink_p.h> #include <private/qwindowsmfdefs_p.h> -#include <QtGui/private/qrhi_p.h> +#include <rhi/qrhi.h> #include <QtCore/qmutex.h> #include <QtCore/qvarlengtharray.h> @@ -53,7 +17,7 @@ #include <qthread.h> #include <qcoreapplication.h> #include <qmath.h> -#include <QtCore/qdebug.h> +#include <qloggingcategory.h> #include <mutex> @@ -62,14 +26,14 @@ QT_BEGIN_NAMESPACE +static Q_LOGGING_CATEGORY(qLcEvrCustomPresenter, "qt.multimedia.evrcustompresenter") + const static MFRatio g_DefaultFrameRate = { 30, 1 }; static const DWORD SCHEDULER_TIMEOUT = 5000; static const MFTIME ONE_SECOND = 10000000; static const LONG ONE_MSEC = 1000; // Function declarations. -static HRESULT setDesiredSampleTime(IMFSample *sample, const LONGLONG& hnsSampleTime, const LONGLONG& hnsDuration); -static HRESULT clearDesiredSampleTime(IMFSample *sample); static HRESULT setMixerSourceRect(IMFTransform *mixer, const MFVideoNormalizedRect& nrcSource); static QVideoFrameFormat::PixelFormat pixelFormatFromMediaType(IMFMediaType *type); @@ -97,45 +61,27 @@ bool qt_evr_setCustomPresenter(IUnknown *evr, EVRCustomPresenter *presenter) class PresentSampleEvent : public QEvent { public: - PresentSampleEvent(IMFSample *sample) - : QEvent(QEvent::Type(EVRCustomPresenter::PresentSample)) - , m_sample(sample) - { - if (m_sample) - m_sample->AddRef(); - } - - ~PresentSampleEvent() override + explicit PresentSampleEvent(const ComPtr<IMFSample> &sample) + : QEvent(static_cast<Type>(EVRCustomPresenter::PresentSample)), m_sample(sample) { - if (m_sample) - m_sample->Release(); } - IMFSample *sample() const { return m_sample; } + ComPtr<IMFSample> sample() const { return m_sample; } private: - IMFSample *m_sample; + const ComPtr<IMFSample> m_sample; }; Scheduler::Scheduler(EVRCustomPresenter *presenter) : m_presenter(presenter) - , m_clock(NULL) , m_threadID(0) - , m_schedulerThread(0) - , m_threadReadyEvent(0) - , m_flushEvent(0) , m_playbackRate(1.0f) - , m_perFrameInterval(0) , m_perFrame_1_4th(0) - , m_lastSampleTime(0) { } Scheduler::~Scheduler() { - qt_evr_safe_release(&m_clock); - for (int i = 0; i < m_scheduledSamples.size(); ++i) - m_scheduledSamples[i]->Release(); m_scheduledSamples.clear(); } @@ -146,13 +92,11 @@ void Scheduler::setFrameRate(const MFRatio& fps) // Convert to a duration. MFFrameRateToAverageTimePerFrame(fps.Numerator, fps.Denominator, &AvgTimePerFrame); - m_perFrameInterval = (MFTIME)AvgTimePerFrame; - // Calculate 1/4th of this value, because we use it frequently. - m_perFrame_1_4th = m_perFrameInterval / 4; + m_perFrame_1_4th = AvgTimePerFrame / 4; } -HRESULT Scheduler::startScheduler(IMFClock *clock) +HRESULT Scheduler::startScheduler(ComPtr<IMFClock> clock) { if (m_schedulerThread) return E_UNEXPECTED; @@ -162,44 +106,39 @@ HRESULT Scheduler::startScheduler(IMFClock *clock) HANDLE hObjects[2]; DWORD dwWait = 0; - if (m_clock) - m_clock->Release(); m_clock = clock; - if (m_clock) - m_clock->AddRef(); // Set a high the timer resolution (ie, short timer period). timeBeginPeriod(1); // Create an event to wait for the thread to start. - m_threadReadyEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + m_threadReadyEvent = EventHandle{ CreateEvent(NULL, FALSE, FALSE, NULL) }; if (!m_threadReadyEvent) { hr = HRESULT_FROM_WIN32(GetLastError()); goto done; } // Create an event to wait for flush commands to complete. - m_flushEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + m_flushEvent = EventHandle{ CreateEvent(NULL, FALSE, FALSE, NULL) }; if (!m_flushEvent) { hr = HRESULT_FROM_WIN32(GetLastError()); goto done; } // Create the scheduler thread. - m_schedulerThread = CreateThread(NULL, 0, schedulerThreadProc, (LPVOID)this, 0, &dwID); + m_schedulerThread = ThreadHandle{ CreateThread(NULL, 0, schedulerThreadProc, (LPVOID)this, 0, &dwID) }; if (!m_schedulerThread) { hr = HRESULT_FROM_WIN32(GetLastError()); goto done; } // Wait for the thread to signal the "thread ready" event. - hObjects[0] = m_threadReadyEvent; - hObjects[1] = m_schedulerThread; + hObjects[0] = m_threadReadyEvent.get(); + hObjects[1] = m_schedulerThread.get(); dwWait = WaitForMultipleObjects(2, hObjects, FALSE, INFINITE); // Wait for EITHER of these handles. if (WAIT_OBJECT_0 != dwWait) { // The thread terminated early for some reason. This is an error condition. - CloseHandle(m_schedulerThread); - m_schedulerThread = NULL; + m_schedulerThread = {}; hr = E_UNEXPECTED; goto done; @@ -209,10 +148,8 @@ HRESULT Scheduler::startScheduler(IMFClock *clock) done: // Regardless success/failure, we are done using the "thread ready" event. - if (m_threadReadyEvent) { - CloseHandle(m_threadReadyEvent); - m_threadReadyEvent = NULL; - } + m_threadReadyEvent = {}; + return hr; } @@ -225,19 +162,14 @@ HRESULT Scheduler::stopScheduler() PostThreadMessage(m_threadID, Terminate, 0, 0); // Wait for the thread to exit. - WaitForSingleObject(m_schedulerThread, INFINITE); + WaitForSingleObject(m_schedulerThread.get(), INFINITE); // Close handles. - CloseHandle(m_schedulerThread); - m_schedulerThread = NULL; - - CloseHandle(m_flushEvent); - m_flushEvent = NULL; + m_schedulerThread = {}; + m_flushEvent = {}; // Discard samples. m_mutex.lock(); - for (int i = 0; i < m_scheduledSamples.size(); ++i) - m_scheduledSamples[i]->Release(); m_scheduledSamples.clear(); m_mutex.unlock(); @@ -255,7 +187,7 @@ HRESULT Scheduler::flush() // Wait for the scheduler thread to signal the flush event, // OR for the thread to terminate. - HANDLE objects[] = { m_flushEvent, m_schedulerThread }; + HANDLE objects[] = { m_flushEvent.get(), m_schedulerThread.get() }; WaitForMultipleObjects(ARRAYSIZE(objects), objects, FALSE, SCHEDULER_TIMEOUT); } @@ -269,7 +201,7 @@ bool Scheduler::areSamplesScheduled() return m_scheduledSamples.count() > 0; } -HRESULT Scheduler::scheduleSample(IMFSample *sample, bool presentNow) +HRESULT Scheduler::scheduleSample(const ComPtr<IMFSample> &sample, bool presentNow) { if (!m_schedulerThread) return MF_E_NOT_INITIALIZED; @@ -277,16 +209,20 @@ HRESULT Scheduler::scheduleSample(IMFSample *sample, bool presentNow) HRESULT hr = S_OK; DWORD dwExitCode = 0; - GetExitCodeThread(m_schedulerThread, &dwExitCode); + GetExitCodeThread(m_schedulerThread.get(), &dwExitCode); if (dwExitCode != STILL_ACTIVE) return E_FAIL; if (presentNow || !m_clock) { m_presenter->presentSample(sample); } else { + if (m_playbackRate > 0.0f && qt_evr_isSampleTimePassed(m_clock.Get(), sample.Get())) { + qCDebug(qLcEvrCustomPresenter) << "Discard the sample, it came too late"; + return hr; + } + // Queue the sample and ask the scheduler thread to wake up. m_mutex.lock(); - sample->AddRef(); m_scheduledSamples.enqueue(sample); m_mutex.unlock(); @@ -301,25 +237,37 @@ HRESULT Scheduler::processSamplesInQueue(LONG *nextSleep) { HRESULT hr = S_OK; LONG wait = 0; - IMFSample *sample = NULL; + + QQueue<ComPtr<IMFSample>> scheduledSamples; + + m_mutex.lock(); + m_scheduledSamples.swap(scheduledSamples); + m_mutex.unlock(); // Process samples until the queue is empty or until the wait time > 0. - while (!m_scheduledSamples.isEmpty()) { - m_mutex.lock(); - sample = m_scheduledSamples.dequeue(); - m_mutex.unlock(); + while (!scheduledSamples.isEmpty()) { + ComPtr<IMFSample> sample = scheduledSamples.dequeue(); // Process the next sample in the queue. If the sample is not ready // for presentation. the value returned in wait is > 0, which // means the scheduler should sleep for that amount of time. + if (isSampleReadyToPresent(sample.Get(), &wait)) { + m_presenter->presentSample(sample.Get()); + continue; + } - hr = processSample(sample, &wait); - qt_evr_safe_release(&sample); - - if (FAILED(hr) || wait > 0) + if (wait > 0) { + // return the sample to scheduler + scheduledSamples.prepend(sample); break; + } } + m_mutex.lock(); + scheduledSamples.append(std::move(m_scheduledSamples)); + m_scheduledSamples.swap(scheduledSamples); + m_mutex.unlock(); + // If the wait time is zero, it means we stopped because the queue is // empty (or an error occurred). Set the wait time to infinite; this will // make the scheduler thread sleep until it gets another thread message. @@ -330,66 +278,50 @@ HRESULT Scheduler::processSamplesInQueue(LONG *nextSleep) return hr; } -HRESULT Scheduler::processSample(IMFSample *sample, LONG *pNextSleep) +bool Scheduler::isSampleReadyToPresent(IMFSample *sample, LONG *pNextSleep) const { - HRESULT hr = S_OK; - - LONGLONG hnsPresentationTime = 0; - LONGLONG hnsTimeNow = 0; - MFTIME hnsSystemTime = 0; - - bool presentNow = true; - LONG nextSleep = 0; - - if (m_clock) { - // Get the sample's time stamp. It is valid for a sample to - // have no time stamp. - hr = sample->GetSampleTime(&hnsPresentationTime); - - // Get the clock time. (But if the sample does not have a time stamp, - // we don't need the clock time.) - if (SUCCEEDED(hr)) - hr = m_clock->GetCorrelatedTime(0, &hnsTimeNow, &hnsSystemTime); - - // Calculate the time until the sample's presentation time. - // A negative value means the sample is late. - LONGLONG hnsDelta = hnsPresentationTime - hnsTimeNow; - if (m_playbackRate < 0) { - // For reverse playback, the clock runs backward. Therefore, the - // delta is reversed. - hnsDelta = - hnsDelta; - } + *pNextSleep = 0; + if (!m_clock) + return true; - if (hnsDelta < - m_perFrame_1_4th) { - // This sample is late. - presentNow = true; - } else if (hnsDelta > (3 * m_perFrame_1_4th)) { - // This sample is still too early. Go to sleep. - nextSleep = MFTimeToMsec(hnsDelta - (3 * m_perFrame_1_4th)); + MFTIME hnsPresentationTime = 0; + MFTIME hnsTimeNow = 0; + MFTIME hnsSystemTime = 0; - // Adjust the sleep time for the clock rate. (The presentation clock runs - // at m_fRate, but sleeping uses the system clock.) - if (m_playbackRate != 0) - nextSleep = (LONG)(nextSleep / qFabs(m_playbackRate)); + // Get the sample's time stamp. It is valid for a sample to + // have no time stamp. + HRESULT hr = sample->GetSampleTime(&hnsPresentationTime); - // Don't present yet. - presentNow = false; - } + // Get the clock time. (But if the sample does not have a time stamp, + // we don't need the clock time.) + if (SUCCEEDED(hr)) + hr = m_clock->GetCorrelatedTime(0, &hnsTimeNow, &hnsSystemTime); + + // Calculate the time until the sample's presentation time. + // A negative value means the sample is late. + MFTIME hnsDelta = hnsPresentationTime - hnsTimeNow; + if (m_playbackRate < 0) { + // For reverse playback, the clock runs backward. Therefore, the + // delta is reversed. + hnsDelta = - hnsDelta; } - if (presentNow) { - m_presenter->presentSample(sample); + if (hnsDelta < - m_perFrame_1_4th) { + // This sample is late - skip. + return false; + } else if (hnsDelta > (3 * m_perFrame_1_4th)) { + // This sample came too early - reschedule + *pNextSleep = MFTimeToMsec(hnsDelta - (3 * m_perFrame_1_4th)); + + // Adjust the sleep time for the clock rate. (The presentation clock runs + // at m_fRate, but sleeping uses the system clock.) + if (m_playbackRate != 0) + *pNextSleep = (LONG)(*pNextSleep / qFabs(m_playbackRate)); + return *pNextSleep == 0; } else { - // The sample is not ready yet. Return it to the queue. - m_mutex.lock(); - sample->AddRef(); - m_scheduledSamples.prepend(sample); - m_mutex.unlock(); + // This sample can be presented right now + return true; } - - *pNextSleep = nextSleep; - - return hr; } DWORD WINAPI Scheduler::schedulerThreadProc(LPVOID parameter) @@ -412,7 +344,7 @@ DWORD Scheduler::schedulerThreadProcPrivate() PeekMessage(&msg, NULL, WM_USER, WM_USER, PM_NOREMOVE); // Signal to the scheduler that the thread is ready. - SetEvent(m_threadReadyEvent); + SetEvent(m_threadReadyEvent.get()); while (!exitThread) { // Wait for a thread message OR until the wait time expires. @@ -435,12 +367,10 @@ DWORD Scheduler::schedulerThreadProcPrivate() case Flush: // Flushing: Clear the sample queue and set the event. m_mutex.lock(); - for (int i = 0; i < m_scheduledSamples.size(); ++i) - m_scheduledSamples[i]->Release(); m_scheduledSamples.clear(); m_mutex.unlock(); wait = INFINITE; - SetEvent(m_flushEvent); + SetEvent(m_flushEvent.get()); break; case Schedule: // Process as many samples as we can. @@ -470,47 +400,44 @@ SamplePool::~SamplePool() clear(); } -HRESULT SamplePool::getSample(IMFSample **sample) +ComPtr<IMFSample> SamplePool::takeSample() { QMutexLocker locker(&m_mutex); - if (!m_initialized) - return MF_E_NOT_INITIALIZED; + Q_ASSERT(m_initialized); + if (!m_initialized) { + qCWarning(qLcEvrCustomPresenter) << "SamplePool is not initialized yet"; + return nullptr; + } - if (m_videoSampleQueue.isEmpty()) - return MF_E_SAMPLEALLOCATOR_EMPTY; + if (m_videoSampleQueue.isEmpty()) { + qCDebug(qLcEvrCustomPresenter) << "SamplePool is empty"; + return nullptr; + } // Get a sample from the allocated queue. // It doesn't matter if we pull them from the head or tail of the list, // but when we get it back, we want to re-insert it onto the opposite end. - // (see ReturnSample) - - IMFSample *taken = m_videoSampleQueue.takeFirst(); + // (see returnSample) - // Give the sample to the caller. - *sample = taken; - (*sample)->AddRef(); - - taken->Release(); - - return S_OK; + return m_videoSampleQueue.takeFirst(); } -HRESULT SamplePool::returnSample(IMFSample *sample) +void SamplePool::returnSample(const ComPtr<IMFSample> &sample) { QMutexLocker locker(&m_mutex); - if (!m_initialized) - return MF_E_NOT_INITIALIZED; + Q_ASSERT(m_initialized); + if (!m_initialized) { + qCWarning(qLcEvrCustomPresenter) << "SamplePool is not initialized yet"; + return; + } m_videoSampleQueue.append(sample); - sample->AddRef(); - - return S_OK; } -HRESULT SamplePool::initialize(QList<IMFSample*> &samples) +HRESULT SamplePool::initialize(QList<ComPtr<IMFSample>> &&samples) { QMutexLocker locker(&m_mutex); @@ -518,16 +445,10 @@ HRESULT SamplePool::initialize(QList<IMFSample*> &samples) return MF_E_INVALIDREQUEST; // Move these samples into our allocated queue. - for (auto sample : qAsConst(samples)) { - sample->AddRef(); - m_videoSampleQueue.append(sample); - } + m_videoSampleQueue.append(std::move(samples)); m_initialized = true; - for (auto sample : qAsConst(samples)) - sample->Release(); - samples.clear(); return S_OK; } @@ -535,8 +456,6 @@ HRESULT SamplePool::clear() { QMutexLocker locker(&m_mutex); - for (auto sample : qAsConst(m_videoSampleQueue)) - sample->Release(); m_videoSampleQueue.clear(); m_initialized = false; @@ -552,14 +471,10 @@ EVRCustomPresenter::EVRCustomPresenter(QVideoSink *sink) , m_scheduler(this) , m_tokenCounter(0) , m_sampleNotify(false) - , m_repaint(false) , m_prerolled(false) , m_endStreaming(false) , m_playbackRate(1.0f) , m_presentEngine(new D3DPresentEngine(sink)) - , m_clock(0) - , m_mixer(0) - , m_mediaEventSink(0) , m_mediaType(0) , m_videoSink(0) , m_canRenderToSurface(false) @@ -580,11 +495,6 @@ EVRCustomPresenter::~EVRCustomPresenter() m_scheduler.stopScheduler(); m_samplePool.clear(); - qt_evr_safe_release(&m_clock); - qt_evr_safe_release(&m_mixer); - qt_evr_safe_release(&m_mediaEventSink); - qt_evr_safe_release(&m_mediaType); - delete m_presentEngine; } @@ -623,7 +533,7 @@ ULONG EVRCustomPresenter::Release() { ULONG uCount = InterlockedDecrement(&m_refCount); if (uCount == 0) - delete this; + deleteLater(); return uCount; } @@ -671,9 +581,9 @@ HRESULT EVRCustomPresenter::InitServicePointers(IMFTopologyServiceLookup *lookup if (isActive()) return MF_E_INVALIDREQUEST; - qt_evr_safe_release(&m_clock); - qt_evr_safe_release(&m_mixer); - qt_evr_safe_release(&m_mediaEventSink); + m_clock.Reset(); + m_mixer.Reset(); + m_mediaEventSink.Reset(); // Ask for the clock. Optional, because the EVR might not have a clock. objectCount = 1; @@ -695,7 +605,7 @@ HRESULT EVRCustomPresenter::InitServicePointers(IMFTopologyServiceLookup *lookup return hr; // Make sure that we can work with this mixer. - hr = configureMixer(m_mixer); + hr = configureMixer(m_mixer.Get()); if (FAILED(hr)) return hr; @@ -729,9 +639,9 @@ HRESULT EVRCustomPresenter::ReleaseServicePointers() setMediaType(NULL); // Release all services that were acquired from InitServicePointers. - qt_evr_safe_release(&m_clock); - qt_evr_safe_release(&m_mixer); - qt_evr_safe_release(&m_mediaEventSink); + m_clock.Reset(); + m_mixer.Reset(); + m_mediaEventSink.Reset(); return S_OK; } @@ -932,8 +842,6 @@ HRESULT EVRCustomPresenter::OnClockSetRate(MFTIME, float rate) // frame-step operation. if ((m_playbackRate == 0.0f) && (rate != 0.0f)) { cancelFrameStep(); - for (auto sample : qAsConst(m_frameStep.samples)) - sample->Release(); m_frameStep.samples.clear(); } @@ -1052,6 +960,13 @@ void EVRCustomPresenter::setSink(QVideoSink *sink) supportedFormatsChanged(); } +void EVRCustomPresenter::setCropRect(QRect cropRect) +{ + m_mutex.lock(); + m_cropRect = cropRect; + m_mutex.unlock(); +} + HRESULT EVRCustomPresenter::configureMixer(IMFTransform *mixer) { // Set the zoom rectangle (ie, the source clipping rectangle). @@ -1129,13 +1044,11 @@ HRESULT EVRCustomPresenter::flush() m_scheduler.flush(); // Flush the frame-step queue. - for (auto sample : qAsConst(m_frameStep.samples)) - sample->Release(); m_frameStep.samples.clear(); if (m_renderState == RenderStopped && m_videoSink) { // Repaint with black. - presentSample(NULL); + presentSample(nullptr); } return S_OK; @@ -1223,9 +1136,6 @@ HRESULT EVRCustomPresenter::prepareFrameStep(DWORD steps) HRESULT EVRCustomPresenter::startFrameStep() { - HRESULT hr = S_OK; - IMFSample *sample = NULL; - if (m_frameStep.state == FrameStepWaitingStart) { // We have a frame-step request, and are waiting for the clock to start. // Set the state to "pending," which means we are waiting for samples. @@ -1233,13 +1143,11 @@ HRESULT EVRCustomPresenter::startFrameStep() // If the frame-step queue already has samples, process them now. while (!m_frameStep.samples.isEmpty() && (m_frameStep.state == FrameStepPending)) { - sample = m_frameStep.samples.takeFirst(); + const ComPtr<IMFSample> sample = m_frameStep.samples.takeFirst(); - hr = deliverFrameStepSample(sample); + const HRESULT hr = deliverFrameStepSample(sample.Get()); if (FAILED(hr)) - goto done; - - qt_evr_safe_release(&sample); + return hr; // We break from this loop when: // (a) the frame-step queue is empty, or @@ -1249,22 +1157,18 @@ HRESULT EVRCustomPresenter::startFrameStep() // We are not frame stepping. Therefore, if the frame-step queue has samples, // we need to process them normally. while (!m_frameStep.samples.isEmpty()) { - sample = m_frameStep.samples.takeFirst(); + const ComPtr<IMFSample> sample = m_frameStep.samples.takeFirst(); - hr = deliverSample(sample, false); + const HRESULT hr = deliverSample(sample.Get()); if (FAILED(hr)) - goto done; - - qt_evr_safe_release(&sample); + return hr; } } -done: - qt_evr_safe_release(&sample); - return hr; + return S_OK; } -HRESULT EVRCustomPresenter::completeFrameStep(IMFSample *sample) +HRESULT EVRCustomPresenter::completeFrameStep(const ComPtr<IMFSample> &sample) { HRESULT hr = S_OK; MFTIME sampleTime = 0; @@ -1342,13 +1246,30 @@ HRESULT EVRCustomPresenter::createOptimalVideoType(IMFMediaType *proposedType, I hr = proposedType->GetUINT64(MF_MT_FRAME_SIZE, &size); width = int(HI32(size)); height = int(LO32(size)); - rcOutput.left = 0; - rcOutput.top = 0; - rcOutput.right = width; - rcOutput.bottom = height; + + if (m_cropRect.isValid()) { + rcOutput.left = m_cropRect.x(); + rcOutput.top = m_cropRect.y(); + rcOutput.right = m_cropRect.x() + m_cropRect.width(); + rcOutput.bottom = m_cropRect.y() + m_cropRect.height(); + + m_sourceRect.left = float(m_cropRect.x()) / width; + m_sourceRect.top = float(m_cropRect.y()) / height; + m_sourceRect.right = float(m_cropRect.x() + m_cropRect.width()) / width; + m_sourceRect.bottom = float(m_cropRect.y() + m_cropRect.height()) / height; + + if (m_mixer) + configureMixer(m_mixer.Get()); + } else { + rcOutput.left = 0; + rcOutput.top = 0; + rcOutput.right = width; + rcOutput.bottom = height; + } // Set the geometric aperture, and disable pan/scan. - displayArea = qt_evr_makeMFArea(0, 0, rcOutput.right, rcOutput.bottom); + displayArea = qt_evr_makeMFArea(0, 0, rcOutput.right - rcOutput.left, + rcOutput.bottom - rcOutput.top); hr = mtOptimal->SetUINT32(MF_MT_PAN_SCAN_ENABLED, FALSE); if (FAILED(hr)) @@ -1389,13 +1310,13 @@ HRESULT EVRCustomPresenter::setMediaType(IMFMediaType *mediaType) // Clearing the media type is allowed in any state (including shutdown). if (!mediaType) { stopSurface(); - qt_evr_safe_release(&m_mediaType); + m_mediaType.Reset(); releaseResources(); return S_OK; } MFRatio fps = { 0, 0 }; - QList<IMFSample*> sampleQueue; + QList<ComPtr<IMFSample>> sampleQueue; // Cannot set the media type after shutdown. HRESULT hr = checkShutdown(); @@ -1404,30 +1325,30 @@ HRESULT EVRCustomPresenter::setMediaType(IMFMediaType *mediaType) // Check if the new type is actually different. // Note: This function safely handles NULL input parameters. - if (qt_evr_areMediaTypesEqual(m_mediaType, mediaType)) + if (qt_evr_areMediaTypesEqual(m_mediaType.Get(), mediaType)) goto done; // Nothing more to do. // We're really changing the type. First get rid of the old type. - qt_evr_safe_release(&m_mediaType); + m_mediaType.Reset(); releaseResources(); // Initialize the presenter engine with the new media type. // The presenter engine allocates the samples. - hr = m_presentEngine->createVideoSamples(mediaType, sampleQueue); + hr = m_presentEngine->createVideoSamples(mediaType, sampleQueue, m_cropRect.size()); if (FAILED(hr)) goto done; // Mark each sample with our token counter. If this batch of samples becomes // invalid, we increment the counter, so that we know they should be discarded. - for (auto sample : qAsConst(sampleQueue)) { + for (auto sample : std::as_const(sampleQueue)) { hr = sample->SetUINT32(MFSamplePresenter_SampleCounter, m_tokenCounter); if (FAILED(hr)) goto done; } // Add the samples to the sample pool. - hr = m_samplePool.initialize(sampleQueue); + hr = m_samplePool.initialize(std::move(sampleQueue)); if (FAILED(hr)) goto done; @@ -1544,21 +1465,9 @@ void EVRCustomPresenter::processOutputLoop() HRESULT EVRCustomPresenter::processOutput() { - HRESULT hr = S_OK; - DWORD status = 0; - LONGLONG mixerStartTime = 0, mixerEndTime = 0; - MFTIME systemTime = 0; - BOOL repaint = m_repaint; // Temporarily store this state flag. - - MFT_OUTPUT_DATA_BUFFER dataBuffer; - ZeroMemory(&dataBuffer, sizeof(dataBuffer)); - - IMFSample *sample = NULL; - // If the clock is not running, we present the first sample, // and then don't present any more until the clock starts. - - if ((m_renderState != RenderStarted) && !m_repaint && m_prerolled) + if ((m_renderState != RenderStarted) && m_prerolled) return S_FALSE; // Make sure we have a pointer to the mixer. @@ -1566,45 +1475,33 @@ HRESULT EVRCustomPresenter::processOutput() return MF_E_INVALIDREQUEST; // Try to get a free sample from the video sample pool. - hr = m_samplePool.getSample(&sample); - if (hr == MF_E_SAMPLEALLOCATOR_EMPTY) // No free samples. Try again when a sample is released. - return S_FALSE; - if (FAILED(hr)) - return hr; + ComPtr<IMFSample> sample = m_samplePool.takeSample(); + if (!sample) + return S_FALSE; // No free samples. Try again when a sample is released. // From now on, we have a valid video sample pointer, where the mixer will // write the video data. - if (m_repaint) { - // Repaint request. Ask the mixer for the most recent sample. - setDesiredSampleTime(sample, m_scheduler.lastSampleTime(), m_scheduler.frameDuration()); - - m_repaint = false; // OK to clear this flag now. - } else { - // Not a repaint request. Clear the desired sample time; the mixer will - // give us the next frame in the stream. - clearDesiredSampleTime(sample); + LONGLONG mixerStartTime = 0, mixerEndTime = 0; + MFTIME systemTime = 0; - if (m_clock) { - // Latency: Record the starting time for ProcessOutput. - m_clock->GetCorrelatedTime(0, &mixerStartTime, &systemTime); - } + if (m_clock) { + // Latency: Record the starting time for ProcessOutput. + m_clock->GetCorrelatedTime(0, &mixerStartTime, &systemTime); } // Now we are ready to get an output sample from the mixer. - dataBuffer.dwStreamID = 0; - dataBuffer.pSample = sample; - dataBuffer.dwStatus = 0; - - hr = m_mixer->ProcessOutput(0, 1, &dataBuffer, &status); + DWORD status = 0; + MFT_OUTPUT_DATA_BUFFER dataBuffer = {}; + dataBuffer.pSample = sample.Get(); + HRESULT hr = m_mixer->ProcessOutput(0, 1, &dataBuffer, &status); + // Important: Release any events returned from the ProcessOutput method. + qt_evr_safe_release(&dataBuffer.pEvents); if (FAILED(hr)) { // Return the sample to the pool. - HRESULT hr2 = m_samplePool.returnSample(sample); - if (FAILED(hr2)) { - hr = hr2; - goto done; - } + m_samplePool.returnSample(sample); + // Handle some known error codes from ProcessOutput. if (hr == MF_E_TRANSFORM_TYPE_NOT_SET) { // The mixer's format is not set. Negotiate a new format. @@ -1617,54 +1514,46 @@ HRESULT EVRCustomPresenter::processOutput() // We have to wait for the mixer to get more input. m_sampleNotify = false; } - } else { - // We got an output sample from the mixer. - if (m_clock && !repaint) { - // Latency: Record the ending time for the ProcessOutput operation, - // and notify the EVR of the latency. + return hr; + } - m_clock->GetCorrelatedTime(0, &mixerEndTime, &systemTime); + // We got an output sample from the mixer. + if (m_clock) { + // Latency: Record the ending time for the ProcessOutput operation, + // and notify the EVR of the latency. - LONGLONG latencyTime = mixerEndTime - mixerStartTime; - notifyEvent(EC_PROCESSING_LATENCY, reinterpret_cast<LONG_PTR>(&latencyTime), 0); - } + m_clock->GetCorrelatedTime(0, &mixerEndTime, &systemTime); - // Set up notification for when the sample is released. - hr = trackSample(sample); - if (FAILED(hr)) - goto done; + LONGLONG latencyTime = mixerEndTime - mixerStartTime; + notifyEvent(EC_PROCESSING_LATENCY, reinterpret_cast<LONG_PTR>(&latencyTime), 0); + } - // Schedule the sample. - if ((m_frameStep.state == FrameStepNone) || repaint) { - hr = deliverSample(sample, repaint); - if (FAILED(hr)) - goto done; - } else { - // We are frame-stepping (and this is not a repaint request). - hr = deliverFrameStepSample(sample); - if (FAILED(hr)) - goto done; - } + // Set up notification for when the sample is released. + hr = trackSample(sample); + if (FAILED(hr)) + return hr; - m_prerolled = true; // We have presented at least one sample now. - } + // Schedule the sample. + if (m_frameStep.state == FrameStepNone) + hr = deliverSample(sample); + else // We are frame-stepping + hr = deliverFrameStepSample(sample); -done: - qt_evr_safe_release(&sample); + if (FAILED(hr)) + return hr; - // Important: Release any events returned from the ProcessOutput method. - qt_evr_safe_release(&dataBuffer.pEvents); - return hr; + m_prerolled = true; // We have presented at least one sample now. + return S_OK; } -HRESULT EVRCustomPresenter::deliverSample(IMFSample *sample, bool repaint) +HRESULT EVRCustomPresenter::deliverSample(const ComPtr<IMFSample> &sample) { - // If we are not actively playing, OR we are scrubbing (rate = 0) OR this is a - // repaint request, then we need to present the sample immediately. Otherwise, + // If we are not actively playing, OR we are scrubbing (rate = 0), + // then we need to present the sample immediately. Otherwise, // schedule it normally. - bool presentNow = ((m_renderState != RenderStarted) || isScrubbing() || repaint); + bool presentNow = ((m_renderState != RenderStarted) || isScrubbing()); HRESULT hr = m_scheduler.scheduleSample(sample, presentNow); @@ -1678,19 +1567,18 @@ HRESULT EVRCustomPresenter::deliverSample(IMFSample *sample, bool repaint) return hr; } -HRESULT EVRCustomPresenter::deliverFrameStepSample(IMFSample *sample) +HRESULT EVRCustomPresenter::deliverFrameStepSample(const ComPtr<IMFSample> &sample) { HRESULT hr = S_OK; IUnknown *unk = NULL; // For rate 0, discard any sample that ends earlier than the clock time. - if (isScrubbing() && m_clock && qt_evr_isSampleTimePassed(m_clock, sample)) { + if (isScrubbing() && m_clock && qt_evr_isSampleTimePassed(m_clock.Get(), sample.Get())) { // Discard this sample. } else if (m_frameStep.state >= FrameStepScheduled) { // A frame was already submitted. Put this sample on the frame-step queue, // in case we are asked to step to the next frame. If frame-stepping is // cancelled, this sample will be processed normally. - sample->AddRef(); m_frameStep.samples.append(sample); } else { // We're ready to frame-step. @@ -1705,11 +1593,10 @@ HRESULT EVRCustomPresenter::deliverFrameStepSample(IMFSample *sample) // This is the right frame, but the clock hasn't started yet. Put the // sample on the frame-step queue. When the clock starts, the sample // will be processed. - sample->AddRef(); m_frameStep.samples.append(sample); } else { // This is the right frame *and* the clock has started. Deliver this sample. - hr = deliverSample(sample, false); + hr = deliverSample(sample); if (FAILED(hr)) goto done; @@ -1734,7 +1621,7 @@ done: return hr; } -HRESULT EVRCustomPresenter::trackSample(IMFSample *sample) +HRESULT EVRCustomPresenter::trackSample(const ComPtr<IMFSample> &sample) { IMFTrackedSample *tracked = NULL; @@ -1811,11 +1698,9 @@ HRESULT EVRCustomPresenter::onSampleFree(IMFAsyncResult *result) if (token == m_tokenCounter) { // Return the sample to the sample pool. - hr = m_samplePool.returnSample(sample); - if (SUCCEEDED(hr)) { - // A free sample is available. Process more data if possible. - processOutputLoop(); - } + m_samplePool.returnSample(sample); + // A free sample is available. Process more data if possible. + processOutputLoop(); } m_mutex.unlock(); @@ -1843,7 +1728,7 @@ float EVRCustomPresenter::getMaxRate(bool thin) UINT monitorRateHz = 0; if (!thin && m_mediaType) { - qt_evr_getFrameRate(m_mediaType, &fps); + qt_evr_getFrameRate(m_mediaType.Get(), &fps); monitorRateHz = m_presentEngine->refreshRate(); if (fps.Denominator && fps.Numerator && monitorRateHz) { @@ -1889,7 +1774,7 @@ void EVRCustomPresenter::stopSurface() } } -void EVRCustomPresenter::presentSample(IMFSample *sample) +void EVRCustomPresenter::presentSample(const ComPtr<IMFSample> &sample) { if (thread() != QThread::currentThread()) { QCoreApplication::postEvent(this, new PresentSampleEvent(sample)); @@ -1910,15 +1795,15 @@ void EVRCustomPresenter::presentSample(IMFSample *sample) frame.setEndTime(frame.endTime() + m_positionOffset); } - QWindowsIUPointer<IMFMediaType> inputStreamType; - if (SUCCEEDED(m_mixer->GetInputCurrentType(0, inputStreamType.address()))) { - auto rotation = static_cast<MFVideoRotationFormat>(MFGetAttributeUINT32(inputStreamType.get(), MF_MT_VIDEO_ROTATION, 0)); + ComPtr<IMFMediaType> inputStreamType; + if (SUCCEEDED(m_mixer->GetInputCurrentType(0, inputStreamType.GetAddressOf()))) { + auto rotation = static_cast<MFVideoRotationFormat>(MFGetAttributeUINT32(inputStreamType.Get(), MF_MT_VIDEO_ROTATION, 0)); switch (rotation) { - case MFVideoRotationFormat_0: frame.setRotationAngle(QVideoFrame::Rotation0); break; - case MFVideoRotationFormat_90: frame.setRotationAngle(QVideoFrame::Rotation90); break; - case MFVideoRotationFormat_180: frame.setRotationAngle(QVideoFrame::Rotation180); break; - case MFVideoRotationFormat_270: frame.setRotationAngle(QVideoFrame::Rotation270); break; - default: frame.setRotationAngle(QVideoFrame::Rotation0); + case MFVideoRotationFormat_0: frame.setRotation(QtVideo::Rotation::None); break; + case MFVideoRotationFormat_90: frame.setRotation(QtVideo::Rotation::Clockwise90); break; + case MFVideoRotationFormat_180: frame.setRotation(QtVideo::Rotation::Clockwise180); break; + case MFVideoRotationFormat_270: frame.setRotation(QtVideo::Rotation::Clockwise270); break; + default: frame.setRotation(QtVideo::Rotation::None); } } @@ -1930,55 +1815,6 @@ void EVRCustomPresenter::positionChanged(qint64 position) m_positionOffset = position * 1000; } -HRESULT setDesiredSampleTime(IMFSample *sample, const LONGLONG &sampleTime, const LONGLONG &duration) -{ - if (!sample) - return E_POINTER; - - HRESULT hr = S_OK; - IMFDesiredSample *desired = NULL; - - hr = sample->QueryInterface(IID_PPV_ARGS(&desired)); - if (SUCCEEDED(hr)) - desired->SetDesiredSampleTimeAndDuration(sampleTime, duration); - - qt_evr_safe_release(&desired); - return hr; -} - -HRESULT clearDesiredSampleTime(IMFSample *sample) -{ - if (!sample) - return E_POINTER; - - HRESULT hr = S_OK; - - IMFDesiredSample *desired = NULL; - IUnknown *unkSwapChain = NULL; - - // We store some custom attributes on the sample, so we need to cache them - // and reset them. - // - // This works around the fact that IMFDesiredSample::Clear() removes all of the - // attributes from the sample. - - UINT32 counter = MFGetAttributeUINT32(sample, MFSamplePresenter_SampleCounter, (UINT32)-1); - - hr = sample->QueryInterface(IID_PPV_ARGS(&desired)); - if (SUCCEEDED(hr)) { - desired->Clear(); - - hr = sample->SetUINT32(MFSamplePresenter_SampleCounter, counter); - if (FAILED(hr)) - goto done; - } - -done: - qt_evr_safe_release(&unkSwapChain); - qt_evr_safe_release(&desired); - return hr; -} - HRESULT setMixerSourceRect(IMFTransform *mixer, const MFVideoNormalizedRect &sourceRect) { if (!mixer) diff --git a/src/plugins/multimedia/windows/evr/evrcustompresenter_p.h b/src/plugins/multimedia/windows/evr/evrcustompresenter_p.h index 1bf443efa..28f1cbc68 100644 --- a/src/plugins/multimedia/windows/evr/evrcustompresenter_p.h +++ b/src/plugins/multimedia/windows/evr/evrcustompresenter_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef EVRCUSTOMPRESENTER_H #define EVRCUSTOMPRESENTER_H @@ -55,8 +19,12 @@ #include <qmutex.h> #include <qqueue.h> #include <qevent.h> +#include <qrect.h> #include <qvideoframeformat.h> #include <qvideosink.h> +#include <qpointer.h> +#include <private/qcomptr_p.h> +#include "evrhelpers_p.h" #include <d3d9.h> #include <dxva2api.h> @@ -142,15 +110,11 @@ public: void setFrameRate(const MFRatio &fps); void setClockRate(float rate) { m_playbackRate = rate; } - const LONGLONG &lastSampleTime() const { return m_lastSampleTime; } - const LONGLONG &frameDuration() const { return m_perFrameInterval; } - - HRESULT startScheduler(IMFClock *clock); + HRESULT startScheduler(ComPtr<IMFClock> clock); HRESULT stopScheduler(); - HRESULT scheduleSample(IMFSample *sample, bool presentNow); + HRESULT scheduleSample(const ComPtr<IMFSample> &sample, bool presentNow); HRESULT processSamplesInQueue(LONG *nextSleep); - HRESULT processSample(IMFSample *sample, LONG *nextSleep); HRESULT flush(); bool areSamplesScheduled(); @@ -160,22 +124,21 @@ public: private: DWORD schedulerThreadProcPrivate(); + bool isSampleReadyToPresent(IMFSample *sample, LONG *pNextSleep) const; EVRCustomPresenter *m_presenter; - QQueue<IMFSample*> m_scheduledSamples; // Samples waiting to be presented. + QQueue<ComPtr<IMFSample>> m_scheduledSamples; // Samples waiting to be presented. - IMFClock *m_clock; // Presentation clock. Can be NULL. + ComPtr<IMFClock> m_clock; // Presentation clock. Can be NULL. DWORD m_threadID; - HANDLE m_schedulerThread; - HANDLE m_threadReadyEvent; - HANDLE m_flushEvent; + ThreadHandle m_schedulerThread; + EventHandle m_threadReadyEvent; + EventHandle m_flushEvent; float m_playbackRate; - MFTIME m_perFrameInterval; // Duration of each frame. - LONGLONG m_perFrame_1_4th; // 1/4th of the frame duration. - MFTIME m_lastSampleTime; // Most recent sample time. + MFTIME m_perFrame_1_4th; // 1/4th of the frame duration. QMutex m_mutex; }; @@ -187,15 +150,15 @@ public: SamplePool(); ~SamplePool(); - HRESULT initialize(QList<IMFSample*> &samples); + HRESULT initialize(QList<ComPtr<IMFSample>> &&samples); HRESULT clear(); - HRESULT getSample(IMFSample **sample); - HRESULT returnSample(IMFSample *sample); + ComPtr<IMFSample> takeSample(); + void returnSample(const ComPtr<IMFSample> &sample); private: QMutex m_mutex; - QList<IMFSample*> m_videoSampleQueue; + QList<ComPtr<IMFSample>> m_videoSampleQueue; bool m_initialized; }; @@ -273,10 +236,11 @@ public: void supportedFormatsChanged(); void setSink(QVideoSink *sink); + void setCropRect(QRect cropRect); void startSurface(); void stopSurface(); - void presentSample(IMFSample *sample); + void presentSample(const ComPtr<IMFSample> &sample); bool event(QEvent *) override; @@ -329,15 +293,15 @@ private: // Managing samples void processOutputLoop(); HRESULT processOutput(); - HRESULT deliverSample(IMFSample *sample, bool repaint); - HRESULT trackSample(IMFSample *sample); + HRESULT deliverSample(const ComPtr<IMFSample> &sample); + HRESULT trackSample(const ComPtr<IMFSample> &sample); void releaseResources(); // Frame-stepping HRESULT prepareFrameStep(DWORD steps); HRESULT startFrameStep(); - HRESULT deliverFrameStepSample(IMFSample *sample); - HRESULT completeFrameStep(IMFSample *sample); + HRESULT deliverFrameStepSample(const ComPtr<IMFSample> &sample); + HRESULT completeFrameStep(const ComPtr<IMFSample> &sample); HRESULT cancelFrameStep(); // Callback when a video sample is released. @@ -348,7 +312,7 @@ private: struct FrameStep { FrameStepState state = FrameStepNone; - QList<IMFSample*> samples; + QList<ComPtr<IMFSample>> samples; DWORD steps = 0; DWORD_PTR sampleNoRef = 0; }; @@ -367,7 +331,6 @@ private: // Rendering state bool m_sampleNotify; // Did the mixer signal it has an input sample? - bool m_repaint; // Do we need to repaint the last sample? bool m_prerolled; // Have we presented at least one sample? bool m_endStreaming; // Did we reach the end of the stream (EOS)? @@ -376,14 +339,15 @@ private: D3DPresentEngine *m_presentEngine; // Rendering engine. (Never null if the constructor succeeds.) - IMFClock *m_clock; // The EVR's clock. - IMFTransform *m_mixer; // The EVR's mixer. - IMediaEventSink *m_mediaEventSink; // The EVR's event-sink interface. - IMFMediaType *m_mediaType; // Output media type + ComPtr<IMFClock> m_clock; // The EVR's clock. + ComPtr<IMFTransform> m_mixer; // The EVR's mixer. + ComPtr<IMediaEventSink> m_mediaEventSink; // The EVR's event-sink interface. + ComPtr<IMFMediaType> m_mediaType; // Output media type - QVideoSink *m_videoSink; + QPointer<QVideoSink> m_videoSink; bool m_canRenderToSurface; qint64 m_positionOffset; // Seek position in microseconds. + QRect m_cropRect; // Video crop rectangle }; bool qt_evr_setCustomPresenter(IUnknown *evr, EVRCustomPresenter *presenter); diff --git a/src/plugins/multimedia/windows/evr/evrd3dpresentengine.cpp b/src/plugins/multimedia/windows/evr/evrd3dpresentengine.cpp index 96c7836cf..517f1d969 100644 --- a/src/plugins/multimedia/windows/evr/evrd3dpresentengine.cpp +++ b/src/plugins/multimedia/windows/evr/evrd3dpresentengine.cpp @@ -1,47 +1,12 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "evrd3dpresentengine_p.h" #include "evrhelpers_p.h" -#include <private/qabstractvideobuffer_p.h> +#include <private/qhwvideobuffer_p.h> +#include <private/qvideoframe_p.h> #include <qvideoframe.h> #include <QDebug> #include <qthread.h> @@ -50,8 +15,7 @@ #include <d3d11_1.h> -#include <private/qrhi_p.h> -#include <private/qrhid3d11_p.h> +#include <rhi/qrhi.h> #if QT_CONFIG(opengl) # include <qopenglcontext.h> @@ -61,32 +25,29 @@ QT_BEGIN_NAMESPACE -Q_LOGGING_CATEGORY(qLcEvrD3DPresentEngine, "qt.multimedia.evrd3dpresentengine") +static Q_LOGGING_CATEGORY(qLcEvrD3DPresentEngine, "qt.multimedia.evrd3dpresentengine"); -class IMFSampleVideoBuffer: public QAbstractVideoBuffer +class IMFSampleVideoBuffer : public QHwVideoBuffer { public: - IMFSampleVideoBuffer(QWindowsIUPointer<IDirect3DDevice9Ex> device, - IMFSample *sample, QRhi *rhi, QVideoFrame::HandleType type = QVideoFrame::NoHandle) - : QAbstractVideoBuffer(type, rhi) - , m_device(device) - , m_mapMode(QVideoFrame::NotMapped) + IMFSampleVideoBuffer(ComPtr<IDirect3DDevice9Ex> device, const ComPtr<IMFSample> &sample, + QRhi *rhi, QVideoFrame::HandleType type = QVideoFrame::NoHandle) + : QHwVideoBuffer(type, rhi), + m_device(device), + m_sample(sample), + m_mapMode(QtVideo::MapMode::NotMapped) { - sample->AddRef(); - m_sample.reset(sample); } ~IMFSampleVideoBuffer() override { - if (m_memSurface && m_mapMode != QVideoFrame::NotMapped) + if (m_memSurface && m_mapMode != QtVideo::MapMode::NotMapped) m_memSurface->UnlockRect(); } - QVideoFrame::MapMode mapMode() const override { return m_mapMode; } - - MapData map(QVideoFrame::MapMode mode) override + MapData map(QtVideo::MapMode mode) override { - if (!m_sample || m_mapMode != QVideoFrame::NotMapped || mode != QVideoFrame::ReadOnly) + if (!m_sample || m_mapMode != QtVideo::MapMode::NotMapped || mode != QtVideo::MapMode::ReadOnly) return {}; D3DSURFACE_DESC desc; @@ -95,123 +56,161 @@ public: return {}; } else { - QWindowsIUPointer<IMFMediaBuffer> buffer; - HRESULT hr = m_sample->GetBufferByIndex(0, buffer.address()); + ComPtr<IMFMediaBuffer> buffer; + HRESULT hr = m_sample->GetBufferByIndex(0, buffer.GetAddressOf()); if (FAILED(hr)) return {}; - QWindowsIUPointer<IDirect3DSurface9> surface; - hr = MFGetService(buffer.get(), MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void **)(surface.address())); + ComPtr<IDirect3DSurface9> surface; + hr = MFGetService(buffer.Get(), MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void **)(surface.GetAddressOf())); if (FAILED(hr)) return {}; if (FAILED(surface->GetDesc(&desc))) return {}; - if (FAILED(m_device->CreateOffscreenPlainSurface(desc.Width, desc.Height, desc.Format, D3DPOOL_SYSTEMMEM, m_memSurface.address(), nullptr))) + if (FAILED(m_device->CreateOffscreenPlainSurface(desc.Width, desc.Height, desc.Format, D3DPOOL_SYSTEMMEM, m_memSurface.GetAddressOf(), nullptr))) return {}; - if (FAILED(m_device->GetRenderTargetData(surface.get(), m_memSurface.get()))) { - m_memSurface.reset(); + if (FAILED(m_device->GetRenderTargetData(surface.Get(), m_memSurface.Get()))) { + m_memSurface.Reset(); return {}; } } D3DLOCKED_RECT rect; - if (FAILED(m_memSurface->LockRect(&rect, NULL, mode == QVideoFrame::ReadOnly ? D3DLOCK_READONLY : 0))) + if (FAILED(m_memSurface->LockRect(&rect, NULL, mode == QtVideo::MapMode::ReadOnly ? D3DLOCK_READONLY : 0))) return {}; m_mapMode = mode; MapData mapData; - mapData.nPlanes = 1; + mapData.planeCount = 1; mapData.bytesPerLine[0] = (int)rect.Pitch; mapData.data[0] = reinterpret_cast<uchar *>(rect.pBits); - mapData.size[0] = (int)(rect.Pitch * desc.Height); + mapData.dataSize[0] = (int)(rect.Pitch * desc.Height); return mapData; } void unmap() override { - if (m_mapMode == QVideoFrame::NotMapped) + if (m_mapMode == QtVideo::MapMode::NotMapped) return; - m_mapMode = QVideoFrame::NotMapped; + m_mapMode = QtVideo::MapMode::NotMapped; if (m_memSurface) m_memSurface->UnlockRect(); } protected: - QWindowsIUPointer<IDirect3DDevice9Ex> m_device; - QWindowsIUPointer<IMFSample> m_sample; + ComPtr<IDirect3DDevice9Ex> m_device; + ComPtr<IMFSample> m_sample; private: - QWindowsIUPointer<IDirect3DSurface9> m_memSurface; - QVideoFrame::MapMode m_mapMode; + ComPtr<IDirect3DSurface9> m_memSurface; + QtVideo::MapMode m_mapMode; +}; + +class QVideoFrameD3D11Textures: public QVideoFrameTextures +{ +public: + QVideoFrameD3D11Textures(std::unique_ptr<QRhiTexture> &&tex, ComPtr<ID3D11Texture2D> &&d3d11tex) + : m_tex(std::move(tex)) + , m_d3d11tex(std::move(d3d11tex)) + {} + + QRhiTexture *texture(uint plane) const override + { + return plane == 0 ? m_tex.get() : nullptr; + }; + +private: + std::unique_ptr<QRhiTexture> m_tex; + ComPtr<ID3D11Texture2D> m_d3d11tex; }; class D3D11TextureVideoBuffer: public IMFSampleVideoBuffer { public: - D3D11TextureVideoBuffer(QWindowsIUPointer<IDirect3DDevice9Ex> device, IMFSample *sample, - QWindowsIUPointer<ID3D11Texture2D> d2d11tex, QRhi *rhi) - : IMFSampleVideoBuffer(device, sample, rhi, QVideoFrame::RhiTextureHandle) - , m_d2d11tex(d2d11tex) + D3D11TextureVideoBuffer(ComPtr<IDirect3DDevice9Ex> device, const ComPtr<IMFSample> &sample, + HANDLE sharedHandle, QRhi *rhi) + : IMFSampleVideoBuffer(std::move(device), sample, rhi, QVideoFrame::RhiTextureHandle) + , m_sharedHandle(sharedHandle) {} - std::unique_ptr<QRhiTexture> texture(int plane) const override + std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *rhi) override { - if (!m_rhi || !m_d2d11tex || plane > 0) + if (!rhi || rhi->backend() != QRhi::D3D11) return {}; - D3D11_TEXTURE2D_DESC desc = {}; - m_d2d11tex->GetDesc(&desc); - QRhiTexture::Format format; - if (desc.Format == DXGI_FORMAT_B8G8R8A8_UNORM) - format = QRhiTexture::BGRA8; - else if (desc.Format == DXGI_FORMAT_R8G8B8A8_UNORM) - format = QRhiTexture::RGBA8; - else + + auto nh = static_cast<const QRhiD3D11NativeHandles*>(rhi->nativeHandles()); + if (!nh) return {}; - std::unique_ptr<QRhiTexture> tex(m_rhi->newTexture(format, QSize{int(desc.Width), int(desc.Height)}, 1, {})); - tex->createFrom({quint64(m_d2d11tex.get()), 0}); - return tex; + auto dev = reinterpret_cast<ID3D11Device *>(nh->dev); + if (!dev) + return {}; + + ComPtr<ID3D11Texture2D> d3d11tex; + HRESULT hr = dev->OpenSharedResource(m_sharedHandle, __uuidof(ID3D11Texture2D), (void**)(d3d11tex.GetAddressOf())); + if (SUCCEEDED(hr)) { + D3D11_TEXTURE2D_DESC desc = {}; + d3d11tex->GetDesc(&desc); + QRhiTexture::Format format; + if (desc.Format == DXGI_FORMAT_B8G8R8A8_UNORM) + format = QRhiTexture::BGRA8; + else if (desc.Format == DXGI_FORMAT_R8G8B8A8_UNORM) + format = QRhiTexture::RGBA8; + else + return {}; + + std::unique_ptr<QRhiTexture> tex(rhi->newTexture(format, QSize{int(desc.Width), int(desc.Height)}, 1, {})); + tex->createFrom({quint64(d3d11tex.Get()), 0}); + return std::make_unique<QVideoFrameD3D11Textures>(std::move(tex), std::move(d3d11tex)); + + } else { + qCDebug(qLcEvrD3DPresentEngine) << "Failed to obtain D3D11Texture2D from D3D9Texture2D handle"; + } + return {}; } private: - QWindowsIUPointer<ID3D11Texture2D> m_d2d11tex; + HANDLE m_sharedHandle = nullptr; }; #if QT_CONFIG(opengl) -class OpenGlVideoBuffer: public IMFSampleVideoBuffer +class QVideoFrameOpenGlTextures : public QVideoFrameTextures { + struct InterOpHandles { + GLuint textureName = 0; + HANDLE device = nullptr; + HANDLE texture = nullptr; + }; + public: - OpenGlVideoBuffer(QWindowsIUPointer<IDirect3DDevice9Ex> device, IMFSample *sample, - const WglNvDxInterop &wglNvDxInterop, HANDLE sharedHandle, QRhi *rhi) - : IMFSampleVideoBuffer(device, sample, rhi, QVideoFrame::RhiTextureHandle) - , m_sharedHandle(sharedHandle) - , m_wgl(wglNvDxInterop) - {} + Q_DISABLE_COPY(QVideoFrameOpenGlTextures); - ~OpenGlVideoBuffer() override - { - if (!m_d3dglHandle) - return; + QVideoFrameOpenGlTextures(std::unique_ptr<QRhiTexture> &&tex, const WglNvDxInterop &wgl, InterOpHandles &handles) + : m_tex(std::move(tex)) + , m_wgl(wgl) + , m_handles(handles) + {} + ~QVideoFrameOpenGlTextures() override { if (QOpenGLContext::currentContext()) { - if (m_glHandle) { - if (!m_wgl.wglDXUnlockObjectsNV(m_d3dglHandle, 1, &m_glHandle)) - qCDebug(qLcEvrD3DPresentEngine) << "Failed to unlock OpenGL texture"; - if (!m_wgl.wglDXUnregisterObjectNV(m_d3dglHandle, m_glHandle)) - qCDebug(qLcEvrD3DPresentEngine) << "Failed to unregister OpenGL texture"; - - QOpenGLFunctions *funcs = QOpenGLContext::currentContext()->functions(); - if (funcs) - funcs->glDeleteTextures(1, &m_glTextureName); - else - qCDebug(qLcEvrD3DPresentEngine) << "Could not delete texture, OpenGL context functions missing"; - } - if (!m_wgl.wglDXCloseDeviceNV(m_d3dglHandle)) + if (!m_wgl.wglDXUnlockObjectsNV(m_handles.device, 1, &m_handles.texture)) + qCDebug(qLcEvrD3DPresentEngine) << "Failed to unlock OpenGL texture"; + + if (!m_wgl.wglDXUnregisterObjectNV(m_handles.device, m_handles.texture)) + qCDebug(qLcEvrD3DPresentEngine) << "Failed to unregister OpenGL texture"; + + QOpenGLFunctions *funcs = QOpenGLContext::currentContext()->functions(); + if (funcs) + funcs->glDeleteTextures(1, &m_handles.textureName); + else + qCDebug(qLcEvrD3DPresentEngine) << "Could not delete texture, OpenGL context functions missing"; + + if (!m_wgl.wglDXCloseDeviceNV(m_handles.device)) qCDebug(qLcEvrD3DPresentEngine) << "Failed to close D3D-GL device"; } else { @@ -219,87 +218,106 @@ public: } } - void mapTextures() override + static std::unique_ptr<QVideoFrameOpenGlTextures> create(const WglNvDxInterop &wgl, QRhi *rhi, + IDirect3DDevice9Ex *device, IDirect3DTexture9 *texture, + HANDLE sharedHandle) { - if (!QOpenGLContext::currentContext()) - return; - - if (m_d3dglHandle) - return; - - QWindowsIUPointer<IMFMediaBuffer> buffer; - HRESULT hr = m_sample->GetBufferByIndex(0, buffer.address()); - if (FAILED(hr)) - return; - - QWindowsIUPointer<IDirect3DSurface9> surface; - hr = MFGetService(buffer.get(), MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void **)(surface.address())); - if (FAILED(hr)) - return; + if (!rhi || rhi->backend() != QRhi::OpenGLES2) + return {}; - hr = surface->GetContainer(IID_IDirect3DTexture9, (void **)m_texture.address()); - if (FAILED(hr)) - return; + if (!QOpenGLContext::currentContext()) + return {}; - m_d3dglHandle = m_wgl.wglDXOpenDeviceNV(m_device.get()); - if (!m_d3dglHandle) { - m_texture.reset(); + InterOpHandles handles = {}; + handles.device = wgl.wglDXOpenDeviceNV(device); + if (!handles.device) { qCDebug(qLcEvrD3DPresentEngine) << "Failed to open D3D device"; - return; + return {}; } - m_wgl.wglDXSetResourceShareHandleNV(m_texture.get(), m_sharedHandle); + wgl.wglDXSetResourceShareHandleNV(texture, sharedHandle); QOpenGLFunctions *funcs = QOpenGLContext::currentContext()->functions(); if (funcs) { - funcs->glGenTextures(1, &m_glTextureName); - m_glHandle = m_wgl.wglDXRegisterObjectNV(m_d3dglHandle, m_texture.get(), m_glTextureName, - GL_TEXTURE_2D, WglNvDxInterop::WGL_ACCESS_READ_ONLY_NV); - if (m_glHandle) { - if (m_wgl.wglDXLockObjectsNV(m_d3dglHandle, 1, &m_glHandle)) - return; + funcs->glGenTextures(1, &handles.textureName); + handles.texture = wgl.wglDXRegisterObjectNV(handles.device, texture, handles.textureName, + GL_TEXTURE_2D, WglNvDxInterop::WGL_ACCESS_READ_ONLY_NV); + if (handles.texture) { + if (wgl.wglDXLockObjectsNV(handles.device, 1, &handles.texture)) { + D3DSURFACE_DESC desc; + texture->GetLevelDesc(0, &desc); + QRhiTexture::Format format; + if (desc.Format == D3DFMT_A8R8G8B8) + format = QRhiTexture::BGRA8; + else if (desc.Format == D3DFMT_A8B8G8R8) + format = QRhiTexture::RGBA8; + else + return {}; + + std::unique_ptr<QRhiTexture> tex(rhi->newTexture(format, QSize{int(desc.Width), int(desc.Height)}, 1, {})); + tex->createFrom({quint64(handles.textureName), 0}); + return std::make_unique<QVideoFrameOpenGlTextures>(std::move(tex), wgl, handles); + } qCDebug(qLcEvrD3DPresentEngine) << "Failed to lock OpenGL texture"; - m_wgl.wglDXUnregisterObjectNV(m_d3dglHandle, m_glHandle); - m_glHandle = nullptr; + wgl.wglDXUnregisterObjectNV(handles.device, handles.texture); } else { qCDebug(qLcEvrD3DPresentEngine) << "Could not register D3D9 texture in OpenGL"; } - funcs->glDeleteTextures(1, &m_glTextureName); - m_glTextureName = 0; + funcs->glDeleteTextures(1, &handles.textureName); } else { qCDebug(qLcEvrD3DPresentEngine) << "Failed generate texture names, OpenGL context functions missing"; } + return {}; } - std::unique_ptr<QRhiTexture> texture(int plane) const override + QRhiTexture *texture(uint plane) const override { - if (!m_rhi || !m_texture || plane > 0) - return {}; + return plane == 0 ? m_tex.get() : nullptr; + }; +private: + std::unique_ptr<QRhiTexture> m_tex; + WglNvDxInterop m_wgl; + InterOpHandles m_handles; +}; - D3DSURFACE_DESC desc; - m_texture->GetLevelDesc(0, &desc); - QRhiTexture::Format format; - if (desc.Format == D3DFMT_A8R8G8B8) - format = QRhiTexture::BGRA8; - else if (desc.Format == D3DFMT_A8B8G8R8) - format = QRhiTexture::RGBA8; - else - return {}; +class OpenGlVideoBuffer: public IMFSampleVideoBuffer +{ +public: + OpenGlVideoBuffer(ComPtr<IDirect3DDevice9Ex> device, const ComPtr<IMFSample> &sample, + const WglNvDxInterop &wglNvDxInterop, HANDLE sharedHandle, QRhi *rhi) + : IMFSampleVideoBuffer(std::move(device), sample, rhi, QVideoFrame::RhiTextureHandle) + , m_sharedHandle(sharedHandle) + , m_wgl(wglNvDxInterop) + {} + + std::unique_ptr<QVideoFrameTextures> mapTextures(QRhi *rhi) override + { + if (!m_texture) { + ComPtr<IMFMediaBuffer> buffer; + HRESULT hr = m_sample->GetBufferByIndex(0, buffer.GetAddressOf()); + if (FAILED(hr)) + return {}; + + ComPtr<IDirect3DSurface9> surface; + hr = MFGetService(buffer.Get(), MR_BUFFER_SERVICE, IID_IDirect3DSurface9, + (void **)(surface.GetAddressOf())); + if (FAILED(hr)) + return {}; - std::unique_ptr<QRhiTexture> tex(m_rhi->newTexture(format, QSize{int(desc.Width), int(desc.Height)}, 1, {})); - tex->createFrom({quint64(m_glTextureName), 0}); - return tex; + hr = surface->GetContainer(IID_IDirect3DTexture9, (void **)m_texture.GetAddressOf()); + if (FAILED(hr)) + return {}; + } + + return QVideoFrameOpenGlTextures::create(m_wgl, rhi, m_device.Get(), m_texture.Get(), m_sharedHandle); } private: - GLuint m_glTextureName = 0; - HANDLE m_d3dglHandle = nullptr; - HANDLE m_glHandle = nullptr; HANDLE m_sharedHandle = nullptr; WglNvDxInterop m_wgl; - QWindowsIUPointer<IDirect3DTexture9> m_texture; + ComPtr<IDirect3DTexture9> m_texture; }; #endif @@ -323,9 +341,9 @@ void D3DPresentEngine::setSink(QVideoSink *sink) m_sink = sink; releaseResources(); - m_device.reset(); - m_devices.reset(); - m_D3D9.reset(); + m_device.Reset(); + m_devices.Reset(); + m_D3D9.Reset(); if (!m_sink) return; @@ -343,10 +361,10 @@ void D3DPresentEngine::setSink(QVideoSink *sink) HRESULT D3DPresentEngine::initializeD3D() { - HRESULT hr = Direct3DCreate9Ex(D3D_SDK_VERSION, m_D3D9.address()); + HRESULT hr = Direct3DCreate9Ex(D3D_SDK_VERSION, m_D3D9.GetAddressOf()); if (SUCCEEDED(hr)) - hr = DXVA2CreateDirect3DDeviceManager9(&m_deviceResetToken, m_devices.address()); + hr = DXVA2CreateDirect3DDeviceManager9(&m_deviceResetToken, m_devices.GetAddressOf()); return hr; } @@ -391,8 +409,16 @@ static bool readWglNvDxInteropProc(WglNvDxInterop &f) return false; } - auto dc = ::GetDC(::GetShellWindow()); - if (!strstr(wglGetExtensionsStringARB(dc), "WGL_NV_DX_interop")) { + HWND hwnd = ::GetShellWindow(); + auto dc = ::GetDC(hwnd); + + const char *wglExtString = wglGetExtensionsStringARB(dc); + if (!wglExtString) + qCDebug(qLcEvrD3DPresentEngine) << "WGL extensions missing (wglGetExtensionsStringARB returned null)"; + + bool hasExtension = wglExtString && strstr(wglExtString, "WGL_NV_DX_interop"); + ReleaseDC(hwnd, dc); + if (!hasExtension) { qCDebug(qLcEvrD3DPresentEngine) << "WGL_NV_DX_interop missing"; return false; } @@ -407,6 +433,20 @@ static bool readWglNvDxInteropProc(WglNvDxInterop &f) } #endif +namespace { + +bool hwTextureRenderingEnabled() { + // add possibility for an user to opt-out HW video rendering + // using the same env. variable as for FFmpeg backend + static bool isDisableConversionSet = false; + static const int disableHwConversion = qEnvironmentVariableIntValue( + "QT_DISABLE_HW_TEXTURES_CONVERSION", &isDisableConversionSet); + + return !isDisableConversionSet || !disableHwConversion; +} + +} + HRESULT D3DPresentEngine::createD3DDevice() { if (!m_D3D9 || !m_devices) @@ -414,23 +454,26 @@ HRESULT D3DPresentEngine::createD3DDevice() m_useTextureRendering = false; UINT adapterID = 0; - QRhi *rhi = m_sink ? m_sink->rhi() : nullptr; - if (rhi) { - if (rhi->backend() == QRhi::D3D11) { - m_useTextureRendering = findD3D11AdapterID(*rhi, m_D3D9.get(), adapterID); + + if (hwTextureRenderingEnabled()) { + QRhi *rhi = m_sink ? m_sink->rhi() : nullptr; + if (rhi) { + if (rhi->backend() == QRhi::D3D11) { + m_useTextureRendering = findD3D11AdapterID(*rhi, m_D3D9.Get(), adapterID); #if QT_CONFIG(opengl) - } else if (rhi->backend() == QRhi::OpenGLES2) { - m_useTextureRendering = readWglNvDxInteropProc(m_wglNvDxInterop); + } else if (rhi->backend() == QRhi::OpenGLES2) { + m_useTextureRendering = readWglNvDxInteropProc(m_wglNvDxInterop); #endif + } else { + qCDebug(qLcEvrD3DPresentEngine) << "Not supported RHI backend type"; + } } else { - qCDebug(qLcEvrD3DPresentEngine) << "Not supported RHI backend type"; + qCDebug(qLcEvrD3DPresentEngine) << "No RHI associated with this sink"; } - } else { - qCDebug(qLcEvrD3DPresentEngine) << "No RHI associated with this sink"; - } - if (!m_useTextureRendering) - qCDebug(qLcEvrD3DPresentEngine) << "Could not find compatible RHI adapter, zero copy disabled"; + if (!m_useTextureRendering) + qCDebug(qLcEvrD3DPresentEngine) << "Could not find compatible RHI adapter, zero copy disabled"; + } D3DCAPS9 ddCaps; ZeroMemory(&ddCaps, sizeof(ddCaps)); @@ -458,7 +501,7 @@ HRESULT D3DPresentEngine::createD3DDevice() pp.Flags = D3DPRESENTFLAG_VIDEO; pp.PresentationInterval = D3DPRESENT_INTERVAL_DEFAULT; - QWindowsIUPointer<IDirect3DDevice9Ex> device; + ComPtr<IDirect3DDevice9Ex> device; hr = m_D3D9->CreateDeviceEx( adapterID, @@ -467,7 +510,7 @@ HRESULT D3DPresentEngine::createD3DDevice() vp | D3DCREATE_NOWINDOWCHANGES | D3DCREATE_MULTITHREADED | D3DCREATE_FPU_PRESERVE, &pp, NULL, - device.address() + device.GetAddressOf() ); if (FAILED(hr)) return hr; @@ -476,7 +519,7 @@ HRESULT D3DPresentEngine::createD3DDevice() if (FAILED(hr)) return hr; - hr = m_devices->ResetDevice(device.get(), m_deviceResetToken); + hr = m_devices->ResetDevice(device.Get(), m_deviceResetToken); if (FAILED(hr)) return hr; @@ -486,7 +529,7 @@ HRESULT D3DPresentEngine::createD3DDevice() bool D3DPresentEngine::isValid() const { - return m_device.get() != nullptr; + return m_device.Get() != nullptr; } void D3DPresentEngine::releaseResources() @@ -502,7 +545,7 @@ HRESULT D3DPresentEngine::getService(REFGUID, REFIID riid, void** ppv) if (!m_devices) { hr = MF_E_UNSUPPORTED_SERVICE; } else { - *ppv = m_devices.get(); + *ppv = m_devices.Get(); m_devices->AddRef(); } } else { @@ -548,7 +591,9 @@ HRESULT D3DPresentEngine::checkFormat(D3DFORMAT format) return ok ? S_OK : D3DERR_NOTAVAILABLE; } -HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, QList<IMFSample*> &videoSampleQueue) +HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, + QList<ComPtr<IMFSample>> &videoSampleQueue, + QSize frameSize) { if (!format || !m_device) return MF_E_UNEXPECTED; @@ -561,6 +606,11 @@ HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, QList<IMFSamp if (FAILED(hr)) return hr; + if (frameSize.isValid() && !frameSize.isEmpty()) { + width = frameSize.width(); + height = frameSize.height(); + } + DWORD d3dFormat = 0; hr = qt_evr_getFourCC(format, &d3dFormat); if (FAILED(hr)) @@ -575,24 +625,24 @@ HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, QList<IMFSamp for (int i = 0; i < PRESENTER_BUFFER_COUNT; i++) { // texture ref cnt is increased by GetSurfaceLevel()/MFCreateVideoSampleFromSurface() // below, so it will be destroyed only when the sample pool is released. - QWindowsIUPointer<IDirect3DTexture9> texture; + ComPtr<IDirect3DTexture9> texture; HANDLE sharedHandle = nullptr; - hr = m_device->CreateTexture(width, height, 1, D3DUSAGE_RENDERTARGET, (D3DFORMAT)d3dFormat, D3DPOOL_DEFAULT, texture.address(), &sharedHandle); + hr = m_device->CreateTexture(width, height, 1, D3DUSAGE_RENDERTARGET, (D3DFORMAT)d3dFormat, D3DPOOL_DEFAULT, texture.GetAddressOf(), &sharedHandle); if (FAILED(hr)) break; - QWindowsIUPointer<IDirect3DSurface9> surface; - hr = texture->GetSurfaceLevel(0, surface.address()); + ComPtr<IDirect3DSurface9> surface; + hr = texture->GetSurfaceLevel(0, surface.GetAddressOf()); if (FAILED(hr)) break; - QWindowsIUPointer<IMFSample> videoSample; - hr = MFCreateVideoSampleFromSurface(surface.get(), videoSample.address()); + ComPtr<IMFSample> videoSample; + hr = MFCreateVideoSampleFromSurface(surface.Get(), videoSample.GetAddressOf()); if (FAILED(hr)) break; - m_sampleTextureHandle[i] = {videoSample.get(), sharedHandle}; - videoSampleQueue.append(videoSample.release()); + m_sampleTextureHandle[i] = {videoSample.Get(), sharedHandle}; + videoSampleQueue.append(videoSample); } if (SUCCEEDED(hr)) { @@ -604,43 +654,33 @@ HRESULT D3DPresentEngine::createVideoSamples(IMFMediaType *format, QList<IMFSamp return hr; } -QVideoFrame D3DPresentEngine::makeVideoFrame(IMFSample *sample) +QVideoFrame D3DPresentEngine::makeVideoFrame(const ComPtr<IMFSample> &sample) { if (!sample) return {}; HANDLE sharedHandle = nullptr; for (const auto &p : m_sampleTextureHandle) - if (p.first == sample) + if (p.first == sample.Get()) sharedHandle = p.second; - QAbstractVideoBuffer *vb = nullptr; + std::unique_ptr<IMFSampleVideoBuffer> vb; QRhi *rhi = m_sink ? m_sink->rhi() : nullptr; if (m_useTextureRendering && sharedHandle && rhi) { if (rhi->backend() == QRhi::D3D11) { - auto nh = static_cast<const QRhiD3D11NativeHandles*>(rhi->nativeHandles()); - if (nh) { - auto dev = reinterpret_cast<ID3D11Device *>(nh->dev); - if (dev) { - QWindowsIUPointer<ID3D11Texture2D> d3d11tex; - HRESULT hr = dev->OpenSharedResource(sharedHandle, __uuidof(ID3D11Texture2D), (void**)(d3d11tex.address())); - if (SUCCEEDED(hr)) - vb = new D3D11TextureVideoBuffer(m_device, sample, d3d11tex, rhi); - else - qCDebug(qLcEvrD3DPresentEngine) << "Failed to obtain D3D11Texture2D from D3D9Texture2D handle"; - } - } + vb = std::make_unique<D3D11TextureVideoBuffer>(m_device, sample, sharedHandle, rhi); #if QT_CONFIG(opengl) } else if (rhi->backend() == QRhi::OpenGLES2) { - vb = new OpenGlVideoBuffer(m_device, sample, m_wglNvDxInterop, sharedHandle, rhi); + vb = std::make_unique<OpenGlVideoBuffer>(m_device, sample, m_wglNvDxInterop, + sharedHandle, rhi); #endif } } if (!vb) - vb = new IMFSampleVideoBuffer(m_device, sample, rhi); + vb = std::make_unique<IMFSampleVideoBuffer>(m_device, sample, rhi); - QVideoFrame frame(vb, m_surfaceFormat); + QVideoFrame frame = QVideoFramePrivate::createFrame(std::move(vb), m_surfaceFormat); // WMF uses 100-nanosecond units, Qt uses microseconds LONGLONG startTime = 0; diff --git a/src/plugins/multimedia/windows/evr/evrd3dpresentengine_p.h b/src/plugins/multimedia/windows/evr/evrd3dpresentengine_p.h index f68ae50f1..93aa90b71 100644 --- a/src/plugins/multimedia/windows/evr/evrd3dpresentengine_p.h +++ b/src/plugins/multimedia/windows/evr/evrd3dpresentengine_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef EVRD3DPRESENTENGINE_H #define EVRD3DPRESENTENGINE_H @@ -52,8 +16,10 @@ // #include <QMutex> +#include <QSize> #include <QVideoFrameFormat> -#include <private/qwindowsiupointer_p.h> +#include <private/qcomptr_p.h> +#include <qpointer.h> #include <d3d9.h> @@ -141,9 +107,10 @@ public: HRESULT checkFormat(D3DFORMAT format); UINT refreshRate() const { return m_displayMode.RefreshRate; } - HRESULT createVideoSamples(IMFMediaType *format, QList<IMFSample*>& videoSampleQueue); + HRESULT createVideoSamples(IMFMediaType *format, QList<ComPtr<IMFSample>> &videoSampleQueue, + QSize frameSize); QVideoFrameFormat videoSurfaceFormat() const { return m_surfaceFormat; } - QVideoFrame makeVideoFrame(IMFSample* sample); + QVideoFrame makeVideoFrame(const ComPtr<IMFSample> &sample); void releaseResources(); void setSink(QVideoSink *sink); @@ -159,13 +126,13 @@ private: UINT m_deviceResetToken; D3DDISPLAYMODE m_displayMode; - QWindowsIUPointer<IDirect3D9Ex> m_D3D9; - QWindowsIUPointer<IDirect3DDevice9Ex> m_device; - QWindowsIUPointer<IDirect3DDeviceManager9> m_devices; + ComPtr<IDirect3D9Ex> m_D3D9; + ComPtr<IDirect3DDevice9Ex> m_device; + ComPtr<IDirect3DDeviceManager9> m_devices; QVideoFrameFormat m_surfaceFormat; - class QVideoSink *m_sink = nullptr; + QPointer<QVideoSink> m_sink; bool m_useTextureRendering = false; #if QT_CONFIG(opengl) WglNvDxInterop m_wglNvDxInterop; diff --git a/src/plugins/multimedia/windows/evr/evrhelpers.cpp b/src/plugins/multimedia/windows/evr/evrhelpers.cpp index 53e68d619..bf4347c69 100644 --- a/src/plugins/multimedia/windows/evr/evrhelpers.cpp +++ b/src/plugins/multimedia/windows/evr/evrhelpers.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "evrhelpers_p.h" diff --git a/src/plugins/multimedia/windows/evr/evrhelpers_p.h b/src/plugins/multimedia/windows/evr/evrhelpers_p.h index d7d6fb298..30779c835 100644 --- a/src/plugins/multimedia/windows/evr/evrhelpers_p.h +++ b/src/plugins/multimedia/windows/evr/evrhelpers_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef EVRHELPERS_H #define EVRHELPERS_H @@ -59,6 +23,7 @@ #include <mfidl.h> #include <mfapi.h> #include <mferror.h> +#include <private/quniquehandle_p.h> QT_BEGIN_NAMESPACE @@ -112,6 +77,16 @@ inline HRESULT qt_evr_getFrameRate(IMFMediaType *pType, MFRatio *pRatio) QVideoFrameFormat::PixelFormat qt_evr_pixelFormatFromD3DFormat(DWORD format); D3DFORMAT qt_evr_D3DFormatFromPixelFormat(QVideoFrameFormat::PixelFormat format); +struct NullHandleTraits +{ + using Type = HANDLE; + static Type invalidValue() { return nullptr; } + static bool close(Type handle) { return CloseHandle(handle) != 0; } +}; + +using EventHandle = QUniqueHandle<NullHandleTraits>; +using ThreadHandle = QUniqueHandle<NullHandleTraits>; + QT_END_NAMESPACE #endif // EVRHELPERS_H diff --git a/src/plugins/multimedia/windows/evr/evrvideowindowcontrol.cpp b/src/plugins/multimedia/windows/evr/evrvideowindowcontrol.cpp index 4ea9fd716..854c9ddb2 100644 --- a/src/plugins/multimedia/windows/evr/evrvideowindowcontrol.cpp +++ b/src/plugins/multimedia/windows/evr/evrvideowindowcontrol.cpp @@ -1,44 +1,10 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "evrvideowindowcontrol_p.h" +QT_BEGIN_NAMESPACE + EvrVideoWindowControl::EvrVideoWindowControl(QVideoSink *parent) : QPlatformVideoSink(parent) , m_windowId(0) @@ -256,3 +222,7 @@ DXVA2_Fixed32 EvrVideoWindowControl::scaleProcAmpValue(DWORD prop, float value) return DXVA2FloatToFixed(scaledValue); } + +QT_END_NAMESPACE + +#include "moc_evrvideowindowcontrol_p.cpp" diff --git a/src/plugins/multimedia/windows/evr/evrvideowindowcontrol_p.h b/src/plugins/multimedia/windows/evr/evrvideowindowcontrol_p.h index dc29e1af7..c4875d28d 100644 --- a/src/plugins/multimedia/windows/evr/evrvideowindowcontrol_p.h +++ b/src/plugins/multimedia/windows/evr/evrvideowindowcontrol_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef EVRVIDEOWINDOWCONTROL_H #define EVRVIDEOWINDOWCONTROL_H diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowscamera.cpp b/src/plugins/multimedia/windows/mediacapture/qwindowscamera.cpp index 53e8650f7..d5e25e1c5 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowscamera.cpp +++ b/src/plugins/multimedia/windows/mediacapture/qwindowscamera.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowscamera_p.h" @@ -133,3 +97,5 @@ void QWindowsCamera::onActiveChanged(bool active) } QT_END_NAMESPACE + +#include "moc_qwindowscamera_p.cpp" diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowscamera_p.h b/src/plugins/multimedia/windows/mediacapture/qwindowscamera_p.h index 379a2cba3..2aec11165 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowscamera_p.h +++ b/src/plugins/multimedia/windows/mediacapture/qwindowscamera_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSCAMERA_H #define QWINDOWSCAMERA_H diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture.cpp b/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture.cpp index e2a1952db..ea66d561a 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture.cpp +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsimagecapture_p.h" @@ -117,11 +81,11 @@ void QWindowsImageCapture::setCaptureSession(QPlatformMediaCaptureSession *sessi if (isReadyForCapture() != readyForCapture) emit readyForCaptureChanged(isReadyForCapture()); - connect(m_mediaDeviceSession, SIGNAL(readyForCaptureChanged(bool)), - this, SIGNAL(readyForCaptureChanged(bool))); + connect(m_mediaDeviceSession, &QWindowsMediaDeviceSession::readyForCaptureChanged, + this, &QWindowsImageCapture::readyForCaptureChanged); - connect(m_mediaDeviceSession, SIGNAL(videoFrameChanged(QVideoFrame)), - this, SLOT(handleVideoFrameChanged(QVideoFrame))); + connect(m_mediaDeviceSession, &QWindowsMediaDeviceSession::videoFrameChanged, + this, &QWindowsImageCapture::handleVideoFrameChanged); } void QWindowsImageCapture::handleVideoFrameChanged(const QVideoFrame &frame) @@ -130,13 +94,23 @@ void QWindowsImageCapture::handleVideoFrameChanged(const QVideoFrame &frame) QImage image = frame.toImage(); + QSize size = m_settings.resolution(); + if (size.isValid() && image.size() != size) { + image = image.scaled(size, Qt::KeepAspectRatioByExpanding); + if (image.size() != size) { + int xoff = (image.size().width() - size.width()) / 2; + int yoff = (image.size().height() - size.height()) / 2; + image = image.copy(xoff, yoff, size.width(), size.height()); + } + } + emit imageExposed(m_captureId); emit imageAvailable(m_captureId, frame); emit imageCaptured(m_captureId, image); QMediaMetaData metaData = this->metaData(); metaData.insert(QMediaMetaData::Date, QDateTime::currentDateTime()); - metaData.insert(QMediaMetaData::Resolution, frame.size()); + metaData.insert(QMediaMetaData::Resolution, size); emit imageMetadataAvailable(m_captureId, metaData); @@ -229,3 +203,5 @@ int QWindowsImageCapture::writerQuality(const QString &writerFormat, } QT_END_NAMESPACE + +#include "moc_qwindowsimagecapture_p.cpp" diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture_p.h b/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture_p.h index 6c3bc8107..746732e73 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture_p.h +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsimagecapture_p.h @@ -1,44 +1,8 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only -#ifndef QWindowsImageCapture_H -#define QWindowsImageCapture_H +#ifndef QWINDOWSIMAGECAPTURE_H +#define QWINDOWSIMAGECAPTURE_H // // W A R N I N G @@ -97,4 +61,4 @@ private: QT_END_NAMESPACE -#endif // QWindowsImageCapture_H +#endif // QWINDOWSIMAGECAPTURE_H diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture.cpp b/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture.cpp index 7f6a061c9..d349b2c43 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture.cpp +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsmediacapture_p.h" @@ -141,3 +105,5 @@ QWindowsMediaDeviceSession *QWindowsMediaCaptureService::session() const } QT_END_NAMESPACE + +#include "moc_qwindowsmediacapture_p.cpp" diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture_p.h b/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture_p.h index 30c3c1abd..579310afd 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture_p.h +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediacapture_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSMEDIACAPTURE_H #define QWINDOWSMEDIACAPTURE_H diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader.cpp b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader.cpp index cbb070595..e99b95ad2 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader.cpp +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsmediadevicereader_p.h" @@ -44,8 +8,9 @@ #include <qmediadevices.h> #include <qaudiodevice.h> #include <private/qmemoryvideobuffer_p.h> +#include <private/qvideoframe_p.h> #include <private/qwindowsmfdefs_p.h> -#include <private/qwindowsiupointer_p.h> +#include <private/qcomptr_p.h> #include <QtCore/qdebug.h> #include <mmdeviceapi.h> @@ -58,7 +23,7 @@ QWindowsMediaDeviceReader::QWindowsMediaDeviceReader(QObject *parent) : QObject(parent) { m_durationTimer.setInterval(100); - connect(&m_durationTimer, SIGNAL(timeout()), this, SLOT(updateDuration())); + connect(&m_durationTimer, &QTimer::timeout, this, &QWindowsMediaDeviceReader::updateDuration); } QWindowsMediaDeviceReader::~QWindowsMediaDeviceReader() @@ -246,7 +211,7 @@ HRESULT QWindowsMediaDeviceReader::prepareVideoStream(DWORD mediaTypeIndex) // and the stride, which we need to convert the frame later hr = MFGetStrideForBitmapInfoHeader(subtype.Data1, m_frameWidth, &m_stride); if (SUCCEEDED(hr)) { - + m_stride = qAbs(m_stride); UINT32 frameRateNum, frameRateDen; hr = MFGetAttributeRatio(m_videoMediaType, MF_MT_FRAME_RATE, &frameRateNum, &frameRateDen); if (SUCCEEDED(hr)) { @@ -674,9 +639,9 @@ QMediaRecorder::Error QWindowsMediaDeviceReader::startRecording( if (!m_active || m_recording || (videoFormat == GUID_NULL && audioFormat == GUID_NULL)) return QMediaRecorder::ResourceError; - QWindowsIUPointer<IMFAttributes> writerAttributes; + ComPtr<IMFAttributes> writerAttributes; - HRESULT hr = MFCreateAttributes(writerAttributes.address(), 2); + HRESULT hr = MFCreateAttributes(writerAttributes.GetAddressOf(), 2); if (FAILED(hr)) return QMediaRecorder::ResourceError; @@ -690,9 +655,9 @@ QMediaRecorder::Error QWindowsMediaDeviceReader::startRecording( if (FAILED(hr)) return QMediaRecorder::ResourceError; - QWindowsIUPointer<IMFSinkWriter> sinkWriter; + ComPtr<IMFSinkWriter> sinkWriter; hr = MFCreateSinkWriterFromURL(reinterpret_cast<LPCWSTR>(fileName.utf16()), - nullptr, writerAttributes.get(), sinkWriter.address()); + nullptr, writerAttributes.Get(), sinkWriter.GetAddressOf()); if (FAILED(hr)) return QMediaRecorder::LocationNotWritable; @@ -738,7 +703,7 @@ QMediaRecorder::Error QWindowsMediaDeviceReader::startRecording( if (FAILED(hr)) return QMediaRecorder::ResourceError; - m_sinkWriter = sinkWriter.release(); + m_sinkWriter = sinkWriter.Detach(); m_lastDuration = -1; m_currentDuration = 0; updateDuration(); @@ -991,9 +956,11 @@ STDMETHODIMP QWindowsMediaDeviceReader::OnReadSample(HRESULT hrStatus, DWORD dwS if (SUCCEEDED(mediaBuffer->Lock(&buffer, nullptr, &bufLen))) { auto bytes = QByteArray(reinterpret_cast<char*>(buffer), bufLen); + QVideoFrameFormat format(QSize(m_frameWidth, m_frameHeight), m_pixelFormat); - QVideoFrame frame(new QMemoryVideoBuffer(bytes, m_stride), - QVideoFrameFormat(QSize(m_frameWidth, m_frameHeight), m_pixelFormat)); + QVideoFrame frame = QVideoFramePrivate::createFrame( + std::make_unique<QMemoryVideoBuffer>(std::move(bytes), m_stride), + std::move(format)); // WMF uses 100-nanosecond units, Qt uses microseconds frame.setStartTime(llTimestamp * 0.1); @@ -1048,3 +1015,5 @@ STDMETHODIMP QWindowsMediaDeviceReader::OnMarker(DWORD, LPVOID) } QT_END_NAMESPACE + +#include "moc_qwindowsmediadevicereader_p.cpp" diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader_p.h b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader_p.h index 0b78be843..4699a463a 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader_p.h +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicereader_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSMEDIADEVICEREADER_H #define QWINDOWSMEDIADEVICEREADER_H @@ -53,8 +17,8 @@ #include <mfapi.h> #include <mfidl.h> -#include <Mferror.h> -#include <Mfreadwrite.h> +#include <mferror.h> +#include <mfreadwrite.h> #include <QtCore/qobject.h> #include <QtCore/qmutex.h> diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession.cpp b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession.cpp index d47188aeb..b13599444 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession.cpp +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsmediadevicesession_p.h" @@ -53,14 +17,22 @@ QWindowsMediaDeviceSession::QWindowsMediaDeviceSession(QObject *parent) : QObject(parent) { m_mediaDeviceReader = new QWindowsMediaDeviceReader(this); - connect(m_mediaDeviceReader, SIGNAL(streamingStarted()), this, SLOT(handleStreamingStarted())); - connect(m_mediaDeviceReader, SIGNAL(streamingStopped()), this, SLOT(handleStreamingStopped())); - connect(m_mediaDeviceReader, SIGNAL(streamingError(int)), this, SLOT(handleStreamingError(int))); - connect(m_mediaDeviceReader, SIGNAL(videoFrameChanged(QVideoFrame)), this, SLOT(handleVideoFrameChanged(QVideoFrame))); - connect(m_mediaDeviceReader, SIGNAL(recordingStarted()), this, SIGNAL(recordingStarted())); - connect(m_mediaDeviceReader, SIGNAL(recordingStopped()), this, SIGNAL(recordingStopped())); - connect(m_mediaDeviceReader, SIGNAL(recordingError(int)), this, SIGNAL(recordingError(int))); - connect(m_mediaDeviceReader, SIGNAL(durationChanged(qint64)), this, SIGNAL(durationChanged(qint64))); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::streamingStarted, + this, &QWindowsMediaDeviceSession::handleStreamingStarted); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::streamingStopped, + this, &QWindowsMediaDeviceSession::handleStreamingStopped); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::streamingError, + this, &QWindowsMediaDeviceSession::handleStreamingError); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::videoFrameChanged, + this, &QWindowsMediaDeviceSession::handleVideoFrameChanged); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::recordingStarted, + this, &QWindowsMediaDeviceSession::recordingStarted); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::recordingStopped, + this, &QWindowsMediaDeviceSession::recordingStopped); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::recordingError, + this, &QWindowsMediaDeviceSession::recordingError); + connect(m_mediaDeviceReader, &QWindowsMediaDeviceReader::durationChanged, + this, &QWindowsMediaDeviceSession::durationChanged); } QWindowsMediaDeviceSession::~QWindowsMediaDeviceSession() @@ -137,10 +109,12 @@ void QWindowsMediaDeviceSession::setVideoSink(QVideoSink *surface) void QWindowsMediaDeviceSession::handleStreamingStarted() { - m_active = true; - m_activating = false; - emit activeChanged(m_active); - emit readyForCaptureChanged(m_active); + if (m_activating) { + m_active = true; + m_activating = false; + emit activeChanged(m_active); + emit readyForCaptureChanged(m_active); + } } void QWindowsMediaDeviceSession::handleStreamingStopped() @@ -153,14 +127,14 @@ void QWindowsMediaDeviceSession::handleStreamingStopped() void QWindowsMediaDeviceSession::handleStreamingError(int errorCode) { if (m_surface) - emit m_surface->platformVideoSink()->setVideoFrame(QVideoFrame()); + m_surface->platformVideoSink()->setVideoFrame(QVideoFrame()); emit streamingError(errorCode); } void QWindowsMediaDeviceSession::handleVideoFrameChanged(const QVideoFrame &frame) { if (m_surface) - emit m_surface->platformVideoSink()->setVideoFrame(frame); + m_surface->platformVideoSink()->setVideoFrame(frame); emit videoFrameChanged(frame); } @@ -398,3 +372,5 @@ quint32 QWindowsMediaDeviceSession::estimateAudioBitRate(const GUID &audioFormat } QT_END_NAMESPACE + +#include "moc_qwindowsmediadevicesession_p.cpp" diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession_p.h b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession_p.h index 232167e77..c3998ce6c 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession_p.h +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediadevicesession_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSMEDIADEVICESESSION_H #define QWINDOWSMEDIADEVICESESSION_H diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder.cpp b/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder.cpp index 97d80c8bf..512110af6 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder.cpp +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsmediaencoder_p.h" @@ -44,7 +8,7 @@ #include "mfmetadata_p.h" #include <QtCore/QUrl> #include <QtCore/QMimeType> -#include <Mferror.h> +#include <mferror.h> #include <shobjidl.h> #include <private/qmediastoragelocation_p.h> #include <private/qmediarecorder_p.h> @@ -81,15 +45,15 @@ void QWindowsMediaEncoder::record(QMediaEncoderSettings &settings) if (m_state != QMediaRecorder::StoppedState) return; - m_sessionWasActive = m_mediaDeviceSession->isActive(); + m_sessionWasActive = m_mediaDeviceSession->isActive() || m_mediaDeviceSession->isActivating(); if (!m_sessionWasActive) { m_mediaDeviceSession->setActive(true); if (!m_mediaDeviceSession->isActivating()) { - error(QMediaRecorder::ResourceError, - QMediaRecorderPrivate::msgFailedStartRecording()); + updateError(QMediaRecorder::ResourceError, + QMediaRecorderPrivate::msgFailedStartRecording()); return; } } @@ -108,7 +72,7 @@ void QWindowsMediaEncoder::record(QMediaEncoderSettings &settings) stateChanged(m_state); } else { - error(ec, QMediaRecorderPrivate::msgFailedStartRecording()); + updateError(ec, QMediaRecorderPrivate::msgFailedStartRecording()); } } @@ -121,7 +85,7 @@ void QWindowsMediaEncoder::pause() m_state = QMediaRecorder::PausedState; stateChanged(m_state); } else { - error(QMediaRecorder::FormatError, tr("Failed to pause recording")); + updateError(QMediaRecorder::FormatError, tr("Failed to pause recording")); } } @@ -134,16 +98,17 @@ void QWindowsMediaEncoder::resume() m_state = QMediaRecorder::RecordingState; stateChanged(m_state); } else { - error(QMediaRecorder::FormatError, tr("Failed to resume recording")); + updateError(QMediaRecorder::FormatError, tr("Failed to resume recording")); } } void QWindowsMediaEncoder::stop() { - if (m_mediaDeviceSession && m_state != QMediaRecorder::StoppedState) + if (m_mediaDeviceSession && m_state != QMediaRecorder::StoppedState) { m_mediaDeviceSession->stopRecording(); - if (!m_sessionWasActive) - m_mediaDeviceSession->setActive(false); + if (!m_sessionWasActive) + m_mediaDeviceSession->setActive(false); + } } @@ -213,11 +178,11 @@ void QWindowsMediaEncoder::onDurationChanged(qint64 duration) void QWindowsMediaEncoder::onStreamingError(int errorCode) { if (errorCode == MF_E_VIDEO_RECORDING_DEVICE_INVALIDATED) - error(QMediaRecorder::ResourceError, tr("Camera is no longer present")); + updateError(QMediaRecorder::ResourceError, tr("Camera is no longer present")); else if (errorCode == MF_E_AUDIO_RECORDING_DEVICE_INVALIDATED) - error(QMediaRecorder::ResourceError, tr("Audio input is no longer present")); + updateError(QMediaRecorder::ResourceError, tr("Audio input is no longer present")); else - error(QMediaRecorder::ResourceError, tr("Streaming error")); + updateError(QMediaRecorder::ResourceError, tr("Streaming error")); if (m_state != QMediaRecorder::StoppedState) { m_mediaDeviceSession->stopRecording(); @@ -229,7 +194,7 @@ void QWindowsMediaEncoder::onStreamingError(int errorCode) void QWindowsMediaEncoder::onRecordingError(int errorCode) { Q_UNUSED(errorCode); - error(QMediaRecorder::ResourceError, tr("Recording error")); + updateError(QMediaRecorder::ResourceError, tr("Recording error")); auto lastState = m_state; m_state = QMediaRecorder::StoppedState; @@ -256,3 +221,5 @@ void QWindowsMediaEncoder::onRecordingStopped() } QT_END_NAMESPACE + +#include "moc_qwindowsmediaencoder_p.cpp" diff --git a/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder_p.h b/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder_p.h index 22295791a..51f35ce9d 100644 --- a/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder_p.h +++ b/src/plugins/multimedia/windows/mediacapture/qwindowsmediaencoder_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only // // W A R N I N G diff --git a/src/plugins/multimedia/windows/mfstream.cpp b/src/plugins/multimedia/windows/mfstream.cpp index c49971025..fb37ce293 100644 --- a/src/plugins/multimedia/windows/mfstream.cpp +++ b/src/plugins/multimedia/windows/mfstream.cpp @@ -1,45 +1,10 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "mfstream_p.h" #include <QtCore/qcoreapplication.h> +QT_BEGIN_NAMESPACE //MFStream is added for supporting QIODevice type of media source. //It is used to delegate invocations from media foundation(through IMFByteStream) to QIODevice. @@ -53,7 +18,6 @@ MFStream::MFStream(QIODevice *stream, bool ownStream) //to make sure invocations on stream //are happened in the same thread of stream object this->moveToThread(stream->thread()); - connect(stream, SIGNAL(readyRead()), this, SLOT(handleReadyRead())); } MFStream::~MFStream() @@ -287,12 +251,6 @@ void MFStream::doRead() } } - -void MFStream::handleReadyRead() -{ - doRead(); -} - void MFStream::customEvent(QEvent *event) { if (event->type() != QEvent::User) { @@ -362,3 +320,7 @@ void MFStream::AsyncReadState::setBytesRead(ULONG cbRead) { m_cbRead = cbRead; } + +QT_END_NAMESPACE + +#include "moc_mfstream_p.cpp" diff --git a/src/plugins/multimedia/windows/mfstream_p.h b/src/plugins/multimedia/windows/mfstream_p.h index 3c8d6b296..a5221ed75 100644 --- a/src/plugins/multimedia/windows/mfstream_p.h +++ b/src/plugins/multimedia/windows/mfstream_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFSTREAM_H #define MFSTREAM_H @@ -58,7 +22,7 @@ #include <QtCore/qcoreevent.h> #include <QtCore/qpointer.h> -QT_USE_NAMESPACE +QT_BEGIN_NAMESPACE class MFStream : public QObject, public IMFByteStream { @@ -150,12 +114,11 @@ private: void doRead(); -private Q_SLOTS: - void handleReadyRead(); - protected: void customEvent(QEvent *event) override; IMFAsyncResult *m_currentReadResult; }; +QT_END_NAMESPACE + #endif diff --git a/src/plugins/multimedia/windows/player/mfactivate.cpp b/src/plugins/multimedia/windows/player/mfactivate.cpp index 05d9321be..644c96529 100644 --- a/src/plugins/multimedia/windows/player/mfactivate.cpp +++ b/src/plugins/multimedia/windows/player/mfactivate.cpp @@ -1,49 +1,11 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "mfactivate_p.h" #include <mfapi.h> MFAbstractActivate::MFAbstractActivate() - : m_attributes(0) - , m_cRef(1) { MFCreateAttributes(&m_attributes, 0); } @@ -53,35 +15,3 @@ MFAbstractActivate::~MFAbstractActivate() if (m_attributes) m_attributes->Release(); } - - -HRESULT MFAbstractActivate::QueryInterface(REFIID riid, LPVOID *ppvObject) -{ - if (!ppvObject) - return E_POINTER; - if (riid == IID_IMFActivate) { - *ppvObject = static_cast<IMFActivate*>(this); - } else if (riid == IID_IMFAttributes) { - *ppvObject = static_cast<IMFAttributes*>(this); - } else if (riid == IID_IUnknown) { - *ppvObject = static_cast<IUnknown*>(static_cast<IMFActivate*>(this)); - } else { - *ppvObject = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; -} - -ULONG MFAbstractActivate::AddRef(void) -{ - return InterlockedIncrement(&m_cRef); -} - -ULONG MFAbstractActivate::Release(void) -{ - ULONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) - delete this; - return cRef; -} diff --git a/src/plugins/multimedia/windows/player/mfactivate_p.h b/src/plugins/multimedia/windows/player/mfactivate_p.h index d3d5cf212..efe75474b 100644 --- a/src/plugins/multimedia/windows/player/mfactivate_p.h +++ b/src/plugins/multimedia/windows/player/mfactivate_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFACTIVATE_H #define MFACTIVATE_H @@ -52,17 +16,27 @@ // #include <mfidl.h> +#include <private/qcomobject_p.h> -class MFAbstractActivate : public IMFActivate +QT_BEGIN_NAMESPACE + +namespace QtPrivate { + +template <> +struct QComObjectTraits<IMFActivate> +{ + static constexpr bool isGuidOf(REFIID riid) noexcept + { + return QComObjectTraits<IMFActivate, IMFAttributes>::isGuidOf(riid); + } +}; + +} // namespace QtPrivate + +class MFAbstractActivate : public QComObject<IMFActivate> { public: explicit MFAbstractActivate(); - virtual ~MFAbstractActivate(); - - //from IUnknown - STDMETHODIMP QueryInterface(REFIID riid, LPVOID *ppvObject) override; - STDMETHODIMP_(ULONG) AddRef(void) override; - STDMETHODIMP_(ULONG) Release(void) override; //from IMFAttributes STDMETHODIMP GetItem(REFGUID guidKey, PROPVARIANT *pValue) override @@ -215,9 +189,14 @@ public: return m_attributes->CopyAllItems(pDest); } +protected: + // Destructor is not public. Caller should call Release. + ~MFAbstractActivate() override; + private: - IMFAttributes *m_attributes; - ULONG m_cRef; + IMFAttributes *m_attributes = nullptr; }; +QT_END_NAMESPACE + #endif // MFACTIVATE_H diff --git a/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol.cpp b/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol.cpp index 105424253..109f7964b 100644 --- a/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol.cpp +++ b/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "mfevrvideowindowcontrol_p.h" diff --git a/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol_p.h b/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol_p.h index d1855705c..1ac90e8ce 100644 --- a/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol_p.h +++ b/src/plugins/multimedia/windows/player/mfevrvideowindowcontrol_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFEVRVIDEOWINDOWCONTROL_H #define MFEVRVIDEOWINDOWCONTROL_H diff --git a/src/plugins/multimedia/windows/player/mfplayercontrol.cpp b/src/plugins/multimedia/windows/player/mfplayercontrol.cpp index 67a5caa7b..ae0022773 100644 --- a/src/plugins/multimedia/windows/player/mfplayercontrol.cpp +++ b/src/plugins/multimedia/windows/player/mfplayercontrol.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "mfplayercontrol_p.h" #include "mfplayersession_p.h" @@ -110,7 +74,8 @@ void MFPlayerControl::pause() if (m_state == QMediaPlayer::PausedState) return; - if (m_session->status() == QMediaPlayer::NoMedia) + if (m_session->status() == QMediaPlayer::NoMedia || + m_session->status() == QMediaPlayer::InvalidMedia) return; changeState(QMediaPlayer::PausedState); @@ -158,7 +123,7 @@ void MFPlayerControl::refreshState() #ifdef DEBUG_MEDIAFOUNDATION qDebug() << "MFPlayerControl::emit stateChanged" << m_state; #endif - emit stateChanged(m_state); + stateChanged(m_state); } void MFPlayerControl::handleStatusChanged() @@ -184,7 +149,7 @@ void MFPlayerControl::handleStatusChanged() default: break; } - emit mediaStatusChanged(m_session->status()); + mediaStatusChanged(m_session->status()); refreshState(); } @@ -198,7 +163,7 @@ void MFPlayerControl::handleVideoAvailable() if (m_videoAvailable) return; m_videoAvailable = true; - emit videoAvailableChanged(m_videoAvailable); + videoAvailableChanged(m_videoAvailable); } void MFPlayerControl::handleAudioAvailable() @@ -206,7 +171,7 @@ void MFPlayerControl::handleAudioAvailable() if (m_audioAvailable) return; m_audioAvailable = true; - emit audioAvailableChanged(m_audioAvailable); + audioAvailableChanged(m_audioAvailable); } void MFPlayerControl::resetAudioVideoAvailable() @@ -218,10 +183,10 @@ void MFPlayerControl::resetAudioVideoAvailable() } if (m_audioAvailable) { m_audioAvailable = false; - emit audioAvailableChanged(m_audioAvailable); + audioAvailableChanged(m_audioAvailable); } if (videoDirty) - emit videoAvailableChanged(m_videoAvailable); + videoAvailableChanged(m_videoAvailable); } void MFPlayerControl::handleDurationUpdate(qint64 duration) @@ -229,7 +194,7 @@ void MFPlayerControl::handleDurationUpdate(qint64 duration) if (m_duration == duration) return; m_duration = duration; - emit durationChanged(m_duration); + durationChanged(m_duration); } void MFPlayerControl::handleSeekableUpdate(bool seekable) @@ -237,7 +202,7 @@ void MFPlayerControl::handleSeekableUpdate(bool seekable) if (m_seekable == seekable) return; m_seekable = seekable; - emit seekableChanged(m_seekable); + seekableChanged(m_seekable); } QMediaPlayer::PlaybackState MFPlayerControl::state() const @@ -316,7 +281,7 @@ void MFPlayerControl::handleError(QMediaPlayer::Error errorCode, const QString& { if (isFatal) stop(); - emit error(int(errorCode), errorString); + error(int(errorCode), errorString); } void MFPlayerControl::setActiveTrack(TrackType type, int index) diff --git a/src/plugins/multimedia/windows/player/mfplayercontrol_p.h b/src/plugins/multimedia/windows/player/mfplayercontrol_p.h index 68c24194b..db863afaa 100644 --- a/src/plugins/multimedia/windows/player/mfplayercontrol_p.h +++ b/src/plugins/multimedia/windows/player/mfplayercontrol_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFPLAYERCONTROL_H #define MFPLAYERCONTROL_H @@ -51,12 +15,12 @@ // We mean it. // -#include "QUrl.h" +#include "qurl.h" #include "private/qplatformmediaplayer_p.h" #include <QtCore/qcoreevent.h> -QT_USE_NAMESPACE +QT_BEGIN_NAMESPACE class MFPlayerSession; @@ -134,4 +98,6 @@ private: MFPlayerSession *m_session; }; +QT_END_NAMESPACE + #endif diff --git a/src/plugins/multimedia/windows/player/mfplayersession.cpp b/src/plugins/multimedia/windows/player/mfplayersession.cpp index be99399ed..996ce35d8 100644 --- a/src/plugins/multimedia/windows/player/mfplayersession.cpp +++ b/src/plugins/multimedia/windows/player/mfplayersession.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "private/qplatformmediaplayer_p.h" @@ -51,7 +15,6 @@ #include "qaudiooutput.h" #include "mfplayercontrol_p.h" -#include "mfevrvideowindowcontrol_p.h" #include "mfvideorenderercontrol_p.h" #include <mfmetadata_p.h> #include <private/qwindowsmfdefs_p.h> @@ -60,41 +23,30 @@ #include "mfplayersession_p.h" #include <mferror.h> #include <nserror.h> +#include <winerror.h> #include "sourceresolver_p.h" -#include "samplegrabber_p.h" -#include "mftvideo_p.h" #include <wmcodecdsp.h> #include <mmdeviceapi.h> #include <propvarutil.h> -#include <Functiondiscoverykeys_devpkey.h> +#include <functiondiscoverykeys_devpkey.h> //#define DEBUG_MEDIAFOUNDATION +QT_BEGIN_NAMESPACE MFPlayerSession::MFPlayerSession(MFPlayerControl *playerControl) - : m_cRef(1) - , m_playerControl(playerControl) - , m_session(0) - , m_presentationClock(0) - , m_rateControl(0) - , m_rateSupport(0) - , m_volumeControl(0) - , m_netsourceStatistics(0) - , m_scrubbing(false) - , m_restoreRate(1) - , m_sourceResolver(0) - , m_hCloseEvent(0) - , m_closing(false) - , m_mediaTypes(0) - , m_pendingRate(1) - , m_status(QMediaPlayer::NoMedia) - , m_audioSampleGrabber(0) - , m_audioSampleGrabberNode(0) - , m_videoProbeMFT(0) + : m_cRef(1), + m_playerControl(playerControl), + m_scrubbing(false), + m_restoreRate(1), + m_closing(false), + m_mediaTypes(0), + m_pendingRate(1), + m_status(QMediaPlayer::NoMedia) { - QObject::connect(this, SIGNAL(sessionEvent(IMFMediaEvent*)), this, SLOT(handleSessionEvent(IMFMediaEvent*))); + connect(this, &MFPlayerSession::sessionEvent, this, &MFPlayerSession::handleSessionEvent); m_signalPositionChangeTimer.setInterval(10); m_signalPositionChangeTimer.setTimerType(Qt::PreciseTimer); @@ -110,8 +62,7 @@ MFPlayerSession::MFPlayerSession(MFPlayerControl *playerControl) m_request.prevCmd = CmdNone; m_request.rate = 1.0f; - m_audioSampleGrabber = new AudioSampleGrabberCallback; - m_videoRendererControl = new MFVideoRendererControl; + m_videoRendererControl = new MFVideoRendererControl(this); } void MFPlayerSession::timeout() @@ -153,7 +104,7 @@ void MFPlayerSession::close() m_closing = true; hr = m_session->Close(); if (SUCCEEDED(hr)) { - DWORD dwWaitResult = WaitForSingleObject(m_hCloseEvent, 2000); + DWORD dwWaitResult = WaitForSingleObject(m_hCloseEvent.get(), 2000); if (dwWaitResult == WAIT_TIMEOUT) { qWarning() << "session close time out!"; } @@ -167,35 +118,19 @@ void MFPlayerSession::close() if (m_sourceResolver) m_sourceResolver->shutdown(); } - if (m_sourceResolver) { - m_sourceResolver->Release(); - m_sourceResolver = 0; - } - if (m_videoProbeMFT) { - m_videoProbeMFT->Release(); - m_videoProbeMFT = 0; - } + m_sourceResolver.Reset(); m_videoRendererControl->releaseActivate(); // } else if (m_playerService->videoWindowControl()) { // m_playerService->videoWindowControl()->releaseActivate(); // } - if (m_session) - m_session->Release(); - m_session = 0; - if (m_hCloseEvent) - CloseHandle(m_hCloseEvent); - m_hCloseEvent = 0; + m_session.Reset(); + m_hCloseEvent = {}; m_lastPosition = -1; + m_position = 0; } -MFPlayerSession::~MFPlayerSession() -{ - m_audioSampleGrabber->Release(); -} - - void MFPlayerSession::load(const QUrl &url, QIODevice *stream) { #ifdef DEBUG_MEDIAFOUNDATION @@ -212,11 +147,12 @@ void MFPlayerSession::load(const QUrl &url, QIODevice *stream) } else if (stream && (!stream->isReadable())) { close(); changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Invalid stream source."), true); + error(QMediaPlayer::ResourceError, tr("Invalid stream source."), true); } else if (createSession()) { changeStatus(QMediaPlayer::LoadingMedia); m_sourceResolver->load(url, stream); - m_updateRoutingOnStart = true; + if (url.isLocalFile()) + m_updateRoutingOnStart = true; } positionChanged(position()); } @@ -240,17 +176,28 @@ void MFPlayerSession::handleSourceError(long hr) errorCode = QMediaPlayer::FormatError; errorString = tr("Unsupported media type."); break; + case MF_E_UNSUPPORTED_SCHEME: + errorCode = QMediaPlayer::ResourceError; + errorString = tr("Unsupported URL scheme."); + break; + case QMM_WININET_E_CANNOT_CONNECT: + errorCode = QMediaPlayer::NetworkError; + errorString = tr("Connection to server could not be established."); + break; default: + qWarning() << "handleSourceError:" + << Qt::showbase << Qt::hex << Qt::uppercasedigits << static_cast<quint32>(hr); errorString = tr("Failed to load source."); break; } changeStatus(QMediaPlayer::InvalidMedia); - emit error(errorCode, errorString, true); + error(errorCode, errorString, true); } void MFPlayerSession::handleMediaSourceReady() { - if (QMediaPlayer::LoadingMedia != m_status || !m_sourceResolver || m_sourceResolver != sender()) + if (QMediaPlayer::LoadingMedia != m_status || !m_sourceResolver + || m_sourceResolver.Get() != sender()) return; #ifdef DEBUG_MEDIAFOUNDATION qDebug() << "handleMediaSourceReady"; @@ -260,23 +207,22 @@ void MFPlayerSession::handleMediaSourceReady() DWORD dwCharacteristics = 0; mediaSource->GetCharacteristics(&dwCharacteristics); - emit seekableUpdate(MFMEDIASOURCE_CAN_SEEK & dwCharacteristics); + seekableUpdate(MFMEDIASOURCE_CAN_SEEK & dwCharacteristics); - IMFPresentationDescriptor* sourcePD; + ComPtr<IMFPresentationDescriptor> sourcePD; hr = mediaSource->CreatePresentationDescriptor(&sourcePD); if (SUCCEEDED(hr)) { m_duration = 0; m_metaData = MFMetaData::fromNative(mediaSource); - emit metaDataChanged(); + metaDataChanged(); sourcePD->GetUINT64(MF_PD_DURATION, &m_duration); //convert from 100 nanosecond to milisecond - emit durationUpdate(qint64(m_duration / 10000)); - setupPlaybackTopology(mediaSource, sourcePD); + durationUpdate(qint64(m_duration / 10000)); + setupPlaybackTopology(mediaSource, sourcePD.Get()); tracksChanged(); - sourcePD->Release(); } else { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Cannot create presentation descriptor."), true); + error(QMediaPlayer::ResourceError, tr("Cannot create presentation descriptor."), true); } } @@ -293,7 +239,7 @@ bool MFPlayerSession::getStreamInfo(IMFStreamDescriptor *stream, *name = QString(); *language = QString(); - IMFMediaTypeHandler *typeHandler = nullptr; + ComPtr<IMFMediaTypeHandler> typeHandler; if (SUCCEEDED(stream->GetMediaTypeHandler(&typeHandler))) { @@ -321,13 +267,10 @@ bool MFPlayerSession::getStreamInfo(IMFStreamDescriptor *stream, *type = Video; } - IMFMediaType *mediaType = nullptr; + ComPtr<IMFMediaType> mediaType; if (SUCCEEDED(typeHandler->GetCurrentMediaType(&mediaType))) { mediaType->GetGUID(MF_MT_SUBTYPE, format); - mediaType->Release(); } - - typeHandler->Release(); } return *type != Unknown; @@ -341,15 +284,15 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat hr = sourcePD->GetStreamDescriptorCount(&cSourceStreams); if (FAILED(hr)) { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Failed to get stream count."), true); + error(QMediaPlayer::ResourceError, tr("Failed to get stream count."), true); return; } - IMFTopology *topology; + ComPtr<IMFTopology> topology; hr = MFCreateTopology(&topology); if (FAILED(hr)) { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Failed to create topology."), true); + error(QMediaPlayer::ResourceError, tr("Failed to create topology."), true); return; } @@ -358,7 +301,7 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat for (DWORD i = 0; i < cSourceStreams; i++) { BOOL selected = FALSE; bool streamAdded = false; - IMFStreamDescriptor *streamDesc = NULL; + ComPtr<IMFStreamDescriptor> streamDesc; HRESULT hr = sourcePD->GetStreamDescriptorByIndex(i, &selected, &streamDesc); if (SUCCEEDED(hr)) { @@ -369,7 +312,8 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat QString streamLanguage; GUID format = GUID_NULL; - if (getStreamInfo(streamDesc, &mediaType, &streamName, &streamLanguage, &format)) { + if (getStreamInfo(streamDesc.Get(), &mediaType, &streamName, &streamLanguage, + &format)) { QPlatformMediaPlayer::TrackType trackType = (mediaType == Audio) ? QPlatformMediaPlayer::AudioStream : QPlatformMediaPlayer::VideoStream; @@ -386,23 +330,19 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat m_trackInfo[trackType].format = format; if (((m_mediaTypes & mediaType) == 0) && selected) { // Check if this type isn't already added - IMFTopologyNode *sourceNode = addSourceNode(topology, source, sourcePD, streamDesc); + ComPtr<IMFTopologyNode> sourceNode = + addSourceNode(topology.Get(), source, sourcePD, streamDesc.Get()); if (sourceNode) { - IMFTopologyNode *outputNode = addOutputNode(mediaType, topology, 0); + ComPtr<IMFTopologyNode> outputNode = + addOutputNode(mediaType, topology.Get(), 0); if (outputNode) { - bool connected = false; - if (mediaType == Audio) { - if (!m_audioSampleGrabberNode) - connected = setupAudioSampleGrabber(topology, sourceNode, outputNode); - } sourceNode->GetTopoNodeID(&m_trackInfo[trackType].sourceNodeId); outputNode->GetTopoNodeID(&m_trackInfo[trackType].outputNodeId); - if (!connected) - hr = sourceNode->ConnectOutput(0, outputNode, 0); + hr = sourceNode->ConnectOutput(0, outputNode.Get(), 0); if (FAILED(hr)) { - emit error(QMediaPlayer::FormatError, tr("Unable to play any stream."), false); + error(QMediaPlayer::FormatError, tr("Unable to play any stream."), false); } else { m_trackInfo[trackType].currentIndex = m_trackInfo[trackType].nativeIndexes.count() - 1; streamAdded = true; @@ -410,51 +350,51 @@ void MFPlayerSession::setupPlaybackTopology(IMFMediaSource *source, IMFPresentat m_mediaTypes |= mediaType; switch (mediaType) { case Audio: - emit audioAvailable(); + audioAvailable(); break; case Video: - emit videoAvailable(); + videoAvailable(); break; default: break; } } - outputNode->Release(); + } else { + // remove the source node if the output node cannot be created + topology->RemoveNode(sourceNode.Get()); } - sourceNode->Release(); } } } if (selected && !streamAdded) sourcePD->DeselectStream(i); - - streamDesc->Release(); } } if (succeededCount == 0) { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Unable to play."), true); + error(QMediaPlayer::ResourceError, tr("Unable to play."), true); } else { if (m_trackInfo[QPlatformMediaPlayer::VideoStream].outputNodeId != TOPOID(-1)) topology = insertMFT(topology, m_trackInfo[QPlatformMediaPlayer::VideoStream].outputNodeId); - hr = m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, topology); + hr = m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, topology.Get()); if (SUCCEEDED(hr)) { m_updatingTopology = true; } else { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Failed to set topology."), true); + error(QMediaPlayer::ResourceError, tr("Failed to set topology."), true); } } - topology->Release(); } -IMFTopologyNode* MFPlayerSession::addSourceNode(IMFTopology* topology, IMFMediaSource* source, - IMFPresentationDescriptor* presentationDesc, IMFStreamDescriptor *streamDesc) +ComPtr<IMFTopologyNode> MFPlayerSession::addSourceNode(IMFTopology *topology, + IMFMediaSource *source, + IMFPresentationDescriptor *presentationDesc, + IMFStreamDescriptor *streamDesc) { - IMFTopologyNode *node = NULL; + ComPtr<IMFTopologyNode> node; HRESULT hr = MFCreateTopologyNode(MF_TOPOLOGY_SOURCESTREAM_NODE, &node); if (SUCCEEDED(hr)) { hr = node->SetUnknown(MF_TOPONODE_SOURCE, source); @@ -463,184 +403,82 @@ IMFTopologyNode* MFPlayerSession::addSourceNode(IMFTopology* topology, IMFMediaS if (SUCCEEDED(hr)) { hr = node->SetUnknown(MF_TOPONODE_STREAM_DESCRIPTOR, streamDesc); if (SUCCEEDED(hr)) { - hr = topology->AddNode(node); + hr = topology->AddNode(node.Get()); if (SUCCEEDED(hr)) return node; } } } - node->Release(); } return NULL; } -IMFTopologyNode* MFPlayerSession::addOutputNode(MediaType mediaType, IMFTopology* topology, DWORD sinkID) +ComPtr<IMFTopologyNode> MFPlayerSession::addOutputNode(MediaType mediaType, IMFTopology *topology, + DWORD sinkID) { - IMFTopologyNode *node = NULL; + ComPtr<IMFTopologyNode> node; if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_OUTPUT_NODE, &node))) return NULL; - IMFActivate *activate = NULL; + ComPtr<IMFActivate> activate; if (mediaType == Audio) { if (m_audioOutput) { + auto id = m_audioOutput->device.id(); + if (id.isEmpty()) { + qInfo() << "No audio output"; + return NULL; + } + HRESULT hr = MFCreateAudioRendererActivate(&activate); if (FAILED(hr)) { qWarning() << "Failed to create audio renderer activate"; - node->Release(); return NULL; } - auto id = m_audioOutput->device.id(); - if (!id.isEmpty()) { - QString s = QString::fromUtf8(id); - hr = activate->SetString(MF_AUDIO_RENDERER_ATTRIBUTE_ENDPOINT_ID, (LPCWSTR)s.utf16()); - } else { - //This is the default one that has been inserted in updateEndpoints(), - //so give the activate a hint that we want to use the device for multimedia playback - //then the media foundation will choose an appropriate one. - - //from MSDN: - //The ERole enumeration defines constants that indicate the role that the system has assigned to an audio endpoint device. - //eMultimedia: Music, movies, narration, and live music recording. - hr = activate->SetUINT32(MF_AUDIO_RENDERER_ATTRIBUTE_ENDPOINT_ROLE, eMultimedia); - } - + QString s = QString::fromUtf8(id); + hr = activate->SetString(MF_AUDIO_RENDERER_ATTRIBUTE_ENDPOINT_ID, (LPCWSTR)s.utf16()); if (FAILED(hr)) { - qWarning() << "Failed to set attribute for audio device" << m_audioOutput->device.description(); - activate->Release(); - node->Release(); + qWarning() << "Failed to set attribute for audio device" + << m_audioOutput->device.description(); return NULL; } } } else if (mediaType == Video) { activate = m_videoRendererControl->createActivate(); + + QSize resolution = m_metaData.value(QMediaMetaData::Resolution).toSize(); + + if (resolution.isValid()) + m_videoRendererControl->setCropRect(QRect(QPoint(), resolution)); + } else { // Unknown stream type. - emit error(QMediaPlayer::FormatError, tr("Unknown stream type."), false); + error(QMediaPlayer::FormatError, tr("Unknown stream type."), false); } - if (!activate - || FAILED(node->SetObject(activate)) - || FAILED(node->SetUINT32(MF_TOPONODE_STREAMID, sinkID)) - || FAILED(node->SetUINT32(MF_TOPONODE_NOSHUTDOWN_ON_REMOVE, FALSE)) - || FAILED(topology->AddNode(node))) { - node->Release(); - node = NULL; + if (!activate || FAILED(node->SetObject(activate.Get())) + || FAILED(node->SetUINT32(MF_TOPONODE_STREAMID, sinkID)) + || FAILED(node->SetUINT32(MF_TOPONODE_NOSHUTDOWN_ON_REMOVE, FALSE)) + || FAILED(topology->AddNode(node.Get()))) { + node.Reset(); } if (activate && mediaType == Audio) - activate->Release(); + activate.Reset(); return node; } -bool MFPlayerSession::addAudioSampleGrabberNode(IMFTopology *topology) -{ - HRESULT hr = S_OK; - IMFMediaType *pType = 0; - IMFActivate *sinkActivate = 0; - do { - hr = MFCreateMediaType(&pType); - if (FAILED(hr)) - break; - - hr = pType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); - if (FAILED(hr)) - break; - - hr = pType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); - if (FAILED(hr)) - break; - - hr = MFCreateSampleGrabberSinkActivate(pType, m_audioSampleGrabber, &sinkActivate); - if (FAILED(hr)) - break; - - // Note: Data is distorted if this attribute is enabled - hr = sinkActivate->SetUINT32(MF_SAMPLEGRABBERSINK_IGNORE_CLOCK, FALSE); - if (FAILED(hr)) - break; - - hr = MFCreateTopologyNode(MF_TOPOLOGY_OUTPUT_NODE, &m_audioSampleGrabberNode); - if (FAILED(hr)) - break; - - hr = m_audioSampleGrabberNode->SetObject(sinkActivate); - if (FAILED(hr)) - break; - - hr = m_audioSampleGrabberNode->SetUINT32(MF_TOPONODE_STREAMID, 0); // Identifier of the stream sink. - if (FAILED(hr)) - break; - - hr = m_audioSampleGrabberNode->SetUINT32(MF_TOPONODE_NOSHUTDOWN_ON_REMOVE, FALSE); - if (FAILED(hr)) - break; - - hr = topology->AddNode(m_audioSampleGrabberNode); - if (FAILED(hr)) - break; - - pType->Release(); - sinkActivate->Release(); - return true; - } while (false); - - if (pType) - pType->Release(); - if (sinkActivate) - sinkActivate->Release(); - if (m_audioSampleGrabberNode) { - m_audioSampleGrabberNode->Release(); - m_audioSampleGrabberNode = NULL; - } - return false; -} - -bool MFPlayerSession::setupAudioSampleGrabber(IMFTopology *topology, IMFTopologyNode *sourceNode, IMFTopologyNode *outputNode) -{ - if (!addAudioSampleGrabberNode(topology)) - return false; - - HRESULT hr = S_OK; - IMFTopologyNode *pTeeNode = NULL; - - IMFMediaTypeHandler *typeHandler = NULL; - IMFMediaType *mediaType = NULL; - do { - hr = MFCreateTopologyNode(MF_TOPOLOGY_TEE_NODE, &pTeeNode); - if (FAILED(hr)) - break; - hr = sourceNode->ConnectOutput(0, pTeeNode, 0); - if (FAILED(hr)) - break; - hr = pTeeNode->ConnectOutput(0, outputNode, 0); - if (FAILED(hr)) - break; - hr = pTeeNode->ConnectOutput(1, m_audioSampleGrabberNode, 0); - if (FAILED(hr)) - break; - } while (false); - - if (pTeeNode) - pTeeNode->Release(); - if (mediaType) - mediaType->Release(); - if (typeHandler) - typeHandler->Release(); - return hr == S_OK; -} - // BindOutputNode // Sets the IMFStreamSink pointer on an output node. // IMFActivate pointer in the output node must be converted to an // IMFStreamSink pointer before the topology loader resolves the topology. HRESULT BindOutputNode(IMFTopologyNode *pNode) { - IUnknown *nodeObject = NULL; - IMFActivate *activate = NULL; - IMFStreamSink *stream = NULL; - IMFMediaSink *sink = NULL; + ComPtr<IUnknown> nodeObject; + ComPtr<IMFActivate> activate; + ComPtr<IMFStreamSink> stream; + ComPtr<IMFMediaSink> sink; HRESULT hr = pNode->GetObject(&nodeObject); if (FAILED(hr)) @@ -666,20 +504,12 @@ HRESULT BindOutputNode(IMFTopologyNode *pNode) // Replace the node's object pointer with the stream sink. if (SUCCEEDED(hr)) { - hr = pNode->SetObject(stream); + hr = pNode->SetObject(stream.Get()); } } else { hr = nodeObject->QueryInterface(IID_PPV_ARGS(&stream)); } - if (nodeObject) - nodeObject->Release(); - if (activate) - activate->Release(); - if (stream) - stream->Release(); - if (sink) - sink->Release(); return hr; } @@ -687,7 +517,7 @@ HRESULT BindOutputNode(IMFTopologyNode *pNode) // Sets the IMFStreamSink pointers on all of the output nodes in a topology. HRESULT BindOutputNodes(IMFTopology *pTopology) { - IMFCollection *collection; + ComPtr<IMFCollection> collection; // Get the collection of output nodes. HRESULT hr = pTopology->GetOutputNodeCollection(&collection); @@ -699,25 +529,22 @@ HRESULT BindOutputNodes(IMFTopology *pTopology) if (SUCCEEDED(hr)) { for (DWORD i = 0; i < cNodes; i++) { - IUnknown *element; + ComPtr<IUnknown> element; hr = collection->GetElement(i, &element); if (FAILED(hr)) break; - IMFTopologyNode *node; - hr = element->QueryInterface(IID_IMFTopologyNode, (void**)&node); - element->Release(); + ComPtr<IMFTopologyNode> node; + hr = element->QueryInterface(IID_IMFTopologyNode, &node); if (FAILED(hr)) break; // Bind this node. - hr = BindOutputNode(node); - node->Release(); + hr = BindOutputNode(node.Get()); if (FAILED(hr)) break; } } - collection->Release(); } return hr; @@ -726,123 +553,38 @@ HRESULT BindOutputNodes(IMFTopology *pTopology) // This method binds output nodes to complete the topology, // then loads the topology and inserts MFT between the output node // and a filter connected to the output node. -IMFTopology *MFPlayerSession::insertMFT(IMFTopology *topology, TOPOID outputNodeId) +ComPtr<IMFTopology> MFPlayerSession::insertMFT(const ComPtr<IMFTopology> &topology, + TOPOID outputNodeId) { bool isNewTopology = false; - IMFTopoLoader *topoLoader = 0; - IMFTopology *resolvedTopology = 0; - IMFCollection *outputNodes = 0; + ComPtr<IMFTopoLoader> topoLoader; + ComPtr<IMFTopology> resolvedTopology; + ComPtr<IMFCollection> outputNodes; do { - if (FAILED(BindOutputNodes(topology))) + if (FAILED(BindOutputNodes(topology.Get()))) break; if (FAILED(MFCreateTopoLoader(&topoLoader))) break; - if (FAILED(topoLoader->Load(topology, &resolvedTopology, NULL))) { + if (FAILED(topoLoader->Load(topology.Get(), &resolvedTopology, NULL))) { // Topology could not be resolved, adding ourselves a color converter // to the topology might solve the problem - insertColorConverter(topology, outputNodeId); - if (FAILED(topoLoader->Load(topology, &resolvedTopology, NULL))) + insertColorConverter(topology.Get(), outputNodeId); + if (FAILED(topoLoader->Load(topology.Get(), &resolvedTopology, NULL))) break; } - if (insertResizer(resolvedTopology)) + if (insertResizer(resolvedTopology.Get())) isNewTopology = true; - - // Get all output nodes and search for video output node. - if (FAILED(resolvedTopology->GetOutputNodeCollection(&outputNodes))) - break; - - DWORD elementCount = 0; - if (FAILED(outputNodes->GetElementCount(&elementCount))) - break; - - for (DWORD n = 0; n < elementCount; n++) { - IUnknown *element = 0; - IMFTopologyNode *node = 0; - IUnknown *outputObject = 0; - IMFTopologyNode *inputNode = 0; - IMFTopologyNode *mftNode = 0; - bool mftAdded = false; - - do { - if (FAILED(outputNodes->GetElement(n, &element))) - break; - - if (FAILED(element->QueryInterface(IID_IMFTopologyNode, (void**)&node))) - break; - - TOPOID id; - if (FAILED(node->GetTopoNodeID(&id))) - break; - - if (id != outputNodeId) - break; - - if (FAILED(node->GetObject(&outputObject))) - break; - - m_videoProbeMFT->setVideoSink(outputObject); - - // Insert MFT between the output node and the node connected to it. - DWORD outputIndex = 0; - if (FAILED(node->GetInput(0, &inputNode, &outputIndex))) - break; - - if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_TRANSFORM_NODE, &mftNode))) - break; - - if (FAILED(mftNode->SetObject(m_videoProbeMFT))) - break; - - if (FAILED(resolvedTopology->AddNode(mftNode))) - break; - - if (FAILED(inputNode->ConnectOutput(0, mftNode, 0))) - break; - - if (FAILED(mftNode->ConnectOutput(0, node, 0))) - break; - - mftAdded = true; - isNewTopology = true; - } while (false); - - if (mftNode) - mftNode->Release(); - if (inputNode) - inputNode->Release(); - if (node) - node->Release(); - if (element) - element->Release(); - if (outputObject) - outputObject->Release(); - - if (mftAdded) - break; - else - m_videoProbeMFT->setVideoSink(NULL); - } } while (false); - if (outputNodes) - outputNodes->Release(); - - if (topoLoader) - topoLoader->Release(); - if (isNewTopology) { - topology->Release(); return resolvedTopology; } - if (resolvedTopology) - resolvedTopology->Release(); - return topology; } @@ -854,26 +596,20 @@ bool MFPlayerSession::insertResizer(IMFTopology *topology) { bool inserted = false; WORD elementCount = 0; - IMFTopologyNode *node = 0; - IUnknown *object = 0; - IWMColorConvProps *colorConv = 0; - IMFTransform *resizer = 0; - IMFTopologyNode *resizerNode = 0; - IMFTopologyNode *inputNode = 0; + ComPtr<IMFTopologyNode> node; + ComPtr<IUnknown> object; + ComPtr<IWMColorConvProps> colorConv; + ComPtr<IMFTransform> resizer; + ComPtr<IMFTopologyNode> resizerNode; + ComPtr<IMFTopologyNode> inputNode; HRESULT hr = topology->GetNodeCount(&elementCount); if (FAILED(hr)) return false; for (WORD i = 0; i < elementCount; ++i) { - if (node) { - node->Release(); - node = 0; - } - if (object) { - object->Release(); - object = 0; - } + node.Reset(); + object.Reset(); if (FAILED(topology->GetNode(i, &node))) break; @@ -888,35 +624,36 @@ bool MFPlayerSession::insertResizer(IMFTopology *topology) if (FAILED(node->GetObject(&object))) break; - if (FAILED(object->QueryInterface(&colorConv))) + if (FAILED(object->QueryInterface(IID_PPV_ARGS(&colorConv)))) continue; - if (FAILED(CoCreateInstance(CLSID_CResizerDMO, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (void**)&resizer))) + if (FAILED(CoCreateInstance(CLSID_CResizerDMO, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, + &resizer))) break; if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_TRANSFORM_NODE, &resizerNode))) break; - if (FAILED(resizerNode->SetObject(resizer))) + if (FAILED(resizerNode->SetObject(resizer.Get()))) break; - if (FAILED(topology->AddNode(resizerNode))) + if (FAILED(topology->AddNode(resizerNode.Get()))) break; DWORD outputIndex = 0; if (FAILED(node->GetInput(0, &inputNode, &outputIndex))) { - topology->RemoveNode(resizerNode); + topology->RemoveNode(resizerNode.Get()); break; } - if (FAILED(inputNode->ConnectOutput(0, resizerNode, 0))) { - topology->RemoveNode(resizerNode); + if (FAILED(inputNode->ConnectOutput(0, resizerNode.Get(), 0))) { + topology->RemoveNode(resizerNode.Get()); break; } - if (FAILED(resizerNode->ConnectOutput(0, node, 0))) { - inputNode->ConnectOutput(0, node, 0); - topology->RemoveNode(resizerNode); + if (FAILED(resizerNode->ConnectOutput(0, node.Get(), 0))) { + inputNode->ConnectOutput(0, node.Get(), 0); + topology->RemoveNode(resizerNode.Get()); break; } @@ -924,19 +661,6 @@ bool MFPlayerSession::insertResizer(IMFTopology *topology) break; } - if (node) - node->Release(); - if (object) - object->Release(); - if (colorConv) - colorConv->Release(); - if (resizer) - resizer->Release(); - if (resizerNode) - resizerNode->Release(); - if (inputNode) - inputNode->Release(); - return inserted; } @@ -946,27 +670,27 @@ bool MFPlayerSession::insertResizer(IMFTopology *topology) // for some reason it fails to do so in some cases, we then do it ourselves. void MFPlayerSession::insertColorConverter(IMFTopology *topology, TOPOID outputNodeId) { - IMFCollection *outputNodes = 0; + ComPtr<IMFCollection> outputNodes; if (FAILED(topology->GetOutputNodeCollection(&outputNodes))) return; DWORD elementCount = 0; if (FAILED(outputNodes->GetElementCount(&elementCount))) - goto done; + return; for (DWORD n = 0; n < elementCount; n++) { - IUnknown *element = 0; - IMFTopologyNode *node = 0; - IMFTopologyNode *inputNode = 0; - IMFTopologyNode *mftNode = 0; - IMFTransform *converter = 0; + ComPtr<IUnknown> element; + ComPtr<IMFTopologyNode> node; + ComPtr<IMFTopologyNode> inputNode; + ComPtr<IMFTopologyNode> mftNode; + ComPtr<IMFTransform> converter; do { if (FAILED(outputNodes->GetElement(n, &element))) break; - if (FAILED(element->QueryInterface(IID_IMFTopologyNode, (void**)&node))) + if (FAILED(element->QueryInterface(IID_IMFTopologyNode, &node))) break; TOPOID id; @@ -983,38 +707,24 @@ void MFPlayerSession::insertColorConverter(IMFTopology *topology, TOPOID outputN if (FAILED(MFCreateTopologyNode(MF_TOPOLOGY_TRANSFORM_NODE, &mftNode))) break; - if (FAILED(CoCreateInstance(CLSID_CColorConvertDMO, NULL, CLSCTX_INPROC_SERVER, IID_IMFTransform, (void**)&converter))) + if (FAILED(CoCreateInstance(CLSID_CColorConvertDMO, NULL, CLSCTX_INPROC_SERVER, + IID_IMFTransform, &converter))) break; - if (FAILED(mftNode->SetObject(converter))) + if (FAILED(mftNode->SetObject(converter.Get()))) break; - if (FAILED(topology->AddNode(mftNode))) + if (FAILED(topology->AddNode(mftNode.Get()))) break; - if (FAILED(inputNode->ConnectOutput(0, mftNode, 0))) + if (FAILED(inputNode->ConnectOutput(0, mftNode.Get(), 0))) break; - if (FAILED(mftNode->ConnectOutput(0, node, 0))) + if (FAILED(mftNode->ConnectOutput(0, node.Get(), 0))) break; } while (false); - - if (mftNode) - mftNode->Release(); - if (inputNode) - inputNode->Release(); - if (node) - node->Release(); - if (element) - element->Release(); - if (converter) - converter->Release(); } - -done: - if (outputNodes) - outputNodes->Release(); } void MFPlayerSession::stop(bool immediate) @@ -1040,7 +750,7 @@ void MFPlayerSession::stop(bool immediate) positionChanged(0); } } else { - emit error(QMediaPlayer::ResourceError, tr("Failed to stop."), true); + error(QMediaPlayer::ResourceError, tr("Failed to stop."), true); } } } @@ -1084,11 +794,8 @@ void MFPlayerSession::start() if (SUCCEEDED(m_session->Start(&GUID_NULL, &varStart))) { m_state.setCommand(CmdStart); m_pendingState = CmdPending; - m_lastSeekPos = m_position; - m_lastSeekSysTime = MFGetSystemTime(); - m_altTiming = false; } else { - emit error(QMediaPlayer::ResourceError, tr("failed to start playback"), true); + error(QMediaPlayer::ResourceError, tr("failed to start playback"), true); } PropVariantClear(&varStart); } @@ -1109,7 +816,7 @@ void MFPlayerSession::pause() m_state.setCommand(CmdPause); m_pendingState = CmdPending; } else { - emit error(QMediaPlayer::ResourceError, tr("Failed to pause."), false); + error(QMediaPlayer::ResourceError, tr("Failed to pause."), false); } if (m_status == QMediaPlayer::EndOfMedia) { setPosition(0); @@ -1126,7 +833,7 @@ void MFPlayerSession::changeStatus(QMediaPlayer::MediaStatus newStatus) qDebug() << "MFPlayerSession::changeStatus" << newStatus; #endif m_status = newStatus; - emit statusChanged(); + statusChanged(); } QMediaPlayer::MediaStatus MFPlayerSession::status() const @@ -1143,27 +850,25 @@ bool MFPlayerSession::createSession() HRESULT hr = MFCreateMediaSession(NULL, &m_session); if (FAILED(hr)) { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Unable to create mediasession."), true); + error(QMediaPlayer::ResourceError, tr("Unable to create mediasession."), true); return false; } - m_hCloseEvent = CreateEvent(NULL, FALSE, FALSE, NULL); + m_hCloseEvent = EventHandle{ CreateEvent(NULL, FALSE, FALSE, NULL) }; - hr = m_session->BeginGetEvent(this, m_session); + hr = m_session->BeginGetEvent(this, m_session.Get()); if (FAILED(hr)) { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::ResourceError, tr("Unable to pull session events."), false); + error(QMediaPlayer::ResourceError, tr("Unable to pull session events."), false); close(); return false; } - m_sourceResolver = new SourceResolver(); - QObject::connect(m_sourceResolver, &SourceResolver::mediaSourceReady, this, &MFPlayerSession::handleMediaSourceReady); - QObject::connect(m_sourceResolver, &SourceResolver::error, this, &MFPlayerSession::handleSourceError); - - m_videoProbeMFT = new MFTransform; -// for (int i = 0; i < m_videoProbes.size(); ++i) -// m_videoProbeMFT->addProbe(m_videoProbes.at(i)); + m_sourceResolver = makeComObject<SourceResolver>(); + QObject::connect(m_sourceResolver.Get(), &SourceResolver::mediaSourceReady, this, + &MFPlayerSession::handleMediaSourceReady); + QObject::connect(m_sourceResolver.Get(), &SourceResolver::error, this, + &MFPlayerSession::handleSourceError); m_position = 0; return true; @@ -1184,13 +889,6 @@ qint64 MFPlayerSession::position() MFTIME time, sysTime; if (FAILED(m_presentationClock->GetCorrelatedTime(0, &time, &sysTime))) return m_position / 10000; - - if (time > 0 && qint64(time) < m_lastSeekPos) - m_altTiming = true; - - if (m_altTiming) - return (m_lastSeekPos + MFGetSystemTime() - m_lastSeekSysTime) / 10000; - return qint64(time / 10000); } return m_position / 10000; @@ -1231,19 +929,14 @@ void MFPlayerSession::setPositionInternal(qint64 position, Command requestCmd) PROPVARIANT varStart; varStart.vt = VT_I8; varStart.hVal.QuadPart = LONGLONG(position * 10000); - if (SUCCEEDED(m_session->Start(NULL, &varStart))) - { - m_lastSeekPos = position * 10000; - m_lastSeekSysTime = MFGetSystemTime(); - m_altTiming = false; - + if (SUCCEEDED(m_session->Start(NULL, &varStart))) { PropVariantClear(&varStart); // Store the pending state. m_state.setCommand(CmdStart); m_state.start = position; m_pendingState = SeekPending; } else { - emit error(QMediaPlayer::ResourceError, tr("Failed to seek."), true); + error(QMediaPlayer::ResourceError, tr("Failed to seek."), true); } } @@ -1258,7 +951,7 @@ void MFPlayerSession::setPlaybackRate(qreal rate) { if (m_scrubbing) { m_restoreRate = rate; - emit playbackRateChanged(rate); + playbackRateChanged(rate); return; } setPlaybackRateInternal(rate); @@ -1417,7 +1110,7 @@ done: m_pendingRate = m_request.rate = m_state.rate = rate; if (rate != 0) m_state.isThin = isThin; - emit playbackRateChanged(rate); + playbackRateChanged(rate); } void MFPlayerSession::scrub(bool enableScrub) @@ -1551,17 +1244,22 @@ ULONG MFPlayerSession::AddRef(void) ULONG MFPlayerSession::Release(void) { LONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) - this->deleteLater(); + if (cRef == 0) { + deleteLater(); + + // In rare cases the session has queued events to be run between deleteLater and deleting, + // so we set the parent control to nullptr in order to prevent crashes in the cases. + m_playerControl = nullptr; + } return cRef; } HRESULT MFPlayerSession::Invoke(IMFAsyncResult *pResult) { - if (pResult->GetStateNoAddRef() != m_session) + if (pResult->GetStateNoAddRef() != m_session.Get()) return S_OK; - IMFMediaEvent *pEvent = NULL; + ComPtr<IMFMediaEvent> pEvent; // Get the event from the event queue. HRESULT hr = m_session->EndGetEvent(pResult, &pEvent); if (FAILED(hr)) { @@ -1571,36 +1269,30 @@ HRESULT MFPlayerSession::Invoke(IMFAsyncResult *pResult) MediaEventType meType = MEUnknown; hr = pEvent->GetType(&meType); if (FAILED(hr)) { - pEvent->Release(); return S_OK; } if (meType == MESessionClosed) { - SetEvent(m_hCloseEvent); - pEvent->Release(); + SetEvent(m_hCloseEvent.get()); return S_OK; } else { - hr = m_session->BeginGetEvent(this, m_session); + hr = m_session->BeginGetEvent(this, m_session.Get()); if (FAILED(hr)) { - pEvent->Release(); return S_OK; } } if (!m_closing) { emit sessionEvent(pEvent); - } else { - pEvent->Release(); } return S_OK; } -void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) +void MFPlayerSession::handleSessionEvent(const ComPtr<IMFMediaEvent> &sessionEvent) { HRESULT hrStatus = S_OK; HRESULT hr = sessionEvent->GetStatus(&hrStatus); if (FAILED(hr) || !m_session) { - sessionEvent->Release(); return; } @@ -1620,7 +1312,7 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) sessionEvent->GetValue(&var); qWarning() << "handleSessionEvent: non fatal error = " << var.ulVal; PropVariantClear(&var); - emit error(QMediaPlayer::ResourceError, tr("Media session non-fatal error."), false); + error(QMediaPlayer::ResourceError, tr("Media session non-fatal error."), false); } break; case MESourceUnknown: @@ -1636,8 +1328,28 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) break; } changeStatus(QMediaPlayer::InvalidMedia); - qWarning() << "handleSessionEvent: serious error = " << hrStatus; - emit error(QMediaPlayer::ResourceError, tr("Media session serious error."), true); + qWarning() << "handleSessionEvent: serious error = " + << Qt::showbase << Qt::hex << Qt::uppercasedigits << static_cast<quint32>(hrStatus); + switch (hrStatus) { + case MF_E_NET_READ: + error(QMediaPlayer::NetworkError, tr("Error reading from the network."), true); + break; + case MF_E_NET_WRITE: + error(QMediaPlayer::NetworkError, tr("Error writing to the network."), true); + break; + case NS_E_FIREWALL: + error(QMediaPlayer::NetworkError, tr("Network packets might be blocked by a firewall."), true); + break; + case MF_E_MEDIAPROC_WRONGSTATE: + error(QMediaPlayer::ResourceError, tr("Media session state error."), true); + break; + case MF_E_INVALID_STREAM_DATA: + error(QMediaPlayer::ResourceError, tr("Invalid stream data."), true); + break; + default: + error(QMediaPlayer::ResourceError, tr("Media session serious error."), true); + break; + } break; case MESessionRateChanged: // If the rate change succeeded, we've already got the rate @@ -1648,7 +1360,7 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) if (SUCCEEDED(sessionEvent->GetValue(&var)) && (var.vt == VT_R4)) { m_state.rate = var.fltVal; } - emit playbackRateChanged(playbackRate()); + playbackRateChanged(playbackRate()); } break; case MESessionScrubSampleComplete : @@ -1702,28 +1414,8 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) case MESessionTopologySet: if (FAILED(hrStatus)) { changeStatus(QMediaPlayer::InvalidMedia); - emit error(QMediaPlayer::FormatError, tr("Unsupported media, a codec is missing."), true); + error(QMediaPlayer::FormatError, tr("Unsupported media, a codec is missing."), true); } else { - if (m_audioSampleGrabberNode) { - IUnknown *obj = 0; - if (SUCCEEDED(m_audioSampleGrabberNode->GetObject(&obj))) { - IMFStreamSink *streamSink = 0; - if (SUCCEEDED(obj->QueryInterface(IID_PPV_ARGS(&streamSink)))) { - IMFMediaTypeHandler *typeHandler = 0; - if (SUCCEEDED(streamSink->GetMediaTypeHandler((&typeHandler)))) { - IMFMediaType *mediaType = 0; - if (SUCCEEDED(typeHandler->GetCurrentMediaType(&mediaType))) { - m_audioSampleGrabber->setFormat(QWindowsAudioUtils::mediaTypeToFormat(mediaType)); - mediaType->Release(); - } - typeHandler->Release(); - } - streamSink->Release(); - } - obj->Release(); - } - } - // Topology is resolved and successfuly set, this happens only after loading a new media. // Make sure we always start the media from the beginning m_lastPosition = -1; @@ -1735,18 +1427,17 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) } if (FAILED(hrStatus)) { - sessionEvent->Release(); return; } switch (meType) { case MEBufferingStarted: changeStatus(QMediaPlayer::StalledMedia); - emit bufferProgressChanged(bufferProgress()); + bufferProgressChanged(bufferProgress()); break; case MEBufferingStopped: changeStatus(QMediaPlayer::BufferedMedia); - emit bufferProgressChanged(bufferProgress()); + bufferProgressChanged(bufferProgress()); break; case MESessionEnded: m_pendingState = NoPending; @@ -1767,14 +1458,15 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) UINT32 status; if (SUCCEEDED(sessionEvent->GetUINT32(MF_EVENT_TOPOLOGY_STATUS, &status))) { if (status == MF_TOPOSTATUS_READY) { - IMFClock* clock; + ComPtr<IMFClock> clock; if (SUCCEEDED(m_session->GetClock(&clock))) { - clock->QueryInterface(IID_IMFPresentationClock, (void**)(&m_presentationClock)); - clock->Release(); + clock->QueryInterface(IID_IMFPresentationClock, &m_presentationClock); } - if (SUCCEEDED(MFGetService(m_session, MF_RATE_CONTROL_SERVICE, IID_PPV_ARGS(&m_rateControl)))) { - if (SUCCEEDED(MFGetService(m_session, MF_RATE_CONTROL_SERVICE, IID_PPV_ARGS(&m_rateSupport)))) { + if (SUCCEEDED(MFGetService(m_session.Get(), MF_RATE_CONTROL_SERVICE, + IID_PPV_ARGS(&m_rateControl)))) { + if (SUCCEEDED(MFGetService(m_session.Get(), MF_RATE_CONTROL_SERVICE, + IID_PPV_ARGS(&m_rateSupport)))) { if (SUCCEEDED(m_rateSupport->IsRateSupported(TRUE, 0, NULL))) m_canScrub = true; } @@ -1787,9 +1479,11 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) } } } - MFGetService(m_session, MFNETSOURCE_STATISTICS_SERVICE, IID_PPV_ARGS(&m_netsourceStatistics)); + MFGetService(m_session.Get(), MFNETSOURCE_STATISTICS_SERVICE, + IID_PPV_ARGS(&m_netsourceStatistics)); - if (SUCCEEDED(MFGetService(m_session, MR_STREAM_VOLUME_SERVICE, IID_PPV_ARGS(&m_volumeControl)))) + if (SUCCEEDED(MFGetService(m_session.Get(), MR_STREAM_VOLUME_SERVICE, + IID_PPV_ARGS(&m_volumeControl)))) setVolumeInternal(m_muted ? 0 : m_volume); m_updatingTopology = false; @@ -1801,8 +1495,6 @@ void MFPlayerSession::handleSessionEvent(IMFMediaEvent *sessionEvent) default: break; } - - sessionEvent->Release(); } void MFPlayerSession::updatePendingCommands(Command command) @@ -1889,33 +1581,14 @@ void MFPlayerSession::clear() if (!m_metaData.isEmpty()) { m_metaData.clear(); - emit metaDataChanged(); + metaDataChanged(); } - if (m_presentationClock) { - m_presentationClock->Release(); - m_presentationClock = NULL; - } - if (m_rateControl) { - m_rateControl->Release(); - m_rateControl = NULL; - } - if (m_rateSupport) { - m_rateSupport->Release(); - m_rateSupport = NULL; - } - if (m_volumeControl) { - m_volumeControl->Release(); - m_volumeControl = NULL; - } - if (m_netsourceStatistics) { - m_netsourceStatistics->Release(); - m_netsourceStatistics = NULL; - } - if (m_audioSampleGrabberNode) { - m_audioSampleGrabberNode->Release(); - m_audioSampleGrabberNode = NULL; - } + m_presentationClock.Reset(); + m_rateControl.Reset(); + m_rateSupport.Reset(); + m_volumeControl.Reset(); + m_netsourceStatistics.Reset(); } void MFPlayerSession::setAudioOutput(QPlatformAudioOutput *device) @@ -1968,7 +1641,7 @@ void MFPlayerSession::setActiveTrack(QPlatformMediaPlayer::TrackType type, int i if (m_trackInfo[QPlatformMediaPlayer::VideoStream].format == MFVideoFormat_HEVC) return; - IMFTopology *topology = nullptr; + ComPtr<IMFTopology> topology; if (SUCCEEDED(m_session->GetFullTopology(QMM_MFSESSION_GETFULLTOPOLOGY_CURRENT, 0, &topology))) { @@ -1978,25 +1651,23 @@ void MFPlayerSession::setActiveTrack(QPlatformMediaPlayer::TrackType type, int i stop(); if (m_trackInfo[type].outputNodeId != TOPOID(-1)) { - IMFTopologyNode *node = nullptr; + ComPtr<IMFTopologyNode> node; if (SUCCEEDED(topology->GetNodeByID(m_trackInfo[type].outputNodeId, &node))) { - topology->RemoveNode(node); - node->Release(); + topology->RemoveNode(node.Get()); m_trackInfo[type].outputNodeId = TOPOID(-1); } } if (m_trackInfo[type].sourceNodeId != TOPOID(-1)) { - IMFTopologyNode *node = nullptr; + ComPtr<IMFTopologyNode> node; if (SUCCEEDED(topology->GetNodeByID(m_trackInfo[type].sourceNodeId, &node))) { - topology->RemoveNode(node); - node->Release(); + topology->RemoveNode(node.Get()); m_trackInfo[type].sourceNodeId = TOPOID(-1); } } IMFMediaSource *mediaSource = m_sourceResolver->mediaSource(); - IMFPresentationDescriptor *sourcePD = nullptr; + ComPtr<IMFPresentationDescriptor> sourcePD; if (SUCCEEDED(mediaSource->CreatePresentationDescriptor(&sourcePD))) { if (m_trackInfo[type].currentIndex >= 0 && m_trackInfo[type].currentIndex < nativeIndexes.count()) @@ -2005,55 +1676,53 @@ void MFPlayerSession::setActiveTrack(QPlatformMediaPlayer::TrackType type, int i m_trackInfo[type].currentIndex = index; if (index == -1) { - m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, topology); + m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, topology.Get()); } else { int nativeIndex = nativeIndexes.at(index); sourcePD->SelectStream(nativeIndex); - IMFStreamDescriptor *streamDesc = nullptr; + ComPtr<IMFStreamDescriptor> streamDesc; BOOL selected = FALSE; if (SUCCEEDED(sourcePD->GetStreamDescriptorByIndex(nativeIndex, &selected, &streamDesc))) { - IMFTopologyNode *sourceNode = addSourceNode(topology, mediaSource, sourcePD, streamDesc); + ComPtr<IMFTopologyNode> sourceNode = addSourceNode( + topology.Get(), mediaSource, sourcePD.Get(), streamDesc.Get()); if (sourceNode) { - IMFTopologyNode *outputNode = addOutputNode(MFPlayerSession::Audio, topology, 0); + ComPtr<IMFTopologyNode> outputNode = + addOutputNode(MFPlayerSession::Audio, topology.Get(), 0); if (outputNode) { - if (SUCCEEDED(sourceNode->ConnectOutput(0, outputNode, 0))) { + if (SUCCEEDED(sourceNode->ConnectOutput(0, outputNode.Get(), 0))) { sourceNode->GetTopoNodeID(&m_trackInfo[type].sourceNodeId); outputNode->GetTopoNodeID(&m_trackInfo[type].outputNodeId); - m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, topology); + m_session->SetTopology(MFSESSION_SETTOPOLOGY_IMMEDIATE, + topology.Get()); } - outputNode->Release(); } - sourceNode->Release(); } - streamDesc->Release(); } } m_updatingTopology = true; - sourcePD->Release(); } - topology->Release(); } } int MFPlayerSession::activeTrack(QPlatformMediaPlayer::TrackType type) { - if (type < 0 || type >= QPlatformMediaPlayer::NTrackTypes) + if (type >= QPlatformMediaPlayer::NTrackTypes) return -1; return m_trackInfo[type].currentIndex; } int MFPlayerSession::trackCount(QPlatformMediaPlayer::TrackType type) { - if (type < 0 || type >= QPlatformMediaPlayer::NTrackTypes) + if (type >= QPlatformMediaPlayer::NTrackTypes) return -1; return m_trackInfo[type].metaData.count(); } QMediaMetaData MFPlayerSession::trackMetaData(QPlatformMediaPlayer::TrackType type, int trackNumber) { - if (type < 0 || type >= QPlatformMediaPlayer::NTrackTypes) + if (type >= QPlatformMediaPlayer::NTrackTypes) return {}; if (trackNumber < 0 || trackNumber >= m_trackInfo[type].metaData.count()) @@ -2062,3 +1731,6 @@ QMediaMetaData MFPlayerSession::trackMetaData(QPlatformMediaPlayer::TrackType ty return m_trackInfo[type].metaData.at(trackNumber); } +QT_END_NAMESPACE + +#include "moc_mfplayersession_p.cpp" diff --git a/src/plugins/multimedia/windows/player/mfplayersession_p.h b/src/plugins/multimedia/windows/player/mfplayersession_p.h index 1f3555ba8..50141a7fb 100644 --- a/src/plugins/multimedia/windows/player/mfplayersession_p.h +++ b/src/plugins/multimedia/windows/player/mfplayersession_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFPLAYERSESSION_H #define MFPLAYERSESSION_H @@ -66,12 +30,12 @@ #include <qaudiodevice.h> #include <qtimer.h> #include "mfplayercontrol_p.h" +#include <private/qcomptr_p.h> +#include <evrhelpers_p.h> QT_BEGIN_NAMESPACE -class QUrl; -QT_END_NAMESPACE -QT_USE_NAMESPACE +class QUrl; class SourceResolver; class MFVideoRendererControl; @@ -86,7 +50,6 @@ class MFPlayerSession : public QObject, public IMFAsyncCallback friend class SourceResolver; public: MFPlayerSession(MFPlayerControl *playerControl = 0); - ~MFPlayerSession(); STDMETHODIMP QueryInterface(REFIID riid, LPVOID *ppvObject) override; @@ -151,11 +114,11 @@ public Q_SLOTS: void updateOutputRouting(); Q_SIGNALS: - void sessionEvent(IMFMediaEvent *sessionEvent); + void sessionEvent(const ComPtr<IMFMediaEvent> &sessionEvent); private Q_SLOTS: void handleMediaSourceReady(); - void handleSessionEvent(IMFMediaEvent *sessionEvent); + void handleSessionEvent(const ComPtr<IMFMediaEvent> &sessionEvent); void handleSourceError(long hr); void timeout(); @@ -163,20 +126,17 @@ private: long m_cRef; MFPlayerControl *m_playerControl = nullptr; MFVideoRendererControl *m_videoRendererControl = nullptr; - IMFMediaSession *m_session; - IMFPresentationClock *m_presentationClock; - IMFRateControl *m_rateControl; - IMFRateSupport *m_rateSupport; - IMFAudioStreamVolume *m_volumeControl; - IPropertyStore *m_netsourceStatistics; + ComPtr<IMFMediaSession> m_session; + ComPtr<IMFPresentationClock> m_presentationClock; + ComPtr<IMFRateControl> m_rateControl; + ComPtr<IMFRateSupport> m_rateSupport; + ComPtr<IMFAudioStreamVolume> m_volumeControl; + ComPtr<IPropertyStore> m_netsourceStatistics; qint64 m_position = 0; qint64 m_restorePosition = -1; qint64 m_timeCounter = 0; UINT64 m_duration = 0; bool m_updatingTopology = false; - qint64 m_lastSeekPos = 0; - MFTIME m_lastSeekSysTime = 0; - bool m_altTiming = false; bool m_updateRoutingOnStart = false; enum Command @@ -199,8 +159,8 @@ private: bool m_scrubbing; float m_restoreRate; - SourceResolver *m_sourceResolver; - HANDLE m_hCloseEvent; + ComPtr<SourceResolver> m_sourceResolver; + EventHandle m_hCloseEvent; bool m_closing; enum MediaType @@ -260,26 +220,21 @@ private: bool createSession(); void setupPlaybackTopology(IMFMediaSource *source, IMFPresentationDescriptor *sourcePD); bool getStreamInfo(IMFStreamDescriptor *stream, MFPlayerSession::MediaType *type, QString *name, QString *language, GUID *format) const; - IMFTopologyNode* addSourceNode(IMFTopology* topology, IMFMediaSource* source, - IMFPresentationDescriptor* presentationDesc, IMFStreamDescriptor *streamDesc); - IMFTopologyNode* addOutputNode(MediaType mediaType, IMFTopology* topology, DWORD sinkID); + ComPtr<IMFTopologyNode> addSourceNode(IMFTopology *topology, IMFMediaSource *source, + IMFPresentationDescriptor *presentationDesc, + IMFStreamDescriptor *streamDesc); + ComPtr<IMFTopologyNode> addOutputNode(MediaType mediaType, IMFTopology *topology, DWORD sinkID); - bool addAudioSampleGrabberNode(IMFTopology* topology); - bool setupAudioSampleGrabber(IMFTopology *topology, IMFTopologyNode *sourceNode, IMFTopologyNode *outputNode); QAudioFormat audioFormatForMFMediaType(IMFMediaType *mediaType) const; - // ### Below can be used to monitor the audio channel. Currently unused. - AudioSampleGrabberCallback *m_audioSampleGrabber; - IMFTopologyNode *m_audioSampleGrabberNode; - IMFTopology *insertMFT(IMFTopology *topology, TOPOID outputNodeId); + ComPtr<IMFTopology> insertMFT(const ComPtr<IMFTopology> &topology, TOPOID outputNodeId); bool insertResizer(IMFTopology *topology); void insertColorConverter(IMFTopology *topology, TOPOID outputNodeId); - // ### Below can be used to monitor the video channel. Functionality currently unused. - MFTransform *m_videoProbeMFT; QTimer m_signalPositionChangeTimer; qint64 m_lastPosition = -1; }; +QT_END_NAMESPACE #endif diff --git a/src/plugins/multimedia/windows/player/mftvideo.cpp b/src/plugins/multimedia/windows/player/mftvideo.cpp deleted file mode 100644 index fcc8d17c5..000000000 --- a/src/plugins/multimedia/windows/player/mftvideo.cpp +++ /dev/null @@ -1,728 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "mftvideo_p.h" -#include <private/qmemoryvideobuffer_p.h> -#include <private/qwindowsmultimediautils_p.h> -#include <mferror.h> -#include <strmif.h> -#include <uuids.h> -#include <InitGuid.h> -#include <d3d9.h> -#include <qdebug.h> - -// This MFT sends all samples it processes to connected video probes. -// Sample is sent to probes in ProcessInput. -// In ProcessOutput this MFT simply returns the original sample. - -// The implementation is based on a boilerplate from the MF SDK example. - -MFTransform::MFTransform(): - m_cRef(1), - m_inputType(0), - m_outputType(0), - m_sample(0), - m_videoSinkTypeHandler(0), - m_bytesPerLine(0) -{ -} - -MFTransform::~MFTransform() -{ - if (m_inputType) - m_inputType->Release(); - - if (m_outputType) - m_outputType->Release(); - - if (m_videoSinkTypeHandler) - m_videoSinkTypeHandler->Release(); -} - -//void MFTransform::addProbe(MFVideoProbeControl *probe) -//{ -// QMutexLocker locker(&m_videoProbeMutex); - -// if (m_videoProbes.contains(probe)) -// return; - -// m_videoProbes.append(probe); -//} - -//void MFTransform::removeProbe(MFVideoProbeControl *probe) -//{ -// QMutexLocker locker(&m_videoProbeMutex); -// m_videoProbes.removeOne(probe); -//} - -void MFTransform::setVideoSink(IUnknown *videoSink) -{ - // This transform supports the same input types as the video sink. - // Store its type handler interface in order to report the correct supported types. - - if (m_videoSinkTypeHandler) { - m_videoSinkTypeHandler->Release(); - m_videoSinkTypeHandler = NULL; - } - - if (videoSink) - videoSink->QueryInterface(IID_PPV_ARGS(&m_videoSinkTypeHandler)); -} - -STDMETHODIMP MFTransform::QueryInterface(REFIID riid, void** ppv) -{ - if (!ppv) - return E_POINTER; - if (riid == IID_IMFTransform) { - *ppv = static_cast<IMFTransform*>(this); - } else if (riid == IID_IUnknown) { - *ppv = static_cast<IUnknown*>(this); - } else { - *ppv = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; -} - -STDMETHODIMP_(ULONG) MFTransform::AddRef() -{ - return InterlockedIncrement(&m_cRef); -} - -STDMETHODIMP_(ULONG) MFTransform::Release() -{ - ULONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) { - delete this; - } - return cRef; -} - -STDMETHODIMP MFTransform::GetStreamLimits(DWORD *pdwInputMinimum, DWORD *pdwInputMaximum, DWORD *pdwOutputMinimum, DWORD *pdwOutputMaximum) -{ - if (!pdwInputMinimum || !pdwInputMaximum || !pdwOutputMinimum || !pdwOutputMaximum) - return E_POINTER; - *pdwInputMinimum = 1; - *pdwInputMaximum = 1; - *pdwOutputMinimum = 1; - *pdwOutputMaximum = 1; - return S_OK; -} - -STDMETHODIMP MFTransform::GetStreamCount(DWORD *pcInputStreams, DWORD *pcOutputStreams) -{ - if (!pcInputStreams || !pcOutputStreams) - return E_POINTER; - - *pcInputStreams = 1; - *pcOutputStreams = 1; - return S_OK; -} - -STDMETHODIMP MFTransform::GetStreamIDs(DWORD dwInputIDArraySize, DWORD *pdwInputIDs, DWORD dwOutputIDArraySize, DWORD *pdwOutputIDs) -{ - // streams are numbered consecutively - Q_UNUSED(dwInputIDArraySize); - Q_UNUSED(pdwInputIDs); - Q_UNUSED(dwOutputIDArraySize); - Q_UNUSED(pdwOutputIDs); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_STREAM_INFO *pStreamInfo) -{ - QMutexLocker locker(&m_mutex); - - if (dwInputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (!pStreamInfo) - return E_POINTER; - - pStreamInfo->cbSize = 0; - pStreamInfo->hnsMaxLatency = 0; - pStreamInfo->cbMaxLookahead = 0; - pStreamInfo->cbAlignment = 0; - pStreamInfo->dwFlags = MFT_INPUT_STREAM_WHOLE_SAMPLES - | MFT_INPUT_STREAM_SINGLE_SAMPLE_PER_BUFFER - | MFT_INPUT_STREAM_PROCESSES_IN_PLACE; - - return S_OK; -} - -STDMETHODIMP MFTransform::GetOutputStreamInfo(DWORD dwOutputStreamID, MFT_OUTPUT_STREAM_INFO *pStreamInfo) -{ - QMutexLocker locker(&m_mutex); - - if (dwOutputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (!pStreamInfo) - return E_POINTER; - - pStreamInfo->cbSize = 0; - pStreamInfo->cbAlignment = 0; - pStreamInfo->dwFlags = MFT_OUTPUT_STREAM_WHOLE_SAMPLES - | MFT_OUTPUT_STREAM_SINGLE_SAMPLE_PER_BUFFER - | MFT_OUTPUT_STREAM_PROVIDES_SAMPLES - | MFT_OUTPUT_STREAM_DISCARDABLE; - - return S_OK; -} - -STDMETHODIMP MFTransform::GetAttributes(IMFAttributes **pAttributes) -{ - // This MFT does not support attributes. - Q_UNUSED(pAttributes); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::GetInputStreamAttributes(DWORD dwInputStreamID, IMFAttributes **pAttributes) -{ - // This MFT does not support input stream attributes. - Q_UNUSED(dwInputStreamID); - Q_UNUSED(pAttributes); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::GetOutputStreamAttributes(DWORD dwOutputStreamID, IMFAttributes **pAttributes) -{ - // This MFT does not support output stream attributes. - Q_UNUSED(dwOutputStreamID); - Q_UNUSED(pAttributes); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::DeleteInputStream(DWORD dwStreamID) -{ - // This MFT has a fixed number of input streams. - Q_UNUSED(dwStreamID); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::AddInputStreams(DWORD cStreams, DWORD *adwStreamIDs) -{ - // This MFT has a fixed number of input streams. - Q_UNUSED(cStreams); - Q_UNUSED(adwStreamIDs); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::GetInputAvailableType(DWORD dwInputStreamID, DWORD dwTypeIndex, IMFMediaType **ppType) -{ - // We support the same input types as the video sink - if (!m_videoSinkTypeHandler) - return E_NOTIMPL; - - if (dwInputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (!ppType) - return E_POINTER; - - return m_videoSinkTypeHandler->GetMediaTypeByIndex(dwTypeIndex, ppType); -} - -STDMETHODIMP MFTransform::GetOutputAvailableType(DWORD dwOutputStreamID, DWORD dwTypeIndex, IMFMediaType **ppType) -{ - // Since we don't modify the samples, the output type must be the same as the input type. - // Report our input type as the only available output type. - - if (dwOutputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (!ppType) - return E_POINTER; - - // Input type must be set first - if (!m_inputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - if (dwTypeIndex > 0) - return MF_E_NO_MORE_TYPES; - - // Return a copy to make sure our type is not modified - if (FAILED(MFCreateMediaType(ppType))) - return E_OUTOFMEMORY; - - return m_inputType->CopyAllItems(*ppType); -} - -STDMETHODIMP MFTransform::SetInputType(DWORD dwInputStreamID, IMFMediaType *pType, DWORD dwFlags) -{ - if (dwInputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - QMutexLocker locker(&m_mutex); - - if (m_sample) - return MF_E_TRANSFORM_CANNOT_CHANGE_MEDIATYPE_WHILE_PROCESSING; - - if (!isMediaTypeSupported(pType)) - return MF_E_INVALIDMEDIATYPE; - - if (dwFlags == MFT_SET_TYPE_TEST_ONLY) - return pType ? S_OK : E_POINTER; - - if (m_inputType) { - m_inputType->Release(); - // Input type has changed, discard output type (if it's set) so it's reset later on - DWORD flags = 0; - if (m_outputType && m_outputType->IsEqual(pType, &flags) != S_OK) { - m_outputType->Release(); - m_outputType = 0; - } - } - - m_inputType = pType; - - if (m_inputType) - m_inputType->AddRef(); - - return S_OK; -} - -STDMETHODIMP MFTransform::SetOutputType(DWORD dwOutputStreamID, IMFMediaType *pType, DWORD dwFlags) -{ - if (dwOutputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (dwFlags == MFT_SET_TYPE_TEST_ONLY && !pType) - return E_POINTER; - - QMutexLocker locker(&m_mutex); - - // Input type must be set first - if (!m_inputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - if (m_sample) - return MF_E_TRANSFORM_CANNOT_CHANGE_MEDIATYPE_WHILE_PROCESSING; - - DWORD flags = 0; - if (pType && m_inputType->IsEqual(pType, &flags) != S_OK) - return MF_E_INVALIDMEDIATYPE; - - if (dwFlags == MFT_SET_TYPE_TEST_ONLY) - return pType ? S_OK : E_POINTER; - - if (m_outputType) - m_outputType->Release(); - - m_outputType = pType; - - if (m_outputType) { - m_outputType->AddRef(); - m_format = videoFormatForMFMediaType(m_outputType, &m_bytesPerLine); - } - - return S_OK; -} - -STDMETHODIMP MFTransform::GetInputCurrentType(DWORD dwInputStreamID, IMFMediaType **ppType) -{ - if (dwInputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (ppType == NULL) - return E_POINTER; - - QMutexLocker locker(&m_mutex); - - if (!m_inputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - // Return a copy to make sure our type is not modified - if (FAILED(MFCreateMediaType(ppType))) - return E_OUTOFMEMORY; - - return m_inputType->CopyAllItems(*ppType); -} - -STDMETHODIMP MFTransform::GetOutputCurrentType(DWORD dwOutputStreamID, IMFMediaType **ppType) -{ - if (dwOutputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (ppType == NULL) - return E_POINTER; - - QMutexLocker locker(&m_mutex); - - if (!m_outputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - // Return a copy to make sure our type is not modified - if (FAILED(MFCreateMediaType(ppType))) - return E_OUTOFMEMORY; - - return m_outputType->CopyAllItems(*ppType); -} - -STDMETHODIMP MFTransform::GetInputStatus(DWORD dwInputStreamID, DWORD *pdwFlags) -{ - if (dwInputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (!pdwFlags) - return E_POINTER; - - QMutexLocker locker(&m_mutex); - - if (!m_inputType || !m_outputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - if (m_sample) - *pdwFlags = 0; - else - *pdwFlags = MFT_INPUT_STATUS_ACCEPT_DATA; - - return S_OK; -} - -STDMETHODIMP MFTransform::GetOutputStatus(DWORD *pdwFlags) -{ - if (!pdwFlags) - return E_POINTER; - - QMutexLocker locker(&m_mutex); - - if (!m_inputType || !m_outputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - if (m_sample) - *pdwFlags = MFT_OUTPUT_STATUS_SAMPLE_READY; - else - *pdwFlags = 0; - - return S_OK; -} - -STDMETHODIMP MFTransform::SetOutputBounds(LONGLONG hnsLowerBound, LONGLONG hnsUpperBound) -{ - Q_UNUSED(hnsLowerBound); - Q_UNUSED(hnsUpperBound); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::ProcessEvent(DWORD dwInputStreamID, IMFMediaEvent *pEvent) -{ - // This MFT ignores all events, and the pipeline should send all events downstream. - Q_UNUSED(dwInputStreamID); - Q_UNUSED(pEvent); - return E_NOTIMPL; -} - -STDMETHODIMP MFTransform::ProcessMessage(MFT_MESSAGE_TYPE eMessage, ULONG_PTR ulParam) -{ - Q_UNUSED(ulParam); - - HRESULT hr = S_OK; - - switch (eMessage) - { - case MFT_MESSAGE_COMMAND_FLUSH: - hr = OnFlush(); - break; - - case MFT_MESSAGE_COMMAND_DRAIN: - // Drain: Tells the MFT not to accept any more input until - // all of the pending output has been processed. That is our - // default behevior already, so there is nothing to do. - break; - - case MFT_MESSAGE_SET_D3D_MANAGER: - // The pipeline should never send this message unless the MFT - // has the MF_SA_D3D_AWARE attribute set to TRUE. However, if we - // do get this message, it's invalid and we don't implement it. - hr = E_NOTIMPL; - break; - - // The remaining messages do not require any action from this MFT. - case MFT_MESSAGE_NOTIFY_BEGIN_STREAMING: - case MFT_MESSAGE_NOTIFY_END_STREAMING: - case MFT_MESSAGE_NOTIFY_END_OF_STREAM: - case MFT_MESSAGE_NOTIFY_START_OF_STREAM: - break; - - default: - break; - } - - return hr; -} - -STDMETHODIMP MFTransform::ProcessInput(DWORD dwInputStreamID, IMFSample *pSample, DWORD dwFlags) -{ - if (dwInputStreamID > 0) - return MF_E_INVALIDSTREAMNUMBER; - - if (dwFlags != 0) - return E_INVALIDARG; // dwFlags is reserved and must be zero. - - QMutexLocker locker(&m_mutex); - - if (!m_inputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - if (m_sample) - return MF_E_NOTACCEPTING; - - // Validate the number of buffers. There should only be a single buffer to hold the video frame. - DWORD dwBufferCount = 0; - HRESULT hr = pSample->GetBufferCount(&dwBufferCount); - if (FAILED(hr)) - return hr; - - if (dwBufferCount == 0) - return E_FAIL; - - if (dwBufferCount > 1) - return MF_E_SAMPLE_HAS_TOO_MANY_BUFFERS; - - m_sample = pSample; - m_sample->AddRef(); - - QMutexLocker lockerProbe(&m_videoProbeMutex); - -// if (!m_videoProbes.isEmpty()) { -// QVideoFrame frame = makeVideoFrame(); - -// for (MFVideoProbeControl* probe : qAsConst(m_videoProbes)) -// probe->bufferProbed(frame); -// } - - return S_OK; -} - -STDMETHODIMP MFTransform::ProcessOutput(DWORD dwFlags, DWORD cOutputBufferCount, MFT_OUTPUT_DATA_BUFFER *pOutputSamples, DWORD *pdwStatus) -{ - if (pOutputSamples == NULL || pdwStatus == NULL) - return E_POINTER; - - if (cOutputBufferCount != 1) - return E_INVALIDARG; - - QMutexLocker locker(&m_mutex); - - if (!m_inputType) - return MF_E_TRANSFORM_TYPE_NOT_SET; - - if (!m_outputType) { - pOutputSamples[0].dwStatus = MFT_OUTPUT_DATA_BUFFER_FORMAT_CHANGE; - return MF_E_TRANSFORM_STREAM_CHANGE; - } - - IMFMediaBuffer *input = NULL; - IMFMediaBuffer *output = NULL; - - if (dwFlags == MFT_PROCESS_OUTPUT_DISCARD_WHEN_NO_BUFFER) - goto done; - else if (dwFlags != 0) - return E_INVALIDARG; - - if (!m_sample) - return MF_E_TRANSFORM_NEED_MORE_INPUT; - - // Since the MFT_OUTPUT_STREAM_PROVIDES_SAMPLES flag is set, the client - // should not be providing samples here - if (pOutputSamples[0].pSample != NULL) - return E_INVALIDARG; - - pOutputSamples[0].pSample = m_sample; - pOutputSamples[0].pSample->AddRef(); - - // Send video frame to probes - // We do it here (instead of inside ProcessInput) to make sure samples discarded by the renderer - // are not sent. - m_videoProbeMutex.lock(); -// if (!m_videoProbes.isEmpty()) { -// QVideoFrame frame = makeVideoFrame(); - -// for (MFVideoProbeControl* probe : qAsConst(m_videoProbes)) -// probe->bufferProbed(frame); -// } - m_videoProbeMutex.unlock(); - -done: - pOutputSamples[0].dwStatus = 0; - *pdwStatus = 0; - - m_sample->Release(); - m_sample = 0; - - if (input) - input->Release(); - if (output) - output->Release(); - - return S_OK; -} - -HRESULT MFTransform::OnFlush() -{ - QMutexLocker locker(&m_mutex); - - if (m_sample) { - m_sample->Release(); - m_sample = 0; - } - return S_OK; -} - -QVideoFrameFormat MFTransform::videoFormatForMFMediaType(IMFMediaType *mediaType, int *bytesPerLine) -{ - UINT32 stride; - if (FAILED(mediaType->GetUINT32(MF_MT_DEFAULT_STRIDE, &stride))) { - *bytesPerLine = 0; - return QVideoFrameFormat(); - } - - *bytesPerLine = (int)stride; - - QSize size; - UINT32 width, height; - if (FAILED(MFGetAttributeSize(mediaType, MF_MT_FRAME_SIZE, &width, &height))) - return QVideoFrameFormat(); - - size.setWidth(width); - size.setHeight(height); - - GUID subtype = GUID_NULL; - if (FAILED(mediaType->GetGUID(MF_MT_SUBTYPE, &subtype))) - return QVideoFrameFormat(); - - QVideoFrameFormat::PixelFormat pixelFormat = - QWindowsMultimediaUtils::pixelFormatFromMediaSubtype(subtype); - QVideoFrameFormat format(size, pixelFormat); - - quint32 num, den; - if (SUCCEEDED(MFGetAttributeRatio(mediaType, MF_MT_FRAME_RATE, &num, &den))) { - format.setFrameRate(qreal(num)/den); - } - - return format; -} - -QVideoFrame MFTransform::makeVideoFrame() -{ - QVideoFrame frame; - - if (!m_format.isValid()) - return frame; - - IMFMediaBuffer *buffer = 0; - - do { - if (FAILED(m_sample->ConvertToContiguousBuffer(&buffer))) - break; - - QByteArray array = dataFromBuffer(buffer, m_format.frameHeight(), &m_bytesPerLine); - if (array.isEmpty()) - break; - - // Wrapping IMFSample or IMFMediaBuffer in a QVideoFrame is not possible because we cannot hold - // IMFSample for a "long" time without affecting the rest of the topology. - // If IMFSample is held for more than 5 frames decoder starts to reuse it even though it hasn't been released it yet. - // That is why we copy data from IMFMediaBuffer here. - frame = QVideoFrame(new QMemoryVideoBuffer(array, m_bytesPerLine), m_format); - - // WMF uses 100-nanosecond units, Qt uses microseconds - LONGLONG startTime = -1; - if (SUCCEEDED(m_sample->GetSampleTime(&startTime))) { - frame.setStartTime(startTime * 0.1); - - LONGLONG duration = -1; - if (SUCCEEDED(m_sample->GetSampleDuration(&duration))) - frame.setEndTime((startTime + duration) * 0.1); - } - } while (false); - - if (buffer) - buffer->Release(); - - return frame; -} - -QByteArray MFTransform::dataFromBuffer(IMFMediaBuffer *buffer, int height, int *bytesPerLine) -{ - QByteArray array; - BYTE *bytes; - DWORD length; - HRESULT hr = buffer->Lock(&bytes, NULL, &length); - if (SUCCEEDED(hr)) { - array = QByteArray((const char *)bytes, (int)length); - buffer->Unlock(); - } else { - // try to lock as Direct3DSurface - IDirect3DSurface9 *surface = 0; - do { - if (FAILED(MFGetService(buffer, MR_BUFFER_SERVICE, IID_IDirect3DSurface9, (void**)&surface))) - break; - - D3DLOCKED_RECT rect; - if (FAILED(surface->LockRect(&rect, NULL, D3DLOCK_READONLY))) - break; - - if (bytesPerLine) - *bytesPerLine = (int)rect.Pitch; - - array = QByteArray((const char *)rect.pBits, rect.Pitch * height); - surface->UnlockRect(); - } while (false); - - if (surface) { - surface->Release(); - surface = 0; - } - } - - return array; -} - -bool MFTransform::isMediaTypeSupported(IMFMediaType *type) -{ - // If we don't have the video sink's type handler, - // assume it supports anything... - if (!m_videoSinkTypeHandler || !type) - return true; - - return m_videoSinkTypeHandler->IsMediaTypeSupported(type, NULL) == S_OK; -} diff --git a/src/plugins/multimedia/windows/player/mftvideo_p.h b/src/plugins/multimedia/windows/player/mftvideo_p.h deleted file mode 100644 index 6c0c2e6e7..000000000 --- a/src/plugins/multimedia/windows/player/mftvideo_p.h +++ /dev/null @@ -1,131 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef MFTRANSFORM_H -#define MFTRANSFORM_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <mfapi.h> -#include <mfidl.h> -#include <QtCore/qlist.h> -#include <QtCore/qmutex.h> -#include <QtMultimedia/qvideoframeformat.h> - -QT_USE_NAMESPACE - -class MFVideoProbeControl; - -QT_BEGIN_NAMESPACE -class QVideoFrame; -QT_END_NAMESPACE - -class MFTransform: public IMFTransform -{ -public: - MFTransform(); - virtual ~MFTransform(); - - void addProbe(MFVideoProbeControl* probe); - void removeProbe(MFVideoProbeControl* probe); - - void setVideoSink(IUnknown *videoSink); - - // IUnknown methods - STDMETHODIMP QueryInterface(REFIID iid, void** ppv) override; - STDMETHODIMP_(ULONG) AddRef() override; - STDMETHODIMP_(ULONG) Release() override; - - // IMFTransform methods - STDMETHODIMP GetStreamLimits(DWORD *pdwInputMinimum, DWORD *pdwInputMaximum, DWORD *pdwOutputMinimum, DWORD *pdwOutputMaximum) override; - STDMETHODIMP GetStreamCount(DWORD *pcInputStreams, DWORD *pcOutputStreams) override; - STDMETHODIMP GetStreamIDs(DWORD dwInputIDArraySize, DWORD *pdwInputIDs, DWORD dwOutputIDArraySize, DWORD *pdwOutputIDs) override; - STDMETHODIMP GetInputStreamInfo(DWORD dwInputStreamID, MFT_INPUT_STREAM_INFO *pStreamInfo) override; - STDMETHODIMP GetOutputStreamInfo(DWORD dwOutputStreamID, MFT_OUTPUT_STREAM_INFO *pStreamInfo) override; - STDMETHODIMP GetAttributes(IMFAttributes **pAttributes) override; - STDMETHODIMP GetInputStreamAttributes(DWORD dwInputStreamID, IMFAttributes **pAttributes) override; - STDMETHODIMP GetOutputStreamAttributes(DWORD dwOutputStreamID, IMFAttributes **pAttributes) override; - STDMETHODIMP DeleteInputStream(DWORD dwStreamID) override; - STDMETHODIMP AddInputStreams(DWORD cStreams, DWORD *adwStreamIDs) override; - STDMETHODIMP GetInputAvailableType(DWORD dwInputStreamID, DWORD dwTypeIndex, IMFMediaType **ppType) override; - STDMETHODIMP GetOutputAvailableType(DWORD dwOutputStreamID,DWORD dwTypeIndex, IMFMediaType **ppType) override; - STDMETHODIMP SetInputType(DWORD dwInputStreamID, IMFMediaType *pType, DWORD dwFlags) override; - STDMETHODIMP SetOutputType(DWORD dwOutputStreamID, IMFMediaType *pType, DWORD dwFlags) override; - STDMETHODIMP GetInputCurrentType(DWORD dwInputStreamID, IMFMediaType **ppType) override; - STDMETHODIMP GetOutputCurrentType(DWORD dwOutputStreamID, IMFMediaType **ppType) override; - STDMETHODIMP GetInputStatus(DWORD dwInputStreamID, DWORD *pdwFlags) override; - STDMETHODIMP GetOutputStatus(DWORD *pdwFlags) override; - STDMETHODIMP SetOutputBounds(LONGLONG hnsLowerBound, LONGLONG hnsUpperBound) override; - STDMETHODIMP ProcessEvent(DWORD dwInputStreamID, IMFMediaEvent *pEvent) override; - STDMETHODIMP ProcessMessage(MFT_MESSAGE_TYPE eMessage, ULONG_PTR ulParam) override; - STDMETHODIMP ProcessInput(DWORD dwInputStreamID, IMFSample *pSample, DWORD dwFlags) override; - STDMETHODIMP ProcessOutput(DWORD dwFlags, DWORD cOutputBufferCount, MFT_OUTPUT_DATA_BUFFER *pOutputSamples, DWORD *pdwStatus) override; - -private: - HRESULT OnFlush(); - static QVideoFrameFormat videoFormatForMFMediaType(IMFMediaType *mediaType, int *bytesPerLine); - QVideoFrame makeVideoFrame(); - QByteArray dataFromBuffer(IMFMediaBuffer *buffer, int height, int *bytesPerLine); - bool isMediaTypeSupported(IMFMediaType *type); - - long m_cRef; - IMFMediaType *m_inputType; - IMFMediaType *m_outputType; - IMFSample *m_sample; - QMutex m_mutex; - - IMFMediaTypeHandler *m_videoSinkTypeHandler; - -// QList<MFVideoProbeControl*> m_videoProbes; - QMutex m_videoProbeMutex; - - QVideoFrameFormat m_format; - int m_bytesPerLine; -}; - -#endif diff --git a/src/plugins/multimedia/windows/player/mfvideorenderercontrol.cpp b/src/plugins/multimedia/windows/player/mfvideorenderercontrol.cpp index af16663e0..7c79c3a8a 100644 --- a/src/plugins/multimedia/windows/player/mfvideorenderercontrol.cpp +++ b/src/plugins/multimedia/windows/player/mfvideorenderercontrol.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "mfvideorenderercontrol_p.h" #include "mfactivate_p.h" @@ -43,2103 +7,28 @@ #include "evrcustompresenter_p.h" #include <private/qplatformvideosink_p.h> -#include <private/qabstractvideobuffer_p.h> -#include <private/qwindowsmfdefs_p.h> -#include <qvideosink.h> -#include <qvideoframeformat.h> -#include <qtimer.h> -#include <qmutex.h> -#include <qcoreevent.h> -#include <qcoreapplication.h> -#include <qthread.h> -#include "guiddef.h" -#include <qdebug.h> - -//#define DEBUG_MEDIAFOUNDATION -#define PAD_TO_DWORD(x) (((x) + 3) & ~3) - -namespace -{ - class MediaSampleVideoBuffer : public QAbstractVideoBuffer - { - public: - MediaSampleVideoBuffer(IMFMediaBuffer *buffer, int bytesPerLine) - : QAbstractVideoBuffer(QVideoFrame::NoHandle) - , m_buffer(buffer) - , m_bytesPerLine(bytesPerLine) - , m_mapMode(QVideoFrame::NotMapped) - { - buffer->AddRef(); - } - - ~MediaSampleVideoBuffer() - { - m_buffer->Release(); - } - - MapData map(QVideoFrame::MapMode mode) override - { - MapData mapData; - if (m_mapMode == QVideoFrame::NotMapped && mode != QVideoFrame::NotMapped) { - BYTE *bytes; - DWORD length; - HRESULT hr = m_buffer->Lock(&bytes, NULL, &length); - if (SUCCEEDED(hr)) { - mapData.nPlanes = 1; - mapData.bytesPerLine[0] = m_bytesPerLine; - mapData.data[0] = reinterpret_cast<uchar *>(bytes); - mapData.size[0] = qsizetype(length); - m_mapMode = mode; - } else { - qWarning("Faild to lock mf buffer!"); - } - } - return mapData; - } - - void unmap() override - { - if (m_mapMode == QVideoFrame::NotMapped) - return; - m_mapMode = QVideoFrame::NotMapped; - m_buffer->Unlock(); - } - - QVideoFrame::MapMode mapMode() const override - { - return m_mapMode; - } - - private: - IMFMediaBuffer *m_buffer; - int m_bytesPerLine; - QVideoFrame::MapMode m_mapMode; - }; - - // Custom interface for handling IMFStreamSink::PlaceMarker calls asynchronously. - static const GUID IID_IMarker = {0xa3ff32de, 0x1031, 0x438a, {0x8b, 0x47, 0x82, 0xf8, 0xac, 0xda, 0x59, 0xb7}}; - MIDL_INTERFACE("a3ff32de-1031-438a-8b47-82f8acda59b7") - IMarker : public IUnknown - { - virtual STDMETHODIMP GetMarkerType(MFSTREAMSINK_MARKER_TYPE *pType) = 0; - virtual STDMETHODIMP GetMarkerValue(PROPVARIANT *pvar) = 0; - virtual STDMETHODIMP GetContext(PROPVARIANT *pvar) = 0; - }; - - class Marker : public IMarker - { - public: - static HRESULT Create( - MFSTREAMSINK_MARKER_TYPE eMarkerType, - const PROPVARIANT* pvarMarkerValue, // Can be NULL. - const PROPVARIANT* pvarContextValue, // Can be NULL. - IMarker **ppMarker) - { - if (ppMarker == NULL) - return E_POINTER; - - HRESULT hr = S_OK; - Marker *pMarker = new Marker(eMarkerType); - if (pMarker == NULL) - hr = E_OUTOFMEMORY; - - // Copy the marker data. - if (SUCCEEDED(hr) && pvarMarkerValue) - hr = PropVariantCopy(&pMarker->m_varMarkerValue, pvarMarkerValue); - - if (SUCCEEDED(hr) && pvarContextValue) - hr = PropVariantCopy(&pMarker->m_varContextValue, pvarContextValue); - - if (SUCCEEDED(hr)) { - *ppMarker = pMarker; - (*ppMarker)->AddRef(); - } - - if (pMarker) - pMarker->Release(); - - return hr; - } - - // IUnknown methods. - STDMETHODIMP QueryInterface(REFIID iid, void** ppv) override - { - if (!ppv) - return E_POINTER; - if (iid == IID_IUnknown) { - *ppv = static_cast<IUnknown*>(this); - } else if (iid == IID_IMarker) { - *ppv = static_cast<IMarker*>(this); - } else { - *ppv = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; - } - - STDMETHODIMP_(ULONG) AddRef() override - { - return InterlockedIncrement(&m_cRef); - } - - STDMETHODIMP_(ULONG) Release() override - { - LONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) - delete this; - // For thread safety, return a temporary variable. - return cRef; - } - - STDMETHODIMP GetMarkerType(MFSTREAMSINK_MARKER_TYPE *pType) override - { - if (pType == NULL) - return E_POINTER; - *pType = m_eMarkerType; - return S_OK; - } - - STDMETHODIMP GetMarkerValue(PROPVARIANT *pvar) override - { - if (pvar == NULL) - return E_POINTER; - return PropVariantCopy(pvar, &m_varMarkerValue); - } - - STDMETHODIMP GetContext(PROPVARIANT *pvar) override - { - if (pvar == NULL) - return E_POINTER; - return PropVariantCopy(pvar, &m_varContextValue); - } - - protected: - MFSTREAMSINK_MARKER_TYPE m_eMarkerType; - PROPVARIANT m_varMarkerValue; - PROPVARIANT m_varContextValue; - - private: - long m_cRef = 1; - - Marker(MFSTREAMSINK_MARKER_TYPE eMarkerType) : m_eMarkerType(eMarkerType) - { - PropVariantInit(&m_varMarkerValue); - PropVariantInit(&m_varContextValue); - } - - virtual ~Marker() - { - PropVariantClear(&m_varMarkerValue); - PropVariantClear(&m_varContextValue); - } - }; - - class MediaStream : public QObject, public IMFStreamSink, public IMFMediaTypeHandler - { - Q_OBJECT - friend class MFVideoRendererControl; - public: - static const DWORD DEFAULT_MEDIA_STREAM_ID = 0x0; - - MediaStream(IMFMediaSink *parent, MFVideoRendererControl *rendererControl) - : m_workQueueCB(this, &MediaStream::onDispatchWorkItem) - , m_rendererControl(rendererControl) - { - m_sink = parent; - - if (FAILED(MFCreateEventQueue(&m_eventQueue))) - qWarning("Failed to create mf event queue!"); - if (FAILED(MFAllocateWorkQueue(&m_workQueueId))) - qWarning("Failed to allocated mf work queue!"); - } - - ~MediaStream() - { - Q_ASSERT(m_shutdown); - } - - //from IUnknown - STDMETHODIMP QueryInterface(REFIID riid, void** ppvObject) override - { - if (!ppvObject) - return E_POINTER; - if (riid == IID_IMFStreamSink) { - *ppvObject = static_cast<IMFStreamSink*>(this); - } else if (riid == IID_IMFMediaEventGenerator) { - *ppvObject = static_cast<IMFMediaEventGenerator*>(this); - } else if (riid == IID_IMFMediaTypeHandler) { - *ppvObject = static_cast<IMFMediaTypeHandler*>(this); - } else if (riid == IID_IUnknown) { - *ppvObject = static_cast<IUnknown*>(static_cast<IMFStreamSink*>(this)); - } else { - *ppvObject = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; - } - - STDMETHODIMP_(ULONG) AddRef(void) override - { - return InterlockedIncrement(&m_cRef); - } - - STDMETHODIMP_(ULONG) Release(void) override - { - LONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) - delete this; - // For thread safety, return a temporary variable. - return cRef; - } - - //from IMFMediaEventGenerator - STDMETHODIMP GetEvent( - DWORD dwFlags, - IMFMediaEvent **ppEvent) override - { - // GetEvent can block indefinitely, so we don't hold the lock. - // This requires some juggling with the event queue pointer. - HRESULT hr = S_OK; - IMFMediaEventQueue *queue = NULL; - - m_mutex.lock(); - if (m_shutdown) - hr = MF_E_SHUTDOWN; - if (SUCCEEDED(hr)) { - queue = m_eventQueue; - queue->AddRef(); - } - m_mutex.unlock(); - - // Now get the event. - if (SUCCEEDED(hr)) { - hr = queue->GetEvent(dwFlags, ppEvent); - queue->Release(); - } - - return hr; - } - - STDMETHODIMP BeginGetEvent( - IMFAsyncCallback *pCallback, - IUnknown *punkState) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_eventQueue->BeginGetEvent(pCallback, punkState); - } - - STDMETHODIMP EndGetEvent( - IMFAsyncResult *pResult, - IMFMediaEvent **ppEvent) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_eventQueue->EndGetEvent(pResult, ppEvent); - } - - STDMETHODIMP QueueEvent( - MediaEventType met, - REFGUID guidExtendedType, - HRESULT hrStatus, - const PROPVARIANT *pvValue) override - { -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "MediaStream::QueueEvent" << met; -#endif - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_eventQueue->QueueEventParamVar(met, guidExtendedType, hrStatus, pvValue); - } - - //from IMFStreamSink - STDMETHODIMP GetMediaSink( - IMFMediaSink **ppMediaSink) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - else if (!ppMediaSink) - return E_INVALIDARG; - - m_sink->AddRef(); - *ppMediaSink = m_sink; - return S_OK; - } - - STDMETHODIMP GetIdentifier( - DWORD *pdwIdentifier) override - { - *pdwIdentifier = MediaStream::DEFAULT_MEDIA_STREAM_ID; - return S_OK; - } - - STDMETHODIMP GetMediaTypeHandler( - IMFMediaTypeHandler **ppHandler) override - { - LPVOID handler = NULL; - HRESULT hr = QueryInterface(IID_IMFMediaTypeHandler, &handler); - *ppHandler = (IMFMediaTypeHandler*)(handler); - return hr; - } - - STDMETHODIMP ProcessSample( - IMFSample *pSample) override - { - if (pSample == NULL) - return E_INVALIDARG; - HRESULT hr = S_OK; - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - if (!m_prerolling) { - hr = validateOperation(OpProcessSample); - if (FAILED(hr)) - return hr; - } - - pSample->AddRef(); - m_sampleQueue.push_back(pSample); - - // Unless we are paused, start an async operation to dispatch the next sample. - if (m_state != State_Paused) - hr = queueAsyncOperation(OpProcessSample); - - return hr; - } - - STDMETHODIMP PlaceMarker( - MFSTREAMSINK_MARKER_TYPE eMarkerType, - const PROPVARIANT *pvarMarkerValue, - const PROPVARIANT *pvarContextValue) override - { - HRESULT hr = S_OK; - QMutexLocker locker(&m_mutex); - IMarker *pMarker = NULL; - if (m_shutdown) - return MF_E_SHUTDOWN; - - hr = validateOperation(OpPlaceMarker); - if (FAILED(hr)) - return hr; - - // Create a marker object and put it on the sample queue. - hr = Marker::Create(eMarkerType, pvarMarkerValue, pvarContextValue, &pMarker); - if (FAILED(hr)) - return hr; - - m_sampleQueue.push_back(pMarker); - - // Unless we are paused, start an async operation to dispatch the next sample/marker. - if (m_state != State_Paused) - hr = queueAsyncOperation(OpPlaceMarker); // Increments ref count on pOp. - return hr; - } - - STDMETHODIMP Flush(void) override - { -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "MediaStream::Flush"; -#endif - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - // Note: Even though we are flushing data, we still need to send - // any marker events that were queued. - clearBufferCache(); - return processSamplesFromQueue(DropSamples); - } - - //from IMFMediaTypeHandler - STDMETHODIMP IsMediaTypeSupported( - IMFMediaType *pMediaType, - IMFMediaType **ppMediaType) override - { - if (ppMediaType) - *ppMediaType = NULL; - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - int index = getMediaTypeIndex(pMediaType); - if (index < 0) { - if (ppMediaType && m_mediaTypes.size() > 0) { - *ppMediaType = m_mediaTypes[0]; - (*ppMediaType)->AddRef(); - } - return MF_E_INVALIDMEDIATYPE; - } - - BOOL compressed = TRUE; - pMediaType->IsCompressedFormat(&compressed); - if (compressed) { - if (ppMediaType && (SUCCEEDED(MFCreateMediaType(ppMediaType)))) { - (*ppMediaType)->CopyAllItems(pMediaType); - (*ppMediaType)->SetUINT32(MF_MT_FIXED_SIZE_SAMPLES, TRUE); - (*ppMediaType)->SetUINT32(MF_MT_COMPRESSED, FALSE); - (*ppMediaType)->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); - } - return MF_E_INVALIDMEDIATYPE; - } - - return S_OK; - } - - STDMETHODIMP GetMediaTypeCount( - DWORD *pdwTypeCount) override - { - if (pdwTypeCount == NULL) - return E_INVALIDARG; - QMutexLocker locker(&m_mutex); - *pdwTypeCount = DWORD(m_mediaTypes.size()); - return S_OK; - } - - STDMETHODIMP GetMediaTypeByIndex( - DWORD dwIndex, - IMFMediaType **ppType) override - { - if (ppType == NULL) - return E_INVALIDARG; - HRESULT hr = S_OK; - QMutexLocker locker(&m_mutex); - if (m_shutdown) - hr = MF_E_SHUTDOWN; - - if (SUCCEEDED(hr)) { - if (dwIndex >= DWORD(m_mediaTypes.size())) - hr = MF_E_NO_MORE_TYPES; - } - - if (SUCCEEDED(hr)) { - *ppType = m_mediaTypes[dwIndex]; - (*ppType)->AddRef(); - } - return hr; - } - - STDMETHODIMP SetCurrentMediaType( - IMFMediaType *pMediaType) override - { - HRESULT hr = S_OK; - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - DWORD flag = MF_MEDIATYPE_EQUAL_MAJOR_TYPES | - MF_MEDIATYPE_EQUAL_FORMAT_TYPES | - MF_MEDIATYPE_EQUAL_FORMAT_DATA; - - if (m_currentMediaType && (m_currentMediaType->IsEqual(pMediaType, &flag) == S_OK)) - return S_OK; - - hr = validateOperation(OpSetMediaType); - - if (SUCCEEDED(hr)) { - int index = getMediaTypeIndex(pMediaType); - if (index >= 0) { - UINT64 size; - hr = pMediaType->GetUINT64(MF_MT_FRAME_SIZE, &size); - if (SUCCEEDED(hr)) { - m_currentFormatIndex = index; - int width = int(HI32(size)); - int height = int(LO32(size)); - QVideoFrameFormat format(QSize(width, height), m_pixelFormats[index]); - m_surfaceFormat = format; - - MFVideoArea viewport; - if (SUCCEEDED(pMediaType->GetBlob(MF_MT_GEOMETRIC_APERTURE, - reinterpret_cast<UINT8*>(&viewport), - sizeof(MFVideoArea), - NULL))) { - - m_surfaceFormat.setViewport(QRect(viewport.OffsetX.value, - viewport.OffsetY.value, - viewport.Area.cx, - viewport.Area.cy)); - } - - if (FAILED(pMediaType->GetUINT32(MF_MT_DEFAULT_STRIDE, (UINT32*)&m_bytesPerLine))) { - m_bytesPerLine = getBytesPerLine(format); - } - - m_state = State_Ready; - if (m_currentMediaType) - m_currentMediaType->Release(); - m_currentMediaType = pMediaType; - pMediaType->AddRef(); - } - } else { - hr = MF_E_INVALIDREQUEST; - } - } - return hr; - } - - STDMETHODIMP GetCurrentMediaType( - IMFMediaType **ppMediaType) override - { - if (ppMediaType == NULL) - return E_INVALIDARG; - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - if (m_currentFormatIndex < 0) - return MF_E_NOT_INITIALIZED; - *ppMediaType = m_currentMediaType; - (*ppMediaType)->AddRef(); - return S_OK; - } - - STDMETHODIMP GetMajorType( - GUID *pguidMajorType) override - { - if (pguidMajorType == NULL) - return E_INVALIDARG; - *pguidMajorType = MFMediaType_Video; - return S_OK; - } - - // - void setSink(QVideoSink *sink) - { - m_mutex.lock(); - m_videoSink = sink; - m_mutex.unlock(); - supportedFormatsChanged(); - } - - void setClock(IMFPresentationClock *presentationClock) - { - QMutexLocker locker(&m_mutex); - if (!m_shutdown) { - if (m_presentationClock) - m_presentationClock->Release(); - m_presentationClock = presentationClock; - if (m_presentationClock) - m_presentationClock->AddRef(); - } - } - - void shutdown() - { - QMutexLocker locker(&m_mutex); - Q_ASSERT(!m_shutdown); - - if (m_currentMediaType) { - m_currentMediaType->Release(); - m_currentMediaType = NULL; - m_currentFormatIndex = -1; - } - - if (m_eventQueue) - m_eventQueue->Shutdown(); - - MFUnlockWorkQueue(m_workQueueId); - - if (m_presentationClock) { - m_presentationClock->Release(); - m_presentationClock = NULL; - } - - clearMediaTypes(); - clearSampleQueue(); - clearBufferCache(); - - if (m_eventQueue) { - m_eventQueue->Release(); - m_eventQueue = NULL; - } - - m_shutdown = true; - } - - HRESULT startPreroll(MFTIME hnsUpcomingStartTime) - { - QMutexLocker locker(&m_mutex); - HRESULT hr = validateOperation(OpPreroll); - if (SUCCEEDED(hr)) { - m_state = State_Prerolling; - m_prerollTargetTime = hnsUpcomingStartTime; - hr = queueAsyncOperation(OpPreroll); - } - return hr; - } - - HRESULT finalize(IMFAsyncCallback *pCallback, IUnknown *punkState) - { - QMutexLocker locker(&m_mutex); - HRESULT hr = S_OK; - hr = validateOperation(OpFinalize); - if (SUCCEEDED(hr) && m_finalizeResult != NULL) - hr = MF_E_INVALIDREQUEST; // The operation is already pending. - - // Create and store the async result object. - if (SUCCEEDED(hr)) - hr = MFCreateAsyncResult(NULL, pCallback, punkState, &m_finalizeResult); - - if (SUCCEEDED(hr)) { - m_state = State_Finalized; - hr = queueAsyncOperation(OpFinalize); - } - return hr; - } - - HRESULT start(MFTIME start) - { -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "MediaStream::start" << start; -#endif - HRESULT hr = S_OK; - QMutexLocker locker(&m_mutex); - if (m_rate != 0) - hr = validateOperation(OpStart); - - if (SUCCEEDED(hr)) { - MFTIME sysTime; - if (start != QMM_PRESENTATION_CURRENT_POSITION) - m_startTime = start; // Cache the start time. - else - m_presentationClock->GetCorrelatedTime(0, &m_startTime, &sysTime); - m_state = State_Started; - hr = queueAsyncOperation(OpStart); - } - return hr; - } - - HRESULT restart() - { -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "MediaStream::restart"; -#endif - QMutexLocker locker(&m_mutex); - HRESULT hr = validateOperation(OpRestart); - if (SUCCEEDED(hr)) { - m_state = State_Started; - hr = queueAsyncOperation(OpRestart); - } - return hr; - } - - HRESULT stop() - { -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "MediaStream::stop"; -#endif - QMutexLocker locker(&m_mutex); - HRESULT hr = validateOperation(OpStop); - if (SUCCEEDED(hr)) { - m_state = State_Stopped; - hr = queueAsyncOperation(OpStop); - } - return hr; - } - - HRESULT pause() - { -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "MediaStream::pause"; -#endif - QMutexLocker locker(&m_mutex); - HRESULT hr = validateOperation(OpPause); - if (SUCCEEDED(hr)) { - m_state = State_Paused; - hr = queueAsyncOperation(OpPause); - } - return hr; - } - - HRESULT setRate(float rate) - { -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "MediaStream::setRate" << rate; -#endif - QMutexLocker locker(&m_mutex); - HRESULT hr = validateOperation(OpSetRate); - if (SUCCEEDED(hr)) { - m_rate = rate; - hr = queueAsyncOperation(OpSetRate); - } - return hr; - } - - void supportedFormatsChanged() - { - QMutexLocker locker(&m_mutex); - m_pixelFormats.clear(); - clearMediaTypes(); - if (!m_videoSink) - return; - for (int f = 0; f < QVideoFrameFormat::NPixelFormats; ++f) { - QVideoFrameFormat::PixelFormat format = QVideoFrameFormat::PixelFormat(f); - IMFMediaType *mediaType; - if (FAILED(MFCreateMediaType(&mediaType))) { - qWarning("Failed to create mf media type!"); - continue; - } - mediaType->SetUINT32(MF_MT_FIXED_SIZE_SAMPLES, TRUE); - mediaType->SetUINT32(MF_MT_COMPRESSED, FALSE); - mediaType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); - mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video); - switch (format) { - case QVideoFrameFormat::Format_BGRA8888: - case QVideoFrameFormat::Format_BGRA8888_Premultiplied: - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_ARGB32); - break; - case QVideoFrameFormat::Format_BGRX8888: - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_RGB32); - break; - case QVideoFrameFormat::Format_AYUV: - case QVideoFrameFormat::Format_AYUV_Premultiplied: - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_AYUV); - break; - case QVideoFrameFormat::Format_YUV420P: - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_I420); - break; - case QVideoFrameFormat::Format_UYVY: - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_UYVY); - break; - case QVideoFrameFormat::Format_YV12: - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_YV12); - break; - case QVideoFrameFormat::Format_NV12: - mediaType->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12); - break; - default: - mediaType->Release(); - continue; - } - // #### QAbstractVideoSurface::supportedPixelFormats() returns formats in descending - // order of preference, while IMFMediaTypeHandler is supposed to return supported - // formats in ascending order of preference. We need to reverse the list. - m_pixelFormats.prepend(format); - m_mediaTypes.prepend(mediaType); - } - } - - void present() - { - QMutexLocker locker(&m_mutex); - if (!m_scheduledBuffer) - return; - QVideoFrame frame = QVideoFrame( - new MediaSampleVideoBuffer(m_scheduledBuffer, m_bytesPerLine), m_surfaceFormat); - frame.setStartTime(m_bufferStartTime * 0.1); - frame.setEndTime((m_bufferStartTime + m_bufferDuration) * 0.1); - m_videoSink->platformVideoSink()->setVideoFrame(frame); - m_scheduledBuffer->Release(); - m_scheduledBuffer = NULL; - if (m_rate != 0) - schedulePresentation(true); - } - - void clearScheduledFrame() - { - QMutexLocker locker(&m_mutex); - if (m_scheduledBuffer) { - m_scheduledBuffer->Release(); - m_scheduledBuffer = NULL; - schedulePresentation(true); - } - } - - enum - { - PresentSurface - }; - - class PresentEvent : public QEvent - { - public: - PresentEvent(MFTIME targetTime) - : QEvent(QEvent::Type(PresentSurface)) - , m_time(targetTime) - { - } - - MFTIME targetTime() - { - return m_time; - } - - private: - MFTIME m_time; - }; - - protected: - HRESULT m_startResult; - - private: - enum FlushState - { - DropSamples = 0, - WriteSamples - }; - - // State enum: Defines the current state of the stream. - enum State - { - State_TypeNotSet = 0, // No media type is set - State_Ready, // Media type is set, Start has never been called. - State_Prerolling, - State_Started, - State_Paused, - State_Stopped, - State_WaitForSurfaceStart, - State_Finalized, - State_Count = State_Finalized + 1 // Number of states - }; - - // StreamOperation: Defines various operations that can be performed on the stream. - enum StreamOperation - { - OpSetMediaType = 0, - OpStart, - OpPreroll, - OpRestart, - OpPause, - OpStop, - OpSetRate, - OpProcessSample, - OpPlaceMarker, - OpFinalize, - - Op_Count = OpFinalize + 1 // Number of operations - }; - - // AsyncOperation: - // Used to queue asynchronous operations. When we call MFPutWorkItem, we use this - // object for the callback state (pState). Then, when the callback is invoked, - // we can use the object to determine which asynchronous operation to perform. - class AsyncOperation : public IUnknown - { - public: - AsyncOperation(StreamOperation op) - :m_op(op) - { - } - - StreamOperation m_op; // The operation to perform. - - //from IUnknown - STDMETHODIMP QueryInterface(REFIID iid, void** ppv) override - { - if (!ppv) - return E_POINTER; - if (iid == IID_IUnknown) { - *ppv = static_cast<IUnknown*>(this); - } else { - *ppv = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; - } - STDMETHODIMP_(ULONG) AddRef() override - { - return InterlockedIncrement(&m_cRef); - } - STDMETHODIMP_(ULONG) Release() override - { - ULONG uCount = InterlockedDecrement(&m_cRef); - if (uCount == 0) - delete this; - // For thread safety, return a temporary variable. - return uCount; - } - - private: - long m_cRef = 1; - virtual ~AsyncOperation() - { - Q_ASSERT(m_cRef == 0); - } - }; - - // ValidStateMatrix: Defines a look-up table that says which operations - // are valid from which states. - static BOOL ValidStateMatrix[State_Count][Op_Count]; - - long m_cRef = 1; - QMutex m_mutex; - - IMFMediaType *m_currentMediaType = nullptr; - State m_state = State_TypeNotSet; - IMFMediaSink *m_sink = nullptr; - IMFMediaEventQueue *m_eventQueue = nullptr; - DWORD m_workQueueId = 0; - AsyncCallback<MediaStream> m_workQueueCB; - QList<IUnknown*> m_sampleQueue; - IMFAsyncResult *m_finalizeResult = nullptr; // Result object for Finalize operation. - MFTIME m_startTime = 0; // Presentation time when the clock started. - - bool m_shutdown = false; - QList<IMFMediaType*> m_mediaTypes; - QList<QVideoFrameFormat::PixelFormat> m_pixelFormats; - int m_currentFormatIndex = -1; - int m_bytesPerLine = 0; - QVideoFrameFormat m_surfaceFormat; - QVideoSink *m_videoSink = nullptr; - MFVideoRendererControl *m_rendererControl = nullptr; - - void clearMediaTypes() - { - for (IMFMediaType* mediaType : qAsConst(m_mediaTypes)) - mediaType->Release(); - m_mediaTypes.clear(); - } - - int getMediaTypeIndex(IMFMediaType *mt) - { - GUID majorType; - if (FAILED(mt->GetMajorType(&majorType))) - return -1; - if (majorType != MFMediaType_Video) - return -1; - - GUID subType; - if (FAILED(mt->GetGUID(MF_MT_SUBTYPE, &subType))) - return -1; - - for (int index = 0; index < m_mediaTypes.size(); ++index) { - GUID st; - m_mediaTypes[index]->GetGUID(MF_MT_SUBTYPE, &st); - if (st == subType) - return index; - } - return -1; - } - - int getBytesPerLine(const QVideoFrameFormat &format) - { - switch (format.pixelFormat()) { - // 32 bpp packed formats. - case QVideoFrameFormat::Format_XBGR8888: - case QVideoFrameFormat::Format_BGRX8888: - case QVideoFrameFormat::Format_XRGB8888: - case QVideoFrameFormat::Format_RGBX8888: - case QVideoFrameFormat::Format_AYUV: - return format.frameWidth() * 4; - // 16 bpp packed formats. - case QVideoFrameFormat::Format_YUYV: - case QVideoFrameFormat::Format_UYVY: - return PAD_TO_DWORD(format.frameWidth() * 2); - // Planar formats. - case QVideoFrameFormat::Format_IMC1: - case QVideoFrameFormat::Format_IMC2: - case QVideoFrameFormat::Format_IMC3: - case QVideoFrameFormat::Format_IMC4: - case QVideoFrameFormat::Format_YV12: - case QVideoFrameFormat::Format_NV12: - case QVideoFrameFormat::Format_YUV420P: - return PAD_TO_DWORD(format.frameWidth()); - default: - return 0; - } - } - - // Callback for MFPutWorkItem. - HRESULT onDispatchWorkItem(IMFAsyncResult* pAsyncResult) - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - HRESULT hr = S_OK; - IUnknown *pState = NULL; - hr = pAsyncResult->GetState(&pState); - if (SUCCEEDED(hr)) { - // The state object is an AsncOperation object. - AsyncOperation *pOp = (AsyncOperation*)pState; - StreamOperation op = pOp->m_op; - switch (op) { - case OpStart: - endPreroll(S_FALSE); - schedulePresentation(true); - // fallthrough - case OpRestart: - endPreroll(S_FALSE); - if (SUCCEEDED(hr)) { - // Send MEStreamSinkStarted. - hr = queueEvent(MEStreamSinkStarted, GUID_NULL, hr, NULL); - // Kick things off by requesting samples... - schedulePresentation(true); - // There might be samples queue from earlier (ie, while paused). - if (SUCCEEDED(hr)) - hr = processSamplesFromQueue(WriteSamples); - } - break; - case OpPreroll: - beginPreroll(); - break; - case OpStop: - // Drop samples from queue. - hr = processSamplesFromQueue(DropSamples); - clearBufferCache(); - // Send the event even if the previous call failed. - hr = queueEvent(MEStreamSinkStopped, GUID_NULL, hr, NULL); - break; - case OpPause: - hr = queueEvent(MEStreamSinkPaused, GUID_NULL, hr, NULL); - break; - case OpSetRate: - hr = queueEvent(MEStreamSinkRateChanged, GUID_NULL, S_OK, NULL); - break; - case OpProcessSample: - case OpPlaceMarker: - hr = dispatchProcessSample(pOp); - break; - case OpFinalize: - endPreroll(S_FALSE); - hr = dispatchFinalize(pOp); - break; - default: - break; - } - } - - if (pState) - pState->Release(); - return hr; - } - - - HRESULT queueEvent(MediaEventType met, REFGUID guidExtendedType, HRESULT hrStatus, const PROPVARIANT* pvValue) - { - HRESULT hr = S_OK; - if (m_shutdown) - hr = MF_E_SHUTDOWN; - if (SUCCEEDED(hr)) - hr = m_eventQueue->QueueEventParamVar(met, guidExtendedType, hrStatus, pvValue); - return hr; - } - - HRESULT validateOperation(StreamOperation op) - { - Q_ASSERT(!m_shutdown); - if (ValidStateMatrix[m_state][op]) - return S_OK; - else - return MF_E_INVALIDREQUEST; - } - - HRESULT queueAsyncOperation(StreamOperation op) - { - HRESULT hr = S_OK; - AsyncOperation *asyncOp = new AsyncOperation(op); - if (asyncOp == NULL) - hr = E_OUTOFMEMORY; - - if (SUCCEEDED(hr)) - hr = MFPutWorkItem(m_workQueueId, &m_workQueueCB, asyncOp); - - if (asyncOp) - asyncOp->Release(); - - return hr; - } - - HRESULT processSamplesFromQueue(FlushState bFlushData) - { - HRESULT hr = S_OK; - QList<IUnknown*>::Iterator pos = m_sampleQueue.begin(); - // Enumerate all of the samples/markers in the queue. - while (pos != m_sampleQueue.end()) { - IUnknown *pUnk = NULL; - IMarker *pMarker = NULL; - IMFSample *pSample = NULL; - pUnk = *pos; - // Figure out if this is a marker or a sample. - if (SUCCEEDED(hr)) { - hr = pUnk->QueryInterface(IID_IMarker, (void**)&pMarker); - if (hr == E_NOINTERFACE) - hr = pUnk->QueryInterface(IID_IMFSample, (void**)&pSample); - } - - // Now handle the sample/marker appropriately. - if (SUCCEEDED(hr)) { - if (pMarker) { - hr = sendMarkerEvent(pMarker, bFlushData); - } else { - Q_ASSERT(pSample != NULL); // Not a marker, must be a sample - if (bFlushData == WriteSamples) - hr = processSampleData(pSample); - } - } - if (pMarker) - pMarker->Release(); - if (pSample) - pSample->Release(); - - if (FAILED(hr)) - break; - - pos++; - } - - clearSampleQueue(); - return hr; - } - - void beginPreroll() - { - if (m_prerolling) - return; - m_prerolling = true; - clearSampleQueue(); - clearBufferCache(); - queueEvent(MEStreamSinkRequestSample, GUID_NULL, S_OK, NULL); - } - - void endPreroll(HRESULT hrStatus) - { - if (!m_prerolling) - return; - m_prerolling = false; - queueEvent(MEStreamSinkPrerolled, GUID_NULL, hrStatus, NULL); - } - MFTIME m_prerollTargetTime = 0; - bool m_prerolling = false; - - void clearSampleQueue() { - for (IUnknown* sample : qAsConst(m_sampleQueue)) - sample->Release(); - m_sampleQueue.clear(); - } - - HRESULT sendMarkerEvent(IMarker *pMarker, FlushState FlushState) - { - HRESULT hr = S_OK; - HRESULT hrStatus = S_OK; // Status code for marker event. - if (FlushState == DropSamples) - hrStatus = E_ABORT; - - PROPVARIANT var; - PropVariantInit(&var); - - // Get the context data. - hr = pMarker->GetContext(&var); - - if (SUCCEEDED(hr)) - hr = queueEvent(MEStreamSinkMarker, GUID_NULL, hrStatus, &var); - - PropVariantClear(&var); - return hr; - } - - HRESULT dispatchProcessSample(AsyncOperation* pOp) - { - HRESULT hr = S_OK; - Q_ASSERT(pOp != NULL); - Q_UNUSED(pOp); - hr = processSamplesFromQueue(WriteSamples); - // We are in the middle of an asynchronous operation, so if something failed, send an error. - if (FAILED(hr)) - hr = queueEvent(MEError, GUID_NULL, hr, NULL); - - return hr; - } - - HRESULT dispatchFinalize(AsyncOperation*) - { - HRESULT hr = S_OK; - // Write any samples left in the queue... - hr = processSamplesFromQueue(WriteSamples); - - // Set the async status and invoke the callback. - m_finalizeResult->SetStatus(hr); - hr = MFInvokeCallback(m_finalizeResult); - return hr; - } - - HRESULT processSampleData(IMFSample *pSample) - { - m_sampleRequested = false; - - LONGLONG time, duration = -1; - HRESULT hr = pSample->GetSampleTime(&time); - if (SUCCEEDED(hr)) - pSample->GetSampleDuration(&duration); - - if (m_prerolling) { - if (SUCCEEDED(hr) && ((time - m_prerollTargetTime) * m_rate) >= 0) { - IMFMediaBuffer *pBuffer = NULL; - hr = pSample->ConvertToContiguousBuffer(&pBuffer); - if (SUCCEEDED(hr)) { - SampleBuffer sb; - sb.m_buffer = pBuffer; - sb.m_time = time; - sb.m_duration = duration; - m_bufferCache.push_back(sb); - endPreroll(S_OK); - } - } else { - queueEvent(MEStreamSinkRequestSample, GUID_NULL, S_OK, NULL); - } - } else { - bool requestSample = true; - // If the time stamp is too early, just discard this sample. - if (SUCCEEDED(hr) && ((time - m_startTime) * m_rate) >= 0) { - IMFMediaBuffer *pBuffer = NULL; - hr = pSample->ConvertToContiguousBuffer(&pBuffer); - if (SUCCEEDED(hr)) { - SampleBuffer sb; - sb.m_buffer = pBuffer; - sb.m_time = time; - sb.m_duration = duration; - m_bufferCache.push_back(sb); - } - if (m_rate == 0) - requestSample = false; - } - schedulePresentation(requestSample); - } - return hr; - } - - class SampleBuffer - { - public: - IMFMediaBuffer *m_buffer; - LONGLONG m_time; - LONGLONG m_duration; - }; - QList<SampleBuffer> m_bufferCache; - static const int BUFFER_CACHE_SIZE = 2; - - void clearBufferCache() - { - for (SampleBuffer sb : qAsConst(m_bufferCache)) - sb.m_buffer->Release(); - m_bufferCache.clear(); - - if (m_scheduledBuffer) { - m_scheduledBuffer->Release(); - m_scheduledBuffer = NULL; - } - } - - void schedulePresentation(bool requestSample) - { - if (m_state == State_Paused || m_state == State_Prerolling) - return; - if (!m_scheduledBuffer) { - //get time from presentation time - MFTIME currentTime = m_startTime, sysTime; - bool timeOK = true; - if (m_rate != 0) { - if (FAILED(m_presentationClock->GetCorrelatedTime(0, ¤tTime, &sysTime))) - timeOK = false; - } - while (!m_bufferCache.isEmpty()) { - SampleBuffer sb = m_bufferCache.takeFirst(); - if (timeOK && ((sb.m_time - currentTime) * m_rate) < 0) { - sb.m_buffer->Release(); -#ifdef DEBUG_MEDIAFOUNDATION - qDebug() << "currentPresentTime =" << float(currentTime / 10000) * 0.001f << " and sampleTime is" << float(sb.m_time / 10000) * 0.001f; -#endif - continue; - } - m_scheduledBuffer = sb.m_buffer; - m_bufferStartTime = sb.m_time; - m_bufferDuration = sb.m_duration; - QCoreApplication::postEvent(m_rendererControl, new PresentEvent(sb.m_time)); - if (m_rate == 0) - queueEvent(MEStreamSinkScrubSampleComplete, GUID_NULL, S_OK, NULL); - break; - } - } - if (requestSample && !m_sampleRequested && m_bufferCache.size() < BUFFER_CACHE_SIZE) { - m_sampleRequested = true; - queueEvent(MEStreamSinkRequestSample, GUID_NULL, S_OK, NULL); - } - } - IMFMediaBuffer *m_scheduledBuffer = nullptr; - MFTIME m_bufferStartTime = -1; - MFTIME m_bufferDuration = -1; - IMFPresentationClock *m_presentationClock = nullptr; - bool m_sampleRequested = false; - float m_rate = 1.f; - }; - - BOOL MediaStream::ValidStateMatrix[MediaStream::State_Count][MediaStream::Op_Count] = - { - // States: Operations: - // SetType Start Preroll, Restart Pause Stop SetRate Sample Marker Finalize - /* NotSet */ TRUE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, - - /* Ready */ TRUE, TRUE, TRUE, FALSE, TRUE, TRUE, TRUE, FALSE, TRUE, TRUE, - - /* Prerolling */ TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, - - /* Start */ FALSE, TRUE, TRUE, FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, - - /* Pause */ FALSE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, TRUE, - - /* Stop */ FALSE, TRUE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE, TRUE, TRUE, - - /*WaitForSurfaceStart*/ FALSE, FALSE, TRUE, FALSE, FALSE, TRUE, TRUE, FALSE, FALSE, TRUE, - - /* Final */ FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE, FALSE - - // Note about states: - // 1. OnClockRestart should only be called from paused state. - // 2. While paused, the sink accepts samples but does not process them. - }; - - class MediaSink : public IMFFinalizableMediaSink, - public IMFClockStateSink, - public IMFMediaSinkPreroll, - public IMFGetService, - public IMFRateSupport - { - public: - MediaSink(MFVideoRendererControl *rendererControl) - { - m_stream = new MediaStream(this, rendererControl); - } - - virtual ~MediaSink() - { - Q_ASSERT(m_shutdown); - } - - void setSurface(QVideoSink *surface) - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return; - m_stream->setSink(surface); - } - - void present() - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return; - m_stream->present(); - } - - void clearScheduledFrame() - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return; - m_stream->clearScheduledFrame(); - } - - MFTIME getTime() - { - QMutexLocker locker(&m_mutex); - if (!m_presentationClock) - return 0; - MFTIME time, sysTime; - m_presentationClock->GetCorrelatedTime(0, &time, &sysTime); - return time; - } - - float getPlayRate() - { - QMutexLocker locker(&m_mutex); - return m_playRate; - } - - //from IUnknown - STDMETHODIMP QueryInterface(REFIID riid, void** ppvObject) override - { - if (!ppvObject) - return E_POINTER; - if (riid == IID_IMFMediaSink) { - *ppvObject = static_cast<IMFMediaSink*>(this); - } else if (riid == IID_IMFGetService) { - *ppvObject = static_cast<IMFGetService*>(this); - } else if (riid == IID_IMFMediaSinkPreroll) { - *ppvObject = static_cast<IMFMediaSinkPreroll*>(this); - } else if (riid == IID_IMFClockStateSink) { - *ppvObject = static_cast<IMFClockStateSink*>(this); - } else if (riid == IID_IMFRateSupport) { - *ppvObject = static_cast<IMFRateSupport*>(this); - } else if (riid == IID_IUnknown) { - *ppvObject = static_cast<IUnknown*>(static_cast<IMFFinalizableMediaSink*>(this)); - } else { - *ppvObject = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; - } - - STDMETHODIMP_(ULONG) AddRef(void) override - { - return InterlockedIncrement(&m_cRef); - } - - STDMETHODIMP_(ULONG) Release(void) override - { - LONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) - delete this; - // For thread safety, return a temporary variable. - return cRef; - } - - // IMFGetService methods - STDMETHODIMP GetService(const GUID &guidService, - const IID &riid, - LPVOID *ppvObject) override - { - if (!ppvObject) - return E_POINTER; - - if (guidService != MF_RATE_CONTROL_SERVICE) - return MF_E_UNSUPPORTED_SERVICE; - - return QueryInterface(riid, ppvObject); - } - - //IMFMediaSinkPreroll - STDMETHODIMP NotifyPreroll(MFTIME hnsUpcomingStartTime) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_stream->startPreroll(hnsUpcomingStartTime); - } - - //from IMFFinalizableMediaSink - STDMETHODIMP BeginFinalize(IMFAsyncCallback *pCallback, IUnknown *punkState) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_stream->finalize(pCallback, punkState); - } - - STDMETHODIMP EndFinalize(IMFAsyncResult *pResult) override - { - HRESULT hr = S_OK; - // Return the status code from the async result. - if (pResult == NULL) - hr = E_INVALIDARG; - else - hr = pResult->GetStatus(); - return hr; - } - - //from IMFMediaSink - STDMETHODIMP GetCharacteristics( - DWORD *pdwCharacteristics) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - *pdwCharacteristics = MEDIASINK_FIXED_STREAMS | MEDIASINK_CAN_PREROLL; - return S_OK; - } - - STDMETHODIMP AddStreamSink( - DWORD, - IMFMediaType *, - IMFStreamSink **) override - { - QMutexLocker locker(&m_mutex); - return m_shutdown ? MF_E_SHUTDOWN : MF_E_STREAMSINKS_FIXED; - } - - STDMETHODIMP RemoveStreamSink( - DWORD) override - { - QMutexLocker locker(&m_mutex); - return m_shutdown ? MF_E_SHUTDOWN : MF_E_STREAMSINKS_FIXED; - } - - STDMETHODIMP GetStreamSinkCount( - DWORD *pcStreamSinkCount) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - *pcStreamSinkCount = 1; - return S_OK; - } - - STDMETHODIMP GetStreamSinkByIndex( - DWORD dwIndex, - IMFStreamSink **ppStreamSink) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - if (dwIndex != 0) - return MF_E_INVALIDINDEX; - - *ppStreamSink = m_stream; - m_stream->AddRef(); - return S_OK; - } - - STDMETHODIMP GetStreamSinkById( - DWORD dwStreamSinkIdentifier, - IMFStreamSink **ppStreamSink) override - { - if (ppStreamSink == NULL) - return E_INVALIDARG; - if (dwStreamSinkIdentifier != MediaStream::DEFAULT_MEDIA_STREAM_ID) - return MF_E_INVALIDSTREAMNUMBER; - - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - *ppStreamSink = m_stream; - m_stream->AddRef(); - return S_OK; - } - - STDMETHODIMP SetPresentationClock( - IMFPresentationClock *pPresentationClock) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - if (m_presentationClock) { - m_presentationClock->RemoveClockStateSink(this); - m_presentationClock->Release(); - } - m_presentationClock = pPresentationClock; - if (m_presentationClock) { - m_presentationClock->AddRef(); - m_presentationClock->AddClockStateSink(this); - } - m_stream->setClock(m_presentationClock); - return S_OK; - } - - STDMETHODIMP GetPresentationClock( - IMFPresentationClock **ppPresentationClock) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - *ppPresentationClock = m_presentationClock; - if (m_presentationClock) { - m_presentationClock->AddRef(); - return S_OK; - } - return MF_E_NO_CLOCK; - } - - STDMETHODIMP Shutdown(void) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - - m_stream->shutdown(); - if (m_presentationClock) { - m_presentationClock->Release(); - m_presentationClock = NULL; - } - m_stream->Release(); - m_stream = NULL; - m_shutdown = true; - return S_OK; - } - - // IMFClockStateSink methods - STDMETHODIMP OnClockStart(MFTIME, LONGLONG llClockStartOffset) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_stream->start(llClockStartOffset); - } - - STDMETHODIMP OnClockStop(MFTIME) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_stream->stop(); - } - - STDMETHODIMP OnClockPause(MFTIME) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_stream->pause(); - } - - STDMETHODIMP OnClockRestart(MFTIME) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - return m_stream->restart(); - } - - STDMETHODIMP OnClockSetRate(MFTIME, float flRate) override - { - QMutexLocker locker(&m_mutex); - if (m_shutdown) - return MF_E_SHUTDOWN; - m_playRate = flRate; - return m_stream->setRate(flRate); - } - - // IMFRateSupport methods - STDMETHODIMP GetFastestRate(MFRATE_DIRECTION eDirection, - BOOL fThin, - float *pflRate) override - { - if (!pflRate) - return E_POINTER; - - *pflRate = (fThin ? 8.f : 2.0f) * (eDirection == MFRATE_FORWARD ? 1 : -1) ; - - return S_OK; - } - - STDMETHODIMP GetSlowestRate(MFRATE_DIRECTION eDirection, - BOOL fThin, - float *pflRate) override - { - Q_UNUSED(eDirection); - Q_UNUSED(fThin); - - if (!pflRate) - return E_POINTER; - - // we support any rate - *pflRate = 0.f; - - return S_OK; - } - - STDMETHODIMP IsRateSupported(BOOL fThin, - float flRate, - float *pflNearestSupportedRate) override - { - HRESULT hr = S_OK; - - if (!qFuzzyIsNull(flRate)) { - MFRATE_DIRECTION direction = flRate > 0.f ? MFRATE_FORWARD - : MFRATE_REVERSE; - - float fastestRate = 0.f; - float slowestRate = 0.f; - GetFastestRate(direction, fThin, &fastestRate); - GetSlowestRate(direction, fThin, &slowestRate); - - if (direction == MFRATE_REVERSE) - qSwap(fastestRate, slowestRate); - - if (flRate < slowestRate || flRate > fastestRate) { - hr = MF_E_UNSUPPORTED_RATE; - if (pflNearestSupportedRate) { - *pflNearestSupportedRate = qBound(slowestRate, - flRate, - fastestRate); - } - } - } else if (pflNearestSupportedRate) { - *pflNearestSupportedRate = flRate; - } - - return hr; - } - - private: - long m_cRef = 1; - QMutex m_mutex; - bool m_shutdown = false; - IMFPresentationClock *m_presentationClock = nullptr; - MediaStream *m_stream = nullptr; - float m_playRate = 1; - }; - - class VideoRendererActivate : public IMFActivate - { - public: - VideoRendererActivate(MFVideoRendererControl *rendererControl) - : m_cRef(1) - , m_sink(0) - , m_rendererControl(rendererControl) - , m_attributes(0) - , m_videoSink(0) - { - MFCreateAttributes(&m_attributes, 0); - m_sink = new MediaSink(rendererControl); - } - - virtual ~VideoRendererActivate() - { - m_attributes->Release(); - } - - //from IUnknown - STDMETHODIMP QueryInterface(REFIID riid, void** ppvObject) override - { - if (!ppvObject) - return E_POINTER; - if (riid == IID_IMFActivate) { - *ppvObject = static_cast<IMFActivate*>(this); - } else if (riid == IID_IMFAttributes) { - *ppvObject = static_cast<IMFAttributes*>(this); - } else if (riid == IID_IUnknown) { - *ppvObject = static_cast<IUnknown*>(static_cast<IMFActivate*>(this)); - } else { - *ppvObject = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; - } - - STDMETHODIMP_(ULONG) AddRef(void) override - { - return InterlockedIncrement(&m_cRef); - } - - STDMETHODIMP_(ULONG) Release(void) override - { - LONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) - delete this; - // For thread safety, return a temporary variable. - return cRef; - } - - //from IMFActivate - STDMETHODIMP ActivateObject(REFIID riid, void **ppv) override - { - if (!ppv) - return E_INVALIDARG; - QMutexLocker locker(&m_mutex); - if (!m_sink) { - m_sink = new MediaSink(m_rendererControl); - if (m_videoSink) - m_sink->setSurface(m_videoSink); - } - return m_sink->QueryInterface(riid, ppv); - } - - STDMETHODIMP ShutdownObject(void) override - { - QMutexLocker locker(&m_mutex); - HRESULT hr = S_OK; - if (m_sink) { - hr = m_sink->Shutdown(); - m_sink->Release(); - m_sink = NULL; - } - return hr; - } - - STDMETHODIMP DetachObject(void) override - { - QMutexLocker locker(&m_mutex); - if (m_sink) { - m_sink->Release(); - m_sink = NULL; - } - return S_OK; - } - - //from IMFAttributes - STDMETHODIMP GetItem( - REFGUID guidKey, - PROPVARIANT *pValue) override - { - return m_attributes->GetItem(guidKey, pValue); - } - - STDMETHODIMP GetItemType( - REFGUID guidKey, - MF_ATTRIBUTE_TYPE *pType) override - { - return m_attributes->GetItemType(guidKey, pType); - } - - STDMETHODIMP CompareItem( - REFGUID guidKey, - REFPROPVARIANT Value, - BOOL *pbResult) override - { - return m_attributes->CompareItem(guidKey, Value, pbResult); - } - - STDMETHODIMP Compare( - IMFAttributes *pTheirs, - MF_ATTRIBUTES_MATCH_TYPE MatchType, - BOOL *pbResult) override - { - return m_attributes->Compare(pTheirs, MatchType, pbResult); - } - - STDMETHODIMP GetUINT32( - REFGUID guidKey, - UINT32 *punValue) override - { - return m_attributes->GetUINT32(guidKey, punValue); - } - - STDMETHODIMP GetUINT64( - REFGUID guidKey, - UINT64 *punValue) override - { - return m_attributes->GetUINT64(guidKey, punValue); - } - - STDMETHODIMP GetDouble( - REFGUID guidKey, - double *pfValue) override - { - return m_attributes->GetDouble(guidKey, pfValue); - } - - STDMETHODIMP GetGUID( - REFGUID guidKey, - GUID *pguidValue) override - { - return m_attributes->GetGUID(guidKey, pguidValue); - } - - STDMETHODIMP GetStringLength( - REFGUID guidKey, - UINT32 *pcchLength) override - { - return m_attributes->GetStringLength(guidKey, pcchLength); - } - - STDMETHODIMP GetString( - REFGUID guidKey, - LPWSTR pwszValue, - UINT32 cchBufSize, - UINT32 *pcchLength) override - { - return m_attributes->GetString(guidKey, pwszValue, cchBufSize, pcchLength); - } - - STDMETHODIMP GetAllocatedString( - REFGUID guidKey, - LPWSTR *ppwszValue, - UINT32 *pcchLength) override - { - return m_attributes->GetAllocatedString(guidKey, ppwszValue, pcchLength); - } - - STDMETHODIMP GetBlobSize( - REFGUID guidKey, - UINT32 *pcbBlobSize) override - { - return m_attributes->GetBlobSize(guidKey, pcbBlobSize); - } - - STDMETHODIMP GetBlob( - REFGUID guidKey, - UINT8 *pBuf, - UINT32 cbBufSize, - UINT32 *pcbBlobSize) override - { - return m_attributes->GetBlob(guidKey, pBuf, cbBufSize, pcbBlobSize); - } - - STDMETHODIMP GetAllocatedBlob( - REFGUID guidKey, - UINT8 **ppBuf, - UINT32 *pcbSize) override - { - return m_attributes->GetAllocatedBlob(guidKey, ppBuf, pcbSize); - } - - STDMETHODIMP GetUnknown( - REFGUID guidKey, - REFIID riid, - LPVOID *ppv) override - { - return m_attributes->GetUnknown(guidKey, riid, ppv); - } - - STDMETHODIMP SetItem( - REFGUID guidKey, - REFPROPVARIANT Value) override - { - return m_attributes->SetItem(guidKey, Value); - } - - STDMETHODIMP DeleteItem( - REFGUID guidKey) override - { - return m_attributes->DeleteItem(guidKey); - } - - STDMETHODIMP DeleteAllItems(void) override - { - return m_attributes->DeleteAllItems(); - } - - STDMETHODIMP SetUINT32( - REFGUID guidKey, - UINT32 unValue) override - { - return m_attributes->SetUINT32(guidKey, unValue); - } - - STDMETHODIMP SetUINT64( - REFGUID guidKey, - UINT64 unValue) override - { - return m_attributes->SetUINT64(guidKey, unValue); - } - - STDMETHODIMP SetDouble( - REFGUID guidKey, - double fValue) override - { - return m_attributes->SetDouble(guidKey, fValue); - } - - STDMETHODIMP SetGUID( - REFGUID guidKey, - REFGUID guidValue) override - { - return m_attributes->SetGUID(guidKey, guidValue); - } - - STDMETHODIMP SetString( - REFGUID guidKey, - LPCWSTR wszValue) override - { - return m_attributes->SetString(guidKey, wszValue); - } - - STDMETHODIMP SetBlob( - REFGUID guidKey, - const UINT8 *pBuf, - UINT32 cbBufSize) override - { - return m_attributes->SetBlob(guidKey, pBuf, cbBufSize); - } - - STDMETHODIMP SetUnknown( - REFGUID guidKey, - IUnknown *pUnknown) override - { - return m_attributes->SetUnknown(guidKey, pUnknown); - } - - STDMETHODIMP LockStore(void) override - { - return m_attributes->LockStore(); - } - - STDMETHODIMP UnlockStore(void) override - { - return m_attributes->UnlockStore(); - } - - STDMETHODIMP GetCount( - UINT32 *pcItems) override - { - return m_attributes->GetCount(pcItems); - } - - STDMETHODIMP GetItemByIndex( - UINT32 unIndex, - GUID *pguidKey, - PROPVARIANT *pValue) override - { - return m_attributes->GetItemByIndex(unIndex, pguidKey, pValue); - } - - STDMETHODIMP CopyAllItems( - IMFAttributes *pDest) override - { - return m_attributes->CopyAllItems(pDest); - } - - ///////////////////////////////// - void setSink(QVideoSink *sink) - { - QMutexLocker locker(&m_mutex); - if (m_videoSink == sink) - return; - - m_videoSink = sink; - - if (!m_sink) - return; - m_sink->setSurface(m_videoSink); - } - - void present() - { - QMutexLocker locker(&m_mutex); - if (!m_sink) - return; - m_sink->present(); - } - - void clearScheduledFrame() - { - QMutexLocker locker(&m_mutex); - if (!m_sink) - return; - m_sink->clearScheduledFrame(); - } - - MFTIME getTime() - { - if (m_sink) - return m_sink->getTime(); - return 0; - } - - float getPlayRate() - { - if (m_sink) - return m_sink->getPlayRate(); - return 1; - } - - private: - long m_cRef; - bool m_shutdown; - MediaSink *m_sink; - MFVideoRendererControl *m_rendererControl; - IMFAttributes *m_attributes; - QVideoSink *m_videoSink; - QMutex m_mutex; - }; -} +QT_BEGIN_NAMESPACE class EVRCustomPresenterActivate : public MFAbstractActivate { public: EVRCustomPresenterActivate(QVideoSink *sink); - ~EVRCustomPresenterActivate() - { } STDMETHODIMP ActivateObject(REFIID riid, void **ppv) override; STDMETHODIMP ShutdownObject() override; STDMETHODIMP DetachObject() override; void setSink(QVideoSink *sink); + void setCropRect(QRect cropRect); private: + // Destructor is not public. Caller should call Release. + ~EVRCustomPresenterActivate() override { } + EVRCustomPresenter *m_presenter; QVideoSink *m_videoSink; + QRect m_cropRect; QMutex m_mutex; }; @@ -2151,10 +40,10 @@ MFVideoRendererControl::MFVideoRendererControl(QObject *parent) MFVideoRendererControl::~MFVideoRendererControl() { - clear(); + releaseActivate(); } -void MFVideoRendererControl::clear() +void MFVideoRendererControl::releaseActivate() { if (m_sink) m_sink->platformVideoSink()->setVideoFrame(QVideoFrame()); @@ -2172,82 +61,35 @@ void MFVideoRendererControl::clear() m_currentActivate = NULL; } -void MFVideoRendererControl::releaseActivate() -{ - clear(); -} - -QVideoSink *MFVideoRendererControl::sink() const -{ - return m_sink; -} - void MFVideoRendererControl::setSink(QVideoSink *sink) { m_sink = sink; if (m_presenterActivate) m_presenterActivate->setSink(m_sink); - else if (m_currentActivate) - static_cast<VideoRendererActivate*>(m_currentActivate)->setSink(m_sink); -} - -void MFVideoRendererControl::customEvent(QEvent *event) -{ - if (m_presenterActivate) - return; - - if (!m_currentActivate) - return; - - if (event->type() == QEvent::Type(MediaStream::PresentSurface)) { - MFTIME targetTime = static_cast<MediaStream::PresentEvent*>(event)->targetTime(); - MFTIME currentTime = static_cast<VideoRendererActivate*>(m_currentActivate)->getTime(); - float playRate = static_cast<VideoRendererActivate*>(m_currentActivate)->getPlayRate(); - if (!qFuzzyIsNull(playRate) && targetTime != currentTime) { - // If the scheduled frame is too late, skip it - const int interval = ((targetTime - currentTime) / 10000) / playRate; - if (interval < 0) - static_cast<VideoRendererActivate*>(m_currentActivate)->clearScheduledFrame(); - else - QTimer::singleShot(interval, this, SLOT(present())); - } else { - present(); - } - return; - } - QObject::customEvent(event); } -void MFVideoRendererControl::present() +void MFVideoRendererControl::setCropRect(const QRect &cropRect) { if (m_presenterActivate) - return; - - if (m_currentActivate) - static_cast<VideoRendererActivate*>(m_currentActivate)->present(); + m_presenterActivate->setCropRect(cropRect); } IMFActivate* MFVideoRendererControl::createActivate() { - clear(); + releaseActivate(); if (m_sink) { // Create the EVR media sink, but replace the presenter with our own if (SUCCEEDED(MFCreateVideoRendererActivate(::GetShellWindow(), &m_currentActivate))) { m_presenterActivate = new EVRCustomPresenterActivate(m_sink); m_currentActivate->SetUnknown(MF_ACTIVATE_CUSTOM_VIDEO_PRESENTER_ACTIVATE, m_presenterActivate); - } else { - m_currentActivate = new VideoRendererActivate(this); } } - setSink(m_sink); - return m_currentActivate; } - EVRCustomPresenterActivate::EVRCustomPresenterActivate(QVideoSink *sink) : MFAbstractActivate() , m_presenter(0) @@ -2261,6 +103,7 @@ HRESULT EVRCustomPresenterActivate::ActivateObject(REFIID riid, void **ppv) QMutexLocker locker(&m_mutex); if (!m_presenter) { m_presenter = new EVRCustomPresenter(m_videoSink); + m_presenter->setCropRect(m_cropRect); } return m_presenter->QueryInterface(riid, ppv); } @@ -2294,5 +137,16 @@ void EVRCustomPresenterActivate::setSink(QVideoSink *sink) m_presenter->setSink(sink); } -#include "moc_mfvideorenderercontrol_p.cpp" -#include "mfvideorenderercontrol.moc" +void EVRCustomPresenterActivate::setCropRect(QRect cropRect) +{ + QMutexLocker locker(&m_mutex); + if (m_cropRect == cropRect) + return; + + m_cropRect = cropRect; + + if (m_presenter) + m_presenter->setCropRect(cropRect); +} + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/windows/player/mfvideorenderercontrol_p.h b/src/plugins/multimedia/windows/player/mfvideorenderercontrol_p.h index 9b48803d9..ed5195240 100644 --- a/src/plugins/multimedia/windows/player/mfvideorenderercontrol_p.h +++ b/src/plugins/multimedia/windows/player/mfvideorenderercontrol_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Mobility Components. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef MFVIDEORENDERERCONTROL_H #define MFVIDEORENDERERCONTROL_H @@ -51,45 +15,33 @@ // We mean it. // -#include "qobject.h" -#include <mfapi.h> -#include <mfidl.h> - -QT_USE_NAMESPACE - -class EVRCustomPresenterActivate; +#include <qobject.h> +#include <qpointer.h> +#include <qrect.h> +#include <mfobjects.h> QT_BEGIN_NAMESPACE +class EVRCustomPresenterActivate; class QVideoSink; -QT_END_NAMESPACE class MFVideoRendererControl : public QObject { - Q_OBJECT public: MFVideoRendererControl(QObject *parent = 0); ~MFVideoRendererControl(); - QVideoSink *sink() const; void setSink(QVideoSink *surface); + void setCropRect(const QRect &cropRect); IMFActivate* createActivate(); void releaseActivate(); -protected: - void customEvent(QEvent *event) override; - -private Q_SLOTS: - void present(); - private: - void clear(); - - QVideoSink *m_sink = nullptr; + QPointer<QVideoSink> m_sink; IMFActivate *m_currentActivate = nullptr; - IMFSampleGrabberSinkCallback *m_callback = nullptr; - EVRCustomPresenterActivate *m_presenterActivate = nullptr; }; +QT_END_NAMESPACE + #endif diff --git a/src/plugins/multimedia/windows/player/samplegrabber.cpp b/src/plugins/multimedia/windows/player/samplegrabber.cpp deleted file mode 100644 index 93bec4ad6..000000000 --- a/src/plugins/multimedia/windows/player/samplegrabber.cpp +++ /dev/null @@ -1,174 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#include "samplegrabber_p.h" - -STDMETHODIMP SampleGrabberCallback::QueryInterface(REFIID riid, void** ppv) -{ - if (!ppv) - return E_POINTER; - if (riid == IID_IMFSampleGrabberSinkCallback) { - *ppv = static_cast<IMFSampleGrabberSinkCallback*>(this); - } else if (riid == IID_IMFClockStateSink) { - *ppv = static_cast<IMFClockStateSink*>(this); - } else if (riid == IID_IUnknown) { - *ppv = static_cast<IUnknown*>(this); - } else { - *ppv = NULL; - return E_NOINTERFACE; - } - AddRef(); - return S_OK; -} - -STDMETHODIMP_(ULONG) SampleGrabberCallback::AddRef() -{ - return InterlockedIncrement(&m_cRef); -} - -STDMETHODIMP_(ULONG) SampleGrabberCallback::Release() -{ - ULONG cRef = InterlockedDecrement(&m_cRef); - if (cRef == 0) { - delete this; - } - return cRef; - -} - -// IMFClockStateSink methods. - -STDMETHODIMP SampleGrabberCallback::OnClockStart(MFTIME hnsSystemTime, LONGLONG llClockStartOffset) -{ - Q_UNUSED(hnsSystemTime); - Q_UNUSED(llClockStartOffset); - return S_OK; -} - -STDMETHODIMP SampleGrabberCallback::OnClockStop(MFTIME hnsSystemTime) -{ - Q_UNUSED(hnsSystemTime); - return S_OK; -} - -STDMETHODIMP SampleGrabberCallback::OnClockPause(MFTIME hnsSystemTime) -{ - Q_UNUSED(hnsSystemTime); - return S_OK; -} - -STDMETHODIMP SampleGrabberCallback::OnClockRestart(MFTIME hnsSystemTime) -{ - Q_UNUSED(hnsSystemTime); - return S_OK; -} - -STDMETHODIMP SampleGrabberCallback::OnClockSetRate(MFTIME hnsSystemTime, float flRate) -{ - Q_UNUSED(hnsSystemTime); - Q_UNUSED(flRate); - return S_OK; -} - -// IMFSampleGrabberSink methods. - -STDMETHODIMP SampleGrabberCallback::OnSetPresentationClock(IMFPresentationClock* pClock) -{ - Q_UNUSED(pClock); - return S_OK; -} - -STDMETHODIMP SampleGrabberCallback::OnShutdown() -{ - return S_OK; -} - -//void AudioSampleGrabberCallback::addProbe(MFAudioProbeControl* probe) -//{ -// QMutexLocker locker(&m_audioProbeMutex); - -// if (m_audioProbes.contains(probe)) -// return; - -// m_audioProbes.append(probe); -//} - -//void AudioSampleGrabberCallback::removeProbe(MFAudioProbeControl* probe) -//{ -// QMutexLocker locker(&m_audioProbeMutex); -// m_audioProbes.removeOne(probe); -//} - -void AudioSampleGrabberCallback::setFormat(const QAudioFormat& format) -{ - m_format = format; -} - -STDMETHODIMP AudioSampleGrabberCallback::OnProcessSample(REFGUID guidMajorMediaType, DWORD dwSampleFlags, - LONGLONG llSampleTime, LONGLONG llSampleDuration, const BYTE * pSampleBuffer, - DWORD dwSampleSize) -{ - Q_UNUSED(dwSampleFlags); - Q_UNUSED(llSampleTime); - Q_UNUSED(llSampleDuration); - Q_UNUSED(pSampleBuffer); - Q_UNUSED(dwSampleSize); - - if (guidMajorMediaType != GUID_NULL && guidMajorMediaType != MFMediaType_Audio) - return S_OK; - - QMutexLocker locker(&m_audioProbeMutex); - -// if (m_audioProbes.isEmpty()) - return S_OK; - - // Check if sample has a presentation time - if (llSampleTime == _I64_MAX) { - // Set default QAudioBuffer start time - llSampleTime = -1; - } else { - // WMF uses 100-nanosecond units, Qt uses microseconds - llSampleTime /= 10; - } - -// for (MFAudioProbeControl* probe : qAsConst(m_audioProbes)) -// probe->bufferProbed((const char*)pSampleBuffer, dwSampleSize, m_format, llSampleTime); - - return S_OK; -} diff --git a/src/plugins/multimedia/windows/player/samplegrabber_p.h b/src/plugins/multimedia/windows/player/samplegrabber_p.h deleted file mode 100644 index ad522ade8..000000000 --- a/src/plugins/multimedia/windows/player/samplegrabber_p.h +++ /dev/null @@ -1,103 +0,0 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ - -#ifndef SAMPLEGRABBER_H -#define SAMPLEGRABBER_H - -// -// W A R N I N G -// ------------- -// -// This file is not part of the Qt API. It exists purely as an -// implementation detail. This header file may change from version to -// version without notice, or even be removed. -// -// We mean it. -// - -#include <QtCore/qmutex.h> -#include <QtCore/qlist.h> -#include <QtMultimedia/qaudioformat.h> -#include <mfapi.h> -#include <mfidl.h> - -class SampleGrabberCallback : public IMFSampleGrabberSinkCallback -{ -public: - // IUnknown methods - STDMETHODIMP QueryInterface(REFIID iid, void** ppv) override; - STDMETHODIMP_(ULONG) AddRef() override; - STDMETHODIMP_(ULONG) Release() override; - - // IMFClockStateSink methods - STDMETHODIMP OnClockStart(MFTIME hnsSystemTime, LONGLONG llClockStartOffset) override; - STDMETHODIMP OnClockStop(MFTIME hnsSystemTime) override; - STDMETHODIMP OnClockPause(MFTIME hnsSystemTime) override; - STDMETHODIMP OnClockRestart(MFTIME hnsSystemTime) override; - STDMETHODIMP OnClockSetRate(MFTIME hnsSystemTime, float flRate) override; - - // IMFSampleGrabberSinkCallback methods - STDMETHODIMP OnSetPresentationClock(IMFPresentationClock* pClock) override; - STDMETHODIMP OnShutdown() override; - -protected: - SampleGrabberCallback() : m_cRef(1) {} - -public: - virtual ~SampleGrabberCallback() {} - -private: - long m_cRef; -}; - -class AudioSampleGrabberCallback: public SampleGrabberCallback { -public: - void setFormat(const QAudioFormat& format); - - STDMETHODIMP OnProcessSample(REFGUID guidMajorMediaType, DWORD dwSampleFlags, - LONGLONG llSampleTime, LONGLONG llSampleDuration, const BYTE * pSampleBuffer, - DWORD dwSampleSize) override; - -private: -// QList<MFAudioProbeControl*> m_audioProbes; - QMutex m_audioProbeMutex; - QAudioFormat m_format; -}; - -#endif // SAMPLEGRABBER_H diff --git a/src/plugins/multimedia/windows/qwindowsformatinfo.cpp b/src/plugins/multimedia/windows/qwindowsformatinfo.cpp index 13f87161e..6ef1f7f7f 100644 --- a/src/plugins/multimedia/windows/qwindowsformatinfo.cpp +++ b/src/plugins/multimedia/windows/qwindowsformatinfo.cpp @@ -1,116 +1,85 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsformatinfo_p.h" #include <mfapi.h> #include <mftransform.h> -#include <private/qwindowsiupointer_p.h> +#include <private/qcomptr_p.h> #include <private/qwindowsmultimediautils_p.h> +#include <private/qcomtaskresource_p.h> #include <QtCore/qlist.h> #include <QtCore/qset.h> +#include <QtCore/qhash.h> #include <QtGui/qimagewriter.h> QT_BEGIN_NAMESPACE +namespace { + template<typename T> -static T codecForFormat(GUID format) = delete; +using CheckedCodecs = QHash<QPair<T, QMediaFormat::ConversionMode>, bool>; -template<> -QMediaFormat::AudioCodec codecForFormat(GUID format) +bool isSupportedMFT(const GUID &category, const MFT_REGISTER_TYPE_INFO &type, QMediaFormat::ConversionMode mode) { - return QWindowsMultimediaUtils::codecForAudioFormat(format); + UINT32 count = 0; + IMFActivate **activateArrayRaw = nullptr; + HRESULT hr = MFTEnumEx( + category, + MFT_ENUM_FLAG_ALL, + (mode == QMediaFormat::Encode) ? nullptr : &type, // Input type + (mode == QMediaFormat::Encode) ? &type : nullptr, // Output type + &activateArrayRaw, + &count + ); + + if (FAILED(hr)) + return false; + + QComTaskResource<IMFActivate *[], QComDeleter> activateArray(activateArrayRaw, count); + for (UINT32 i = 0; i < count; ++i) { + ComPtr<IMFTransform> transform; + hr = activateArray[i]->ActivateObject(IID_PPV_ARGS(transform.GetAddressOf())); + if (SUCCEEDED(hr)) + return true; + } + + return false; } -template<> -QMediaFormat::VideoCodec codecForFormat(GUID format) +bool isSupportedCodec(QMediaFormat::AudioCodec codec, QMediaFormat::ConversionMode mode) { - return QWindowsMultimediaUtils::codecForVideoFormat(format); + return isSupportedMFT((mode == QMediaFormat::Encode) ? MFT_CATEGORY_AUDIO_ENCODER : MFT_CATEGORY_AUDIO_DECODER, + { MFMediaType_Audio, QWindowsMultimediaUtils::audioFormatForCodec(codec) }, + mode); } -template<typename T> -static QSet<T> getCodecSet(GUID category) +bool isSupportedCodec(QMediaFormat::VideoCodec codec, QMediaFormat::ConversionMode mode) { - QSet<T> codecSet; - IMFActivate **activateArray = nullptr; - UINT32 num = 0; - - HRESULT hr = MFTEnumEx(category, MFT_ENUM_FLAG_ALL, nullptr, nullptr, &activateArray, &num); - - if (SUCCEEDED(hr)) { - for (UINT32 i = 0; i < num; ++i) { - QWindowsIUPointer<IMFTransform> transform; - UINT32 typeIndex = 0; - - hr = activateArray[i]->ActivateObject(IID_PPV_ARGS(transform.address())); - - while (SUCCEEDED(hr)) { - QWindowsIUPointer<IMFMediaType> mediaType; - - if (category == MFT_CATEGORY_AUDIO_ENCODER || category == MFT_CATEGORY_VIDEO_ENCODER) - hr = transform->GetOutputAvailableType(0, typeIndex++, mediaType.address()); - else - hr = transform->GetInputAvailableType(0, typeIndex++, mediaType.address()); - - if (SUCCEEDED(hr)) { - GUID subtype = GUID_NULL; - hr = mediaType->GetGUID(MF_MT_SUBTYPE, &subtype); - if (SUCCEEDED(hr)) - codecSet.insert(codecForFormat<T>(subtype)); - } - } - } + return isSupportedMFT((mode == QMediaFormat::Encode) ? MFT_CATEGORY_VIDEO_ENCODER : MFT_CATEGORY_VIDEO_DECODER, + { MFMediaType_Video, QWindowsMultimediaUtils::videoFormatForCodec(codec) }, + mode); +} + +template <typename T> +bool isSupportedCodec(T codec, QMediaFormat::ConversionMode m, CheckedCodecs<T> &checkedCodecs) +{ + if (auto it = checkedCodecs.constFind(qMakePair(codec, m)); it != checkedCodecs.constEnd()) + return it.value(); - for (UINT32 i = 0; i < num; ++i) - activateArray[i]->Release(); + const bool supported = isSupportedCodec(codec, m); - CoTaskMemFree(activateArray); - } + checkedCodecs.insert(qMakePair(codec, m), supported); + return supported; +} - return codecSet; } static QList<QImageCapture::FileFormat> getImageFormatList() { QList<QImageCapture::FileFormat> list; - auto formats = QImageWriter::supportedImageFormats(); + const auto formats = QImageWriter::supportedImageFormats(); for (const auto &f : formats) { auto format = QString::fromUtf8(f); @@ -185,31 +154,25 @@ QWindowsFormatInfo::QWindowsFormatInfo() QMediaFormat::WMV, }; - const auto audioDecoders = getCodecSet<QMediaFormat::AudioCodec>(MFT_CATEGORY_AUDIO_DECODER); - const auto audioEncoders = getCodecSet<QMediaFormat::AudioCodec>(MFT_CATEGORY_AUDIO_ENCODER); - const auto videoDecoders = getCodecSet<QMediaFormat::VideoCodec>(MFT_CATEGORY_VIDEO_DECODER); - const auto videoEncoders = getCodecSet<QMediaFormat::VideoCodec>(MFT_CATEGORY_VIDEO_ENCODER); + CheckedCodecs<QMediaFormat::AudioCodec> checkedAudioCodecs; + CheckedCodecs<QMediaFormat::VideoCodec> checkedVideoCodecs; - for (const auto &codecMap : containerTable) { - - const QSet<QMediaFormat::AudioCodec> mapAudioSet(codecMap.audio.cbegin(), codecMap.audio.cend()); - const QSet<QMediaFormat::VideoCodec> mapVideoSet(codecMap.video.cbegin(), codecMap.video.cend()); + auto ensureCodecs = [&] (CodecMap &codecs, QMediaFormat::ConversionMode mode) { + codecs.audio.removeIf([&] (auto codec) { return !isSupportedCodec(codec, mode, checkedAudioCodecs); }); + codecs.video.removeIf([&] (auto codec) { return !isSupportedCodec(codec, mode, checkedVideoCodecs); }); + return !codecs.video.empty() || !codecs.audio.empty(); + }; + for (const auto &codecMap : containerTable) { if (decoderFormats.contains(codecMap.format)) { - CodecMap m; - m.format = codecMap.format; - m.audio = (audioDecoders & mapAudioSet).values(); - m.video = (videoDecoders & mapVideoSet).values(); - if (!m.video.empty() || !m.audio.empty()) + auto m = codecMap; + if (ensureCodecs(m, QMediaFormat::Decode)) decoders.append(m); } if (encoderFormats.contains(codecMap.format)) { - CodecMap m; - m.format = codecMap.format; - m.audio = (audioEncoders & mapAudioSet).values(); - m.video = (videoEncoders & mapVideoSet).values(); - if (!m.video.empty() || !m.audio.empty()) + auto m = codecMap; + if (ensureCodecs(m, QMediaFormat::Encode)) encoders.append(m); } } diff --git a/src/plugins/multimedia/windows/qwindowsformatinfo_p.h b/src/plugins/multimedia/windows/qwindowsformatinfo_p.h index eeca80a6e..31e6dd986 100644 --- a/src/plugins/multimedia/windows/qwindowsformatinfo_p.h +++ b/src/plugins/multimedia/windows/qwindowsformatinfo_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSFORMATSINFO_H #define QWINDOWSFORMATSINFO_H diff --git a/src/plugins/multimedia/windows/qwindowsintegration.cpp b/src/plugins/multimedia/windows/qwindowsintegration.cpp index d9238ac1c..1053f3c95 100644 --- a/src/plugins/multimedia/windows/qwindowsintegration.cpp +++ b/src/plugins/multimedia/windows/qwindowsintegration.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsintegration_p.h" #include <private/qwindowsmediadevices_p.h> @@ -63,66 +27,66 @@ public: QPlatformMediaIntegration* create(const QString &name) override { - if (name == QLatin1String("windows")) + if (name == u"windows") return new QWindowsMediaIntegration; return nullptr; } }; QWindowsMediaIntegration::QWindowsMediaIntegration() + : QPlatformMediaIntegration(QLatin1String("windows")) { CoInitialize(NULL); MFStartup(MF_VERSION); - - m_videoDevices = new QWindowsVideoDevices(this); } QWindowsMediaIntegration::~QWindowsMediaIntegration() { - delete m_formatInfo; - MFShutdown(); CoUninitialize(); } -QPlatformMediaFormatInfo *QWindowsMediaIntegration::formatInfo() +QPlatformMediaFormatInfo *QWindowsMediaIntegration::createFormatInfo() +{ + return new QWindowsFormatInfo(); +} + +QPlatformVideoDevices *QWindowsMediaIntegration::createVideoDevices() { - if (!m_formatInfo) - m_formatInfo = new QWindowsFormatInfo(); - return m_formatInfo; + return new QWindowsVideoDevices(this); } -QPlatformMediaCaptureSession *QWindowsMediaIntegration::createCaptureSession() +QMaybe<QPlatformMediaCaptureSession *> QWindowsMediaIntegration::createCaptureSession() { return new QWindowsMediaCaptureService(); } -QPlatformAudioDecoder *QWindowsMediaIntegration::createAudioDecoder(QAudioDecoder *decoder) +QMaybe<QPlatformAudioDecoder *> QWindowsMediaIntegration::createAudioDecoder(QAudioDecoder *decoder) { return new MFAudioDecoderControl(decoder); } -QPlatformMediaPlayer *QWindowsMediaIntegration::createPlayer(QMediaPlayer *parent) +QMaybe<QPlatformMediaPlayer *> QWindowsMediaIntegration::createPlayer(QMediaPlayer *parent) { return new MFPlayerControl(parent); } -QPlatformCamera *QWindowsMediaIntegration::createCamera(QCamera *camera) +QMaybe<QPlatformCamera *> QWindowsMediaIntegration::createCamera(QCamera *camera) { return new QWindowsCamera(camera); } -QPlatformMediaRecorder *QWindowsMediaIntegration::createRecorder(QMediaRecorder *recorder) +QMaybe<QPlatformMediaRecorder *> QWindowsMediaIntegration::createRecorder(QMediaRecorder *recorder) { return new QWindowsMediaEncoder(recorder); } -QPlatformImageCapture *QWindowsMediaIntegration::createImageCapture(QImageCapture *imageCapture) +QMaybe<QPlatformImageCapture *> QWindowsMediaIntegration::createImageCapture(QImageCapture *imageCapture) { return new QWindowsImageCapture(imageCapture); } -QPlatformVideoSink *QWindowsMediaIntegration::createVideoSink(QVideoSink *sink) +QMaybe<QPlatformVideoSink *> QWindowsMediaIntegration::createVideoSink(QVideoSink *sink) { return new MFEvrVideoWindowControl(sink); } diff --git a/src/plugins/multimedia/windows/qwindowsintegration_p.h b/src/plugins/multimedia/windows/qwindowsintegration_p.h index a685298ab..29498fa42 100644 --- a/src/plugins/multimedia/windows/qwindowsintegration_p.h +++ b/src/plugins/multimedia/windows/qwindowsintegration_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2021 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSINTEGRATION_H #define QWINDOWSINTEGRATION_H @@ -61,23 +25,25 @@ class QWindowsFormatInfo; class QWindowsMediaIntegration : public QPlatformMediaIntegration { + Q_OBJECT public: QWindowsMediaIntegration(); ~QWindowsMediaIntegration(); - QPlatformMediaFormatInfo *formatInfo() override; + QMaybe<QPlatformMediaCaptureSession *> createCaptureSession() override; - QPlatformMediaCaptureSession *createCaptureSession() override; + QMaybe<QPlatformAudioDecoder *> createAudioDecoder(QAudioDecoder *decoder) override; + QMaybe<QPlatformMediaPlayer *> createPlayer(QMediaPlayer *parent) override; + QMaybe<QPlatformCamera *> createCamera(QCamera *camera) override; + QMaybe<QPlatformMediaRecorder *> createRecorder(QMediaRecorder *recorder) override; + QMaybe<QPlatformImageCapture *> createImageCapture(QImageCapture *imageCapture) override; - QPlatformAudioDecoder *createAudioDecoder(QAudioDecoder *decoder) override; - QPlatformMediaPlayer *createPlayer(QMediaPlayer *parent) override; - QPlatformCamera *createCamera(QCamera *camera) override; - QPlatformMediaRecorder *createRecorder(QMediaRecorder *recorder) override; - QPlatformImageCapture *createImageCapture(QImageCapture *imageCapture) override; + QMaybe<QPlatformVideoSink *> createVideoSink(QVideoSink *sink) override; - QPlatformVideoSink *createVideoSink(QVideoSink *sink) override; +protected: + QPlatformMediaFormatInfo *createFormatInfo() override; - QWindowsFormatInfo *m_formatInfo = nullptr; + QPlatformVideoDevices *createVideoDevices() override; }; QT_END_NAMESPACE diff --git a/src/plugins/multimedia/windows/qwindowsvideodevices.cpp b/src/plugins/multimedia/windows/qwindowsvideodevices.cpp index e35ab3597..8e5081d3b 100644 --- a/src/plugins/multimedia/windows/qwindowsvideodevices.cpp +++ b/src/plugins/multimedia/windows/qwindowsvideodevices.cpp @@ -1,53 +1,21 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qwindowsvideodevices_p.h" #include <private/qcameradevice_p.h> #include <private/qwindowsmfdefs_p.h> #include <private/qwindowsmultimediautils_p.h> +#include <private/qcomptr_p.h> +#include <private/qcomtaskresource_p.h> -#include <Dbt.h> +#include <dbt.h> #include <mfapi.h> #include <mfreadwrite.h> -#include <Mferror.h> +#include <mferror.h> + +QT_BEGIN_NAMESPACE LRESULT QT_WIN_CALLBACK deviceNotificationWndProc(HWND hWnd, UINT message, WPARAM wParam, LPARAM lParam) { @@ -57,7 +25,7 @@ LRESULT QT_WIN_CALLBACK deviceNotificationWndProc(HWND hWnd, UINT message, WPARA auto wmd = reinterpret_cast<QWindowsVideoDevices *>(GetWindowLongPtr(hWnd, GWLP_USERDATA)); if (wmd) { if (wParam == DBT_DEVICEARRIVAL || wParam == DBT_DEVICEREMOVECOMPLETE) { - wmd->videoInputsChanged(); + emit wmd->videoInputsChanged(); } } } @@ -92,6 +60,8 @@ static HWND createMessageOnlyWindow() QWindowsVideoDevices::QWindowsVideoDevices(QPlatformMediaIntegration *integration) : QPlatformVideoDevices(integration) { + CoInitialize(nullptr); + m_videoDeviceMsgWindow = createMessageOnlyWindow(); if (m_videoDeviceMsgWindow) { SetWindowLongPtr(m_videoDeviceMsgWindow, GWLP_USERDATA, (LONG_PTR)this); @@ -126,127 +96,133 @@ QWindowsVideoDevices::~QWindowsVideoDevices() DestroyWindow(m_videoDeviceMsgWindow); UnregisterClass(windowClassName, GetModuleHandle(nullptr)); } + + CoUninitialize(); } -QList<QCameraDevice> QWindowsVideoDevices::videoDevices() const +static std::optional<QCameraFormat> createCameraFormat(IMFMediaType *mediaFormat) { - QList<QCameraDevice> cameras; + GUID subtype = GUID_NULL; + if (FAILED(mediaFormat->GetGUID(MF_MT_SUBTYPE, &subtype))) + return {}; - IMFAttributes *pAttributes = NULL; - IMFActivate **ppDevices = NULL; + auto pixelFormat = QWindowsMultimediaUtils::pixelFormatFromMediaSubtype(subtype); + if (pixelFormat == QVideoFrameFormat::Format_Invalid) + return {}; - // Create an attribute store to specify the enumeration parameters. - HRESULT hr = MFCreateAttributes(&pAttributes, 1); + UINT32 width = 0u; + UINT32 height = 0u; + if (FAILED(MFGetAttributeSize(mediaFormat, MF_MT_FRAME_SIZE, &width, &height))) + return {}; + QSize resolution{ int(width), int(height) }; + + UINT32 num = 0u; + UINT32 den = 0u; + float minFr = 0.f; + float maxFr = 0.f; + + if (SUCCEEDED(MFGetAttributeRatio(mediaFormat, MF_MT_FRAME_RATE_RANGE_MIN, &num, &den))) + minFr = float(num) / float(den); + + if (SUCCEEDED(MFGetAttributeRatio(mediaFormat, MF_MT_FRAME_RATE_RANGE_MAX, &num, &den))) + maxFr = float(num) / float(den); + + auto *f = new QCameraFormatPrivate{ QSharedData(), pixelFormat, resolution, minFr, maxFr }; + return f->create(); +} + +static QString getString(IMFActivate *device, const IID &id) +{ + QComTaskResource<WCHAR> str; + UINT32 length = 0; + HRESULT hr = device->GetAllocatedString(id, str.address(), &length); if (SUCCEEDED(hr)) { - // Source type: video capture devices - hr = pAttributes->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, - MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID); - - if (SUCCEEDED(hr)) { - // Enumerate devices. - UINT32 count; - hr = MFEnumDeviceSources(pAttributes, &ppDevices, &count); - if (SUCCEEDED(hr)) { - // Iterate through devices. - for (int index = 0; index < int(count); index++) { - QCameraDevicePrivate *info = new QCameraDevicePrivate; - - IMFMediaSource *pSource = NULL; - IMFSourceReader *reader = NULL; - - WCHAR *deviceName = NULL; - UINT32 deviceNameLength = 0; - UINT32 deviceIdLength = 0; - WCHAR *deviceId = NULL; - - hr = ppDevices[index]->GetAllocatedString(MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME, - &deviceName, &deviceNameLength); - if (SUCCEEDED(hr)) - info->description = QString::fromWCharArray(deviceName); - CoTaskMemFree(deviceName); - - hr = ppDevices[index]->GetAllocatedString( - MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK, &deviceId, - &deviceIdLength); - if (SUCCEEDED(hr)) - info->id = QString::fromWCharArray(deviceId).toUtf8(); - CoTaskMemFree(deviceId); - - // Create the media source object. - ppDevices[index]->ActivateObject(IID_PPV_ARGS(&pSource)); - // Create the media source reader. - hr = MFCreateSourceReaderFromMediaSource(pSource, NULL, &reader); - if (SUCCEEDED(hr)) { - QList<QSize> photoResolutions; - QList<QCameraFormat> videoFormats; - - DWORD dwMediaTypeIndex = 0; - IMFMediaType *mediaFormat = NULL; - GUID subtype = GUID_NULL; - HRESULT mediaFormatResult = S_OK; - - UINT32 frameRateMin = 0u; - UINT32 frameRateMax = 0u; - UINT32 denominator = 0u; - UINT32 width = 0u; - UINT32 height = 0u; - - while (SUCCEEDED(mediaFormatResult)) { - // Loop through the supported formats for the video device - mediaFormatResult = reader->GetNativeMediaType( - (DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, dwMediaTypeIndex, - &mediaFormat); - if (mediaFormatResult == MF_E_NO_MORE_TYPES) - break; - else if (SUCCEEDED(mediaFormatResult)) { - QVideoFrameFormat::PixelFormat pixelFormat = QVideoFrameFormat::Format_Invalid; - QSize resolution; - float minFr = .0; - float maxFr = .0; - - if (SUCCEEDED(mediaFormat->GetGUID(MF_MT_SUBTYPE, &subtype))) - pixelFormat = QWindowsMultimediaUtils::pixelFormatFromMediaSubtype(subtype); - - if (SUCCEEDED(MFGetAttributeSize(mediaFormat, MF_MT_FRAME_SIZE, &width, - &height))) { - resolution.rheight() = (int)height; - resolution.rwidth() = (int)width; - photoResolutions << resolution; - } - - if (SUCCEEDED(MFGetAttributeRatio(mediaFormat, MF_MT_FRAME_RATE_RANGE_MIN, - &frameRateMin, &denominator))) - minFr = qreal(frameRateMin) / denominator; - if (SUCCEEDED(MFGetAttributeRatio(mediaFormat, MF_MT_FRAME_RATE_RANGE_MAX, - &frameRateMax, &denominator))) - maxFr = qreal(frameRateMax) / denominator; - - auto *f = new QCameraFormatPrivate { QSharedData(), pixelFormat, - resolution, minFr, maxFr }; - videoFormats << f->create(); - } - ++dwMediaTypeIndex; - } - if (mediaFormat) - mediaFormat->Release(); - - info->videoFormats = videoFormats; - info->photoResolutions = photoResolutions; - } - if (reader) - reader->Release(); - cameras.append(info->create()); - } - } - for (DWORD i = 0; i < count; i++) { - if (ppDevices[i]) - ppDevices[i]->Release(); + return QString::fromWCharArray(str.get()); + } else { + return {}; + } +} + +static std::optional<QCameraDevice> createCameraDevice(IMFActivate *device) +{ + auto info = std::make_unique<QCameraDevicePrivate>(); + info->description = getString(device, MF_DEVSOURCE_ATTRIBUTE_FRIENDLY_NAME); + info->id = getString(device, MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_SYMBOLIC_LINK).toUtf8(); + + IMFMediaSource *source = NULL; + HRESULT hr = device->ActivateObject(IID_PPV_ARGS(&source)); + if (FAILED(hr)) + return {}; + + ComPtr<IMFSourceReader> reader; + hr = MFCreateSourceReaderFromMediaSource(source, NULL, reader.GetAddressOf()); + if (FAILED(hr)) + return {}; + + QList<QSize> photoResolutions; + QList<QCameraFormat> videoFormats; + for (DWORD i = 0;; ++i) { + // Loop through the supported formats for the video device + ComPtr<IMFMediaType> mediaFormat; + hr = reader->GetNativeMediaType((DWORD)MF_SOURCE_READER_FIRST_VIDEO_STREAM, i, + mediaFormat.GetAddressOf()); + if (FAILED(hr)) + break; + + auto maybeCamera = createCameraFormat(mediaFormat.Get()); + if (maybeCamera) { + videoFormats << *maybeCamera; + photoResolutions << maybeCamera->resolution(); + } + } + + info->videoFormats = videoFormats; + info->photoResolutions = photoResolutions; + return info.release()->create(); +} + +static QList<QCameraDevice> readCameraDevices(IMFAttributes *attr) +{ + QList<QCameraDevice> cameras; + UINT32 count = 0; + IMFActivate **devicesRaw = nullptr; + HRESULT hr = MFEnumDeviceSources(attr, &devicesRaw, &count); + if (SUCCEEDED(hr)) { + QComTaskResource<IMFActivate *[], QComDeleter> devices(devicesRaw, count); + + for (UINT32 i = 0; i < count; i++) { + IMFActivate *device = devices[i]; + if (device) { + auto maybeCamera = createCameraDevice(device); + if (maybeCamera) + cameras << *maybeCamera; } - CoTaskMemFree(ppDevices); } } - if (pAttributes) - pAttributes->Release(); + return cameras; +} + +QList<QCameraDevice> QWindowsVideoDevices::videoDevices() const +{ + QList<QCameraDevice> cameras; + + ComPtr<IMFAttributes> attr; + HRESULT hr = MFCreateAttributes(attr.GetAddressOf(), 2); + if (FAILED(hr)) + return {}; + + hr = attr->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE, + MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID); + if (SUCCEEDED(hr)) { + cameras << readCameraDevices(attr.Get()); + + hr = attr->SetGUID(MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_CATEGORY, + QMM_KSCATEGORY_SENSOR_CAMERA); + if (SUCCEEDED(hr)) + cameras << readCameraDevices(attr.Get()); + } return cameras; } + +QT_END_NAMESPACE diff --git a/src/plugins/multimedia/windows/qwindowsvideodevices_p.h b/src/plugins/multimedia/windows/qwindowsvideodevices_p.h index 10d2eabc5..f8f5ed920 100644 --- a/src/plugins/multimedia/windows/qwindowsvideodevices_p.h +++ b/src/plugins/multimedia/windows/qwindowsvideodevices_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2022 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QWINDOWSVIDEODEVICES_H #define QWINDOWSVIDEODEVICES_H diff --git a/src/plugins/multimedia/windows/sourceresolver.cpp b/src/plugins/multimedia/windows/sourceresolver.cpp index 93af15a74..52fb024be 100644 --- a/src/plugins/multimedia/windows/sourceresolver.cpp +++ b/src/plugins/multimedia/windows/sourceresolver.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "mfstream_p.h" #include "sourceresolver_p.h" @@ -45,6 +9,8 @@ #include <QtCore/qdebug.h> #include <QtMultimedia/qmediaplayer.h> +QT_BEGIN_NAMESPACE + /* SourceResolver is separated from MFPlayerSession to handle the work of resolving a media source asynchronously. You call SourceResolver::load to request resolving a media source asynchronously, @@ -323,3 +289,6 @@ bool SourceResolver::State::fromStream() const return m_fromStream; } +QT_END_NAMESPACE + +#include "moc_sourceresolver_p.cpp" diff --git a/src/plugins/multimedia/windows/sourceresolver_p.h b/src/plugins/multimedia/windows/sourceresolver_p.h index 0aab4cc19..57ac6fc9d 100644 --- a/src/plugins/multimedia/windows/sourceresolver_p.h +++ b/src/plugins/multimedia/windows/sourceresolver_p.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 The Qt Company Ltd. -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef SOURCERESOLVER_H #define SOURCERESOLVER_H @@ -54,6 +18,8 @@ #include "mfstream_p.h" #include <QUrl> +QT_BEGIN_NAMESPACE + class SourceResolver: public QObject, public IMFAsyncCallback { Q_OBJECT @@ -112,4 +78,6 @@ private: QMutex m_mutex; }; +QT_END_NAMESPACE + #endif diff --git a/src/plugins/videonode/CMakeLists.txt b/src/plugins/videonode/CMakeLists.txt index 8b19f7f1f..176628276 100644 --- a/src/plugins/videonode/CMakeLists.txt +++ b/src/plugins/videonode/CMakeLists.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + # Generated from videonode.pro. if(QT_FEATURE_gpu_vivante) diff --git a/src/plugins/videonode/imx6/CMakeLists.txt b/src/plugins/videonode/imx6/CMakeLists.txt index 2a959db97..659f73469 100644 --- a/src/plugins/videonode/imx6/CMakeLists.txt +++ b/src/plugins/videonode/imx6/CMakeLists.txt @@ -1,3 +1,6 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + # Generated from imx6.pro. ##################################################################### diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp index 4e8a4e5d2..599dbb9e2 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp +++ b/src/plugins/videonode/imx6/qsgvivantevideomaterial.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <GLES2/gl2.h> #include <GLES2/gl2ext.h> @@ -44,6 +8,7 @@ #include "qsgvivantevideomaterialshader.h" #include "qsgvivantevideonode.h" #include "private/qsgvideotexture_p.h" +#include "private/qvideoframe_p.h" #include <QOpenGLContext> #include <QThread> @@ -52,7 +17,9 @@ #include <QtMultimedia/private/qtmultimediaglobal_p.h> #include "private/qgstvideobuffer_p.h" +#if GST_CHECK_VERSION(1,14,0) #include <gst/allocators/gstphysmemory.h> +#endif //#define QT_VIVANTE_VIDEO_DEBUG @@ -176,7 +143,7 @@ GLuint QSGVivanteVideoMaterial::vivanteMapping(QVideoFrame vF) clearTextures(); } - if (vF.map(QVideoFrame::ReadOnly)) { + if (vF.map(QtVideo::MapMode::ReadOnly)) { if (mMappable) { if (!mBitsToTextureMap.contains(vF.bits())) { @@ -225,7 +192,7 @@ GLuint QSGVivanteVideoMaterial::vivanteMapping(QVideoFrame vF) GLuint physical = ~0U; #if GST_CHECK_VERSION(1,14,0) - auto buffer = reinterpret_cast<QGstVideoBuffer *>(vF.buffer()); + auto buffer = reinterpret_cast<QGstVideoBuffer *>(QVideoFramePrivate::buffer(vF)); auto mem = gst_buffer_peek_memory(buffer->buffer(), 0); auto phys_addr = gst_is_phys_memory(mem) ? gst_phys_memory_get_phys_addr(mem) : 0; if (phys_addr) diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterial.h b/src/plugins/videonode/imx6/qsgvivantevideomaterial.h index 36397eaa9..0c3976e3a 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideomaterial.h +++ b/src/plugins/videonode/imx6/qsgvivantevideomaterial.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QSGVIDEOMATERIAL_VIVMAP_H #define QSGVIDEOMATERIAL_VIVMAP_H diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.cpp b/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.cpp index 9e529c3df..ed78192b6 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.cpp +++ b/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qsgvivantevideomaterialshader.h" #include "qsgvivantevideonode.h" diff --git a/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.h b/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.h index b0e19d28e..73edff7d2 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.h +++ b/src/plugins/videonode/imx6/qsgvivantevideomaterialshader.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QSGVIDEOMATERIALSHADER_VIVANTE_H #define QSGVIDEOMATERIALSHADER_VIVANTE_H diff --git a/src/plugins/videonode/imx6/qsgvivantevideonode.cpp b/src/plugins/videonode/imx6/qsgvivantevideonode.cpp index b879c1435..a800c0e62 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideonode.cpp +++ b/src/plugins/videonode/imx6/qsgvivantevideonode.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include <GLES2/gl2.h> #include <GLES2/gl2ext.h> diff --git a/src/plugins/videonode/imx6/qsgvivantevideonode.h b/src/plugins/videonode/imx6/qsgvivantevideonode.h index 24b9ee492..05ab4f25b 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideonode.h +++ b/src/plugins/videonode/imx6/qsgvivantevideonode.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QSGVIDEONODE_VIVANTE_H #define QSGVIDEONODE_VIVANTE_H diff --git a/src/plugins/videonode/imx6/qsgvivantevideonodefactory.cpp b/src/plugins/videonode/imx6/qsgvivantevideonodefactory.cpp index 8086f0190..ca6550776 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideonodefactory.cpp +++ b/src/plugins/videonode/imx6/qsgvivantevideonodefactory.cpp @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #include "qsgvivantevideonodefactory.h" #include "qsgvivantevideonode.h" diff --git a/src/plugins/videonode/imx6/qsgvivantevideonodefactory.h b/src/plugins/videonode/imx6/qsgvivantevideonodefactory.h index 0e90d89d8..d81582788 100644 --- a/src/plugins/videonode/imx6/qsgvivantevideonodefactory.h +++ b/src/plugins/videonode/imx6/qsgvivantevideonodefactory.h @@ -1,41 +1,5 @@ -/**************************************************************************** -** -** Copyright (C) 2016 Pelagicore AG -** Contact: https://www.qt.io/licensing/ -** -** This file is part of the Qt Toolkit. -** -** $QT_BEGIN_LICENSE:LGPL$ -** Commercial License Usage -** Licensees holding valid commercial Qt licenses may use this file in -** accordance with the commercial license agreement provided with the -** Software or, alternatively, in accordance with the terms contained in -** a written agreement between you and The Qt Company. For licensing terms -** and conditions see https://www.qt.io/terms-conditions. For further -** information use the contact form at https://www.qt.io/contact-us. -** -** GNU Lesser General Public License Usage -** Alternatively, this file may be used under the terms of the GNU Lesser -** General Public License version 3 as published by the Free Software -** Foundation and appearing in the file LICENSE.LGPL3 included in the -** packaging of this file. Please review the following information to -** ensure the GNU Lesser General Public License version 3 requirements -** will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -** -** GNU General Public License Usage -** Alternatively, this file may be used under the terms of the GNU -** General Public License version 2.0 or (at your option) the GNU General -** Public license version 3 or any later version approved by the KDE Free -** Qt Foundation. The licenses are as published by the Free Software -** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -** included in the packaging of this file. Please review the following -** information to ensure the GNU General Public License requirements will -** be met: https://www.gnu.org/licenses/gpl-2.0.html and -** https://www.gnu.org/licenses/gpl-3.0.html. -** -** $QT_END_LICENSE$ -** -****************************************************************************/ +// Copyright (C) 2016 Pelagicore AG +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only #ifndef QSGVIDEONODEFACTORY_VIVANTE_H #define QSGVIDEONODEFACTORY_VIVANTE_H diff --git a/src/plugins/videonode/imx6/shaders/compile.bat b/src/plugins/videonode/imx6/shaders/compile.bat index 712bee6c5..5827ee9c0 100755 --- a/src/plugins/videonode/imx6/shaders/compile.bat +++ b/src/plugins/videonode/imx6/shaders/compile.bat @@ -1,41 +1,5 @@ -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: -:: :: Copyright (C) 2020 The Qt Company Ltd. -:: Contact: https://www.qt.io/licensing/ -:: -:: This file is part of the QtQuick module of the Qt Toolkit. -:: -:: $QT_BEGIN_LICENSE:LGPL$ -:: Commercial License Usage -:: Licensees holding valid commercial Qt licenses may use this file in -:: accordance with the commercial license agreement provided with the -:: Software or, alternatively, in accordance with the terms contained in -:: a written agreement between you and The Qt Company. For licensing terms -:: and conditions see https://www.qt.io/terms-conditions. For further -:: information use the contact form at https://www.qt.io/contact-us. -:: -:: GNU Lesser General Public License Usage -:: Alternatively, this file may be used under the terms of the GNU Lesser -:: General Public License version 3 as published by the Free Software -:: Foundation and appearing in the file LICENSE.LGPL3 included in the -:: packaging of this file. Please review the following information to -:: ensure the GNU Lesser General Public License version 3 requirements -:: will be met: https://www.gnu.org/licenses/lgpl-3.0.html. -:: -:: GNU General Public License Usage -:: Alternatively, this file may be used under the terms of the GNU -:: General Public License version 2.0 or (at your option) the GNU General -:: Public license version 3 or any later version approved by the KDE Free -:: Qt Foundation. The licenses are as published by the Free Software -:: Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3 -:: included in the packaging of this file. Please review the following -:: information to ensure the GNU General Public License requirements will -:: be met: https://www.gnu.org/licenses/gpl-2.0.html and -:: https://www.gnu.org/licenses/gpl-3.0.html. -:: -:: $QT_END_LICENSE$ -:: -::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::::: +:: SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GPL-3.0-only WITH Qt-GPL-exception-1.0 qsb -b --glsl "150,120,100 es" --hlsl 50 --msl 12 -o rgba.vert.qsb rgba.vert qsb --glsl "150,120,100 es" --hlsl 50 --msl 12 -o rgba.frag.qsb rgba.frag |