diff options
Diffstat (limited to 'src/multimedia/windows')
20 files changed, 2951 insertions, 0 deletions
diff --git a/src/multimedia/windows/qcomptr_p.h b/src/multimedia/windows/qcomptr_p.h new file mode 100644 index 000000000..befc97f11 --- /dev/null +++ b/src/multimedia/windows/qcomptr_p.h @@ -0,0 +1,33 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QCOMPTR_P_H +#define QCOMPTR_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qt_windows.h> +#include <wrl/client.h> + +using Microsoft::WRL::ComPtr; + +template<typename T, typename... Args> +ComPtr<T> makeComObject(Args &&...args) +{ + ComPtr<T> p; + // Don't use Attach because of MINGW64 bug + // #892 Microsoft::WRL::ComPtr::Attach leaks references + *p.GetAddressOf() = new T(std::forward<Args>(args)...); + return p; +} + +#endif diff --git a/src/multimedia/windows/qcomtaskresource_p.h b/src/multimedia/windows/qcomtaskresource_p.h new file mode 100644 index 000000000..90554da5e --- /dev/null +++ b/src/multimedia/windows/qcomtaskresource_p.h @@ -0,0 +1,152 @@ +// Copyright (C) 2023 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#ifndef QCOMTASKRESOURCE_P_H +#define QCOMTASKRESOURCE_P_H + +#include <QtCore/qassert.h> + +#include <objbase.h> +#include <algorithm> +#include <type_traits> +#include <utility> + +class QEmptyDeleter final +{ +public: + template<typename T> + void operator()(T /*element*/) const + { + } +}; + +class QComDeleter final +{ +public: + template<typename T> + void operator()(T element) const + { + element->Release(); + } +}; + +template<typename T> +class QComTaskResourceBase +{ +public: + QComTaskResourceBase(const QComTaskResourceBase<T> &source) = delete; + QComTaskResourceBase &operator=(const QComTaskResourceBase<T> &right) = delete; + + explicit operator bool() const { return m_resource != nullptr; } + + T *get() const { return m_resource; } + +protected: + QComTaskResourceBase() = default; + explicit QComTaskResourceBase(T *const resource) : m_resource(resource) { } + + T *release() { return std::exchange(m_resource, nullptr); } + + void reset(T *const resource = nullptr) + { + if (m_resource != resource) { + if (m_resource) + CoTaskMemFree(m_resource); + m_resource = resource; + } + } + + T *m_resource = nullptr; +}; + +template<typename T, typename TElementDeleter = QEmptyDeleter> +class QComTaskResource final : public QComTaskResourceBase<T> +{ + using Base = QComTaskResourceBase<T>; + +public: + using Base::QComTaskResourceBase; + + ~QComTaskResource() { reset(); } + + T *operator->() const { return m_resource; } + T &operator*() const { return *m_resource; } + + T **address() + { + Q_ASSERT(m_resource == nullptr); + return &m_resource; + } + + using Base::release; + using Base::reset; + +private: + using Base::m_resource; +}; + +template<typename T, typename TElementDeleter> +class QComTaskResource<T[], TElementDeleter> final : public QComTaskResourceBase<T> +{ + using Base = QComTaskResourceBase<T>; + +public: + QComTaskResource() = default; + explicit QComTaskResource(T *const resource, const std::size_t size) + : Base(resource), m_size(size) + { + } + + ~QComTaskResource() { reset(); } + + T &operator[](const std::size_t index) const + { + Q_ASSERT(index < m_size); + return m_resource[index]; + } + + T *release() + { + m_size = 0; + + return Base::release(); + } + + void reset() { reset(nullptr, 0); } + + void reset(T *const resource, const std::size_t size) + { + if (m_resource != resource) { + resetElements(); + + Base::reset(resource); + + m_size = size; + } + } + +private: + void resetElements() + { + if constexpr (!std::is_same_v<TElementDeleter, QEmptyDeleter>) { + std::for_each(m_resource, m_resource + m_size, TElementDeleter()); + } + } + + std::size_t m_size = 0; + + using Base::m_resource; +}; + +#endif diff --git a/src/multimedia/windows/qwindowsaudiodevice.cpp b/src/multimedia/windows/qwindowsaudiodevice.cpp new file mode 100644 index 000000000..f22567cbf --- /dev/null +++ b/src/multimedia/windows/qwindowsaudiodevice.cpp @@ -0,0 +1,229 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// INTERNAL USE ONLY: Do NOT use for any other purpose. +// + +#include "qwindowsaudiodevice_p.h" +#include "qwindowsaudioutils_p.h" + +#include <QtCore/qt_windows.h> +#include <QtCore/QDataStream> +#include <QtCore/QIODevice> + +#include <audioclient.h> +#include <mmsystem.h> + +#include <initguid.h> +#include <wtypes.h> +#include <propkeydef.h> +#include <mmdeviceapi.h> + +QT_BEGIN_NAMESPACE + +QWindowsAudioDeviceInfo::QWindowsAudioDeviceInfo(QByteArray dev, ComPtr<IMMDevice> immDev, int waveID, const QString &description, QAudioDevice::Mode mode) + : QAudioDevicePrivate(dev, mode), + m_devId(waveID), + m_immDev(std::move(immDev)) +{ + Q_ASSERT(m_immDev); + + this->description = description; + + ComPtr<IAudioClient> audioClient; + HRESULT hr = m_immDev->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, nullptr, + (void **)audioClient.GetAddressOf()); + if (SUCCEEDED(hr)) { + WAVEFORMATEX *pwfx = nullptr; + hr = audioClient->GetMixFormat(&pwfx); + if (SUCCEEDED(hr)) + preferredFormat = QWindowsAudioUtils::waveFormatExToFormat(*pwfx); + } + + if (!preferredFormat.isValid()) { + preferredFormat.setSampleRate(44100); + preferredFormat.setChannelCount(2); + preferredFormat.setSampleFormat(QAudioFormat::Int16); + } + + DWORD fmt = 0; + + if(mode == QAudioDevice::Output) { + WAVEOUTCAPS woc; + if (waveOutGetDevCaps(m_devId, &woc, sizeof(WAVEOUTCAPS)) == MMSYSERR_NOERROR) + fmt = woc.dwFormats; + } else { + WAVEINCAPS woc; + if (waveInGetDevCaps(m_devId, &woc, sizeof(WAVEINCAPS)) == MMSYSERR_NOERROR) + fmt = woc.dwFormats; + } + + if (!fmt) + return; + + // Check sample size + if ((fmt & WAVE_FORMAT_1M08) + || (fmt & WAVE_FORMAT_1S08) + || (fmt & WAVE_FORMAT_2M08) + || (fmt & WAVE_FORMAT_2S08) + || (fmt & WAVE_FORMAT_4M08) + || (fmt & WAVE_FORMAT_4S08) + || (fmt & WAVE_FORMAT_48M08) + || (fmt & WAVE_FORMAT_48S08) + || (fmt & WAVE_FORMAT_96M08) + || (fmt & WAVE_FORMAT_96S08)) { + supportedSampleFormats.append(QAudioFormat::UInt8); + } + if ((fmt & WAVE_FORMAT_1M16) + || (fmt & WAVE_FORMAT_1S16) + || (fmt & WAVE_FORMAT_2M16) + || (fmt & WAVE_FORMAT_2S16) + || (fmt & WAVE_FORMAT_4M16) + || (fmt & WAVE_FORMAT_4S16) + || (fmt & WAVE_FORMAT_48M16) + || (fmt & WAVE_FORMAT_48S16) + || (fmt & WAVE_FORMAT_96M16) + || (fmt & WAVE_FORMAT_96S16)) { + supportedSampleFormats.append(QAudioFormat::Int16); + } + + minimumSampleRate = std::numeric_limits<int>::max(); + maximumSampleRate = 0; + // Check sample rate + if ((fmt & WAVE_FORMAT_1M08) + || (fmt & WAVE_FORMAT_1S08) + || (fmt & WAVE_FORMAT_1M16) + || (fmt & WAVE_FORMAT_1S16)) { + minimumSampleRate = qMin(minimumSampleRate, 11025); + maximumSampleRate = qMax(maximumSampleRate, 11025); + } + if ((fmt & WAVE_FORMAT_2M08) + || (fmt & WAVE_FORMAT_2S08) + || (fmt & WAVE_FORMAT_2M16) + || (fmt & WAVE_FORMAT_2S16)) { + minimumSampleRate = qMin(minimumSampleRate, 22050); + maximumSampleRate = qMax(maximumSampleRate, 22050); + } + if ((fmt & WAVE_FORMAT_4M08) + || (fmt & WAVE_FORMAT_4S08) + || (fmt & WAVE_FORMAT_4M16) + || (fmt & WAVE_FORMAT_4S16)) { + minimumSampleRate = qMin(minimumSampleRate, 44100); + maximumSampleRate = qMax(maximumSampleRate, 44100); + } + if ((fmt & WAVE_FORMAT_48M08) + || (fmt & WAVE_FORMAT_48S08) + || (fmt & WAVE_FORMAT_48M16) + || (fmt & WAVE_FORMAT_48S16)) { + minimumSampleRate = qMin(minimumSampleRate, 48000); + maximumSampleRate = qMax(maximumSampleRate, 48000); + } + if ((fmt & WAVE_FORMAT_96M08) + || (fmt & WAVE_FORMAT_96S08) + || (fmt & WAVE_FORMAT_96M16) + || (fmt & WAVE_FORMAT_96S16)) { + minimumSampleRate = qMin(minimumSampleRate, 96000); + maximumSampleRate = qMax(maximumSampleRate, 96000); + } + if (minimumSampleRate == std::numeric_limits<int>::max()) + minimumSampleRate = 0; + + minimumChannelCount = std::numeric_limits<int>::max(); + maximumChannelCount = 0; + // Check channel count + if (fmt & WAVE_FORMAT_1M08 + || fmt & WAVE_FORMAT_1M16 + || fmt & WAVE_FORMAT_2M08 + || fmt & WAVE_FORMAT_2M16 + || fmt & WAVE_FORMAT_4M08 + || fmt & WAVE_FORMAT_4M16 + || fmt & WAVE_FORMAT_48M08 + || fmt & WAVE_FORMAT_48M16 + || fmt & WAVE_FORMAT_96M08 + || fmt & WAVE_FORMAT_96M16) { + minimumChannelCount = 1; + maximumChannelCount = 1; + } + if (fmt & WAVE_FORMAT_1S08 + || fmt & WAVE_FORMAT_1S16 + || fmt & WAVE_FORMAT_2S08 + || fmt & WAVE_FORMAT_2S16 + || fmt & WAVE_FORMAT_4S08 + || fmt & WAVE_FORMAT_4S16 + || fmt & WAVE_FORMAT_48S08 + || fmt & WAVE_FORMAT_48S16 + || fmt & WAVE_FORMAT_96S08 + || fmt & WAVE_FORMAT_96S16) { + minimumChannelCount = qMin(minimumChannelCount, 2); + maximumChannelCount = qMax(maximumChannelCount, 2); + } + + if (minimumChannelCount == std::numeric_limits<int>::max()) + minimumChannelCount = 0; + + // WAVEOUTCAPS and WAVEINCAPS contains information only for the previously tested parameters. + // WaveOut and WaveInt might actually support more formats, the only way to know is to try + // opening the device with it. + QAudioFormat testFormat; + testFormat.setChannelCount(maximumChannelCount); + testFormat.setSampleRate(maximumSampleRate); + const QAudioFormat defaultTestFormat(testFormat); + + // Check if float samples are supported + testFormat.setSampleFormat(QAudioFormat::Float); + if (testSettings(testFormat)) + supportedSampleFormats.append(QAudioFormat::Float); + + // Check channel counts > 2 + testFormat = defaultTestFormat; + for (int i = 18; i > 2; --i) { // <mmreg.h> defines 18 different channels + testFormat.setChannelCount(i); + if (testSettings(testFormat)) { + maximumChannelCount = i; + break; + } + } + + channelConfiguration = QAudioFormat::defaultChannelConfigForChannelCount(maximumChannelCount); + + ComPtr<IPropertyStore> props; + hr = m_immDev->OpenPropertyStore(STGM_READ, props.GetAddressOf()); + if (SUCCEEDED(hr)) { + PROPVARIANT var; + PropVariantInit(&var); + hr = props->GetValue(PKEY_AudioEndpoint_PhysicalSpeakers, &var); + if (SUCCEEDED(hr) && var.uintVal != 0) + channelConfiguration = QWindowsAudioUtils::maskToChannelConfig(var.uintVal, maximumChannelCount); + } +} + +QWindowsAudioDeviceInfo::~QWindowsAudioDeviceInfo() +{ +} + +bool QWindowsAudioDeviceInfo::testSettings(const QAudioFormat& format) const +{ + WAVEFORMATEXTENSIBLE wfx; + if (QWindowsAudioUtils::formatToWaveFormatExtensible(format, wfx)) { + // query only, do not open device + if (mode == QAudioDevice::Output) { + return (waveOutOpen(NULL, UINT_PTR(m_devId), &wfx.Format, 0, 0, + WAVE_FORMAT_QUERY) == MMSYSERR_NOERROR); + } else { // AudioInput + return (waveInOpen(NULL, UINT_PTR(m_devId), &wfx.Format, 0, 0, + WAVE_FORMAT_QUERY) == MMSYSERR_NOERROR); + } + } + + return false; +} + +QT_END_NAMESPACE diff --git a/src/multimedia/windows/qwindowsaudiodevice_p.h b/src/multimedia/windows/qwindowsaudiodevice_p.h new file mode 100644 index 000000000..b2af4bfe0 --- /dev/null +++ b/src/multimedia/windows/qwindowsaudiodevice_p.h @@ -0,0 +1,60 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + + +#ifndef QWINDOWSAUDIODEVICEINFO_H +#define QWINDOWSAUDIODEVICEINFO_H + +#include <QtCore/qbytearray.h> +#include <QtCore/qstringlist.h> +#include <QtCore/qlist.h> +#include <QtCore/qdebug.h> + +#include <QtMultimedia/qaudiodevice.h> +#include <private/qaudiosystem_p.h> +#include <private/qaudiodevice_p.h> +#include <qcomptr_p.h> + +struct IMMDevice; + +QT_BEGIN_NAMESPACE + +const unsigned int MAX_SAMPLE_RATES = 5; +const unsigned int SAMPLE_RATES[] = { 8000, 11025, 22050, 44100, 48000 }; + +class QWindowsAudioDeviceInfo : public QAudioDevicePrivate +{ +public: + QWindowsAudioDeviceInfo(QByteArray dev, ComPtr<IMMDevice> immdev, int waveID, const QString &description, QAudioDevice::Mode mode); + ~QWindowsAudioDeviceInfo(); + + bool open(); + void close(); + + bool testSettings(const QAudioFormat& format) const; + + int waveId() const { return m_devId; } + ComPtr<IMMDevice> immDev() const { return m_immDev; } + +private: + quint32 m_devId; + ComPtr<IMMDevice> m_immDev; +}; + + + +QT_END_NAMESPACE + + +#endif // QWINDOWSAUDIODEVICEINFO_H diff --git a/src/multimedia/windows/qwindowsaudiosink.cpp b/src/multimedia/windows/qwindowsaudiosink.cpp new file mode 100644 index 000000000..404496970 --- /dev/null +++ b/src/multimedia/windows/qwindowsaudiosink.cpp @@ -0,0 +1,388 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// INTERNAL USE ONLY: Do NOT use for any other purpose. +// + +#include "qwindowsaudiosink_p.h" +#include "qwindowsaudioutils_p.h" +#include "qwindowsmultimediautils_p.h" +#include "qcomtaskresource_p.h" + +#include <QtCore/QDataStream> +#include <QtCore/qtimer.h> +#include <QtCore/qloggingcategory.h> +#include <QtCore/qpointer.h> + +#include <private/qaudiohelpers_p.h> + +#include <audioclient.h> +#include <mmdeviceapi.h> + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLcAudioOutput, "qt.multimedia.audiooutput") + +using namespace QWindowsMultimediaUtils; + +class OutputPrivate : public QIODevice +{ + Q_OBJECT +public: + OutputPrivate(QWindowsAudioSink &audio) : QIODevice(&audio), audioDevice(audio) {} + ~OutputPrivate() override = default; + + qint64 readData(char *, qint64) override { return 0; } + qint64 writeData(const char *data, qint64 len) override { return audioDevice.push(data, len); } + +private: + QWindowsAudioSink &audioDevice; +}; + +std::optional<quint32> audioClientFramesAvailable(IAudioClient *client) +{ + auto framesAllocated = QWindowsAudioUtils::audioClientFramesAllocated(client); + auto framesInUse = QWindowsAudioUtils::audioClientFramesInUse(client); + + if (framesAllocated && framesInUse) + return *framesAllocated - *framesInUse; + return {}; +} + +QWindowsAudioSink::QWindowsAudioSink(ComPtr<IMMDevice> device, QObject *parent) : + QPlatformAudioSink(parent), + m_timer(new QTimer(this)), + m_pushSource(new OutputPrivate(*this)), + m_device(std::move(device)) +{ + m_pushSource->open(QIODevice::WriteOnly|QIODevice::Unbuffered); + m_timer->setSingleShot(true); + m_timer->setTimerType(Qt::PreciseTimer); +} + +QWindowsAudioSink::~QWindowsAudioSink() +{ + close(); +} + +qint64 QWindowsAudioSink::remainingPlayTimeUs() +{ + auto framesInUse = QWindowsAudioUtils::audioClientFramesInUse(m_audioClient.Get()); + return m_resampler.outputFormat().durationForFrames(framesInUse ? *framesInUse : 0); +} + +void QWindowsAudioSink::deviceStateChange(QAudio::State state, QAudio::Error error) +{ + if (state != deviceState) { + if (state == QAudio::ActiveState) { + m_audioClient->Start(); + qCDebug(qLcAudioOutput) << "Audio client started"; + + } else if (deviceState == QAudio::ActiveState) { + m_timer->stop(); + m_audioClient->Stop(); + qCDebug(qLcAudioOutput) << "Audio client stopped"; + } + + QPointer<QWindowsAudioSink> thisGuard(this); + deviceState = state; + emit stateChanged(deviceState); + if (!thisGuard) + return; + } + + if (error != errorState) { + errorState = error; + emit errorChanged(error); + } +} + +QAudioFormat QWindowsAudioSink::format() const +{ + return m_format; +} + +void QWindowsAudioSink::setFormat(const QAudioFormat& fmt) +{ + if (deviceState == QAudio::StoppedState) + m_format = fmt; +} + +void QWindowsAudioSink::pullSource() +{ + qCDebug(qLcAudioOutput) << "Pull source"; + if (!m_pullSource) + return; + + auto bytesAvailable = m_pullSource->isOpen() ? qsizetype(m_pullSource->bytesAvailable()) : 0; + auto readLen = qMin(bytesFree(), bytesAvailable); + if (readLen > 0) { + QByteArray samples = m_pullSource->read(readLen); + if (samples.size() == 0) { + deviceStateChange(QAudio::IdleState, QAudio::IOError); + return; + } else { + write(samples.data(), samples.size()); + } + } + + auto playTimeUs = remainingPlayTimeUs(); + if (playTimeUs == 0) { + deviceStateChange(QAudio::IdleState, m_pullSource->atEnd() ? QAudio::NoError : QAudio::UnderrunError); + } else { + deviceStateChange(QAudio::ActiveState, QAudio::NoError); + m_timer->start(playTimeUs / 2000); + } +} + +void QWindowsAudioSink::start(QIODevice* device) +{ + qCDebug(qLcAudioOutput) << "start(ioDevice)" << deviceState; + if (deviceState != QAudio::StoppedState) + close(); + + if (device == nullptr) + return; + + if (!open()) { + errorState = QAudio::OpenError; + emit errorChanged(QAudio::OpenError); + return; + } + + m_pullSource = device; + + connect(device, &QIODevice::readyRead, this, &QWindowsAudioSink::pullSource); + m_timer->disconnect(); + m_timer->callOnTimeout(this, &QWindowsAudioSink::pullSource); + pullSource(); +} + +qint64 QWindowsAudioSink::push(const char *data, qint64 len) +{ + if (deviceState == QAudio::StoppedState) + return -1; + + qint64 written = write(data, len); + if (written > 0) { + deviceStateChange(QAudio::ActiveState, QAudio::NoError); + m_timer->start(remainingPlayTimeUs() /1000); + } + + return written; +} + +QIODevice* QWindowsAudioSink::start() +{ + qCDebug(qLcAudioOutput) << "start()"; + if (deviceState != QAudio::StoppedState) + close(); + + if (!open()) { + errorState = QAudio::OpenError; + emit errorChanged(QAudio::OpenError); + return nullptr; + } + + deviceStateChange(QAudio::IdleState, QAudio::NoError); + + m_timer->disconnect(); + m_timer->callOnTimeout(this, [this](){ + deviceStateChange(QAudio::IdleState, QAudio::UnderrunError); + }); + + return m_pushSource.get(); +} + +bool QWindowsAudioSink::open() +{ + if (m_audioClient) + return true; + + HRESULT hr = m_device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, + nullptr, (void**)m_audioClient.GetAddressOf()); + if (FAILED(hr)) { + qCWarning(qLcAudioOutput) << "Failed to activate audio device" << errorString(hr); + return false; + } + + auto resetClient = qScopeGuard([this](){ m_audioClient.Reset(); }); + + QComTaskResource<WAVEFORMATEX> pwfx; + hr = m_audioClient->GetMixFormat(pwfx.address()); + if (FAILED(hr)) { + qCWarning(qLcAudioOutput) << "Format unsupported" << errorString(hr); + return false; + } + + if (!m_resampler.setup(m_format, QWindowsAudioUtils::waveFormatExToFormat(*pwfx))) { + qCWarning(qLcAudioOutput) << "Failed to set up resampler"; + return false; + } + + if (m_bufferSize == 0) + m_bufferSize = m_format.sampleRate() * m_format.bytesPerFrame() / 2; + + REFERENCE_TIME requestedDuration = m_format.durationForBytes(m_bufferSize) * 10; + + hr = m_audioClient->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, requestedDuration, 0, pwfx.get(), + nullptr); + + if (FAILED(hr)) { + qCWarning(qLcAudioOutput) << "Failed to initialize audio client" << errorString(hr); + return false; + } + + auto framesAllocated = QWindowsAudioUtils::audioClientFramesAllocated(m_audioClient.Get()); + if (!framesAllocated) { + qCWarning(qLcAudioOutput) << "Failed to get audio client buffer size"; + return false; + } + + m_bufferSize = m_format.bytesForDuration( + m_resampler.outputFormat().durationForFrames(*framesAllocated)); + + hr = m_audioClient->GetService(__uuidof(IAudioRenderClient), (void**)m_renderClient.GetAddressOf()); + if (FAILED(hr)) { + qCWarning(qLcAudioOutput) << "Failed to obtain audio client rendering service" + << errorString(hr); + return false; + } + + resetClient.dismiss(); + + return true; +} + +void QWindowsAudioSink::close() +{ + qCDebug(qLcAudioOutput) << "close()"; + if (deviceState == QAudio::StoppedState) + return; + + deviceStateChange(QAudio::StoppedState, QAudio::NoError); + + if (m_pullSource) + disconnect(m_pullSource, &QIODevice::readyRead, this, &QWindowsAudioSink::pullSource); + m_audioClient.Reset(); + m_renderClient.Reset(); + m_pullSource = nullptr; +} + +qsizetype QWindowsAudioSink::bytesFree() const +{ + if (!m_audioClient) + return 0; + + auto framesAvailable = audioClientFramesAvailable(m_audioClient.Get()); + if (framesAvailable) + return m_resampler.inputBufferSize(*framesAvailable * m_resampler.outputFormat().bytesPerFrame()); + return 0; +} + +void QWindowsAudioSink::setBufferSize(qsizetype value) +{ + if (!m_audioClient) + m_bufferSize = value; +} + +qint64 QWindowsAudioSink::processedUSecs() const +{ + if (!m_audioClient) + return 0; + + return m_format.durationForBytes(m_resampler.totalInputBytes()); +} + +qint64 QWindowsAudioSink::write(const char *data, qint64 len) +{ + Q_ASSERT(m_audioClient); + Q_ASSERT(m_renderClient); + + qCDebug(qLcAudioOutput) << "write()" << len; + + auto framesAvailable = audioClientFramesAvailable(m_audioClient.Get()); + if (!framesAvailable) + return -1; + + auto maxBytesCanWrite = m_format.bytesForDuration( + m_resampler.outputFormat().durationForFrames(*framesAvailable)); + qsizetype writeSize = qMin(maxBytesCanWrite, len); + + QByteArray writeBytes = m_resampler.resample({data, writeSize}); + qint32 writeFramesNum = m_resampler.outputFormat().framesForBytes(writeBytes.size()); + + quint8 *buffer = nullptr; + HRESULT hr = m_renderClient->GetBuffer(writeFramesNum, &buffer); + if (FAILED(hr)) { + qCWarning(qLcAudioOutput) << "Failed to get buffer" << errorString(hr); + return -1; + } + + if (m_volume < qreal(1.0)) + QAudioHelperInternal::qMultiplySamples(m_volume, m_resampler.outputFormat(), writeBytes.data(), buffer, writeBytes.size()); + else + std::memcpy(buffer, writeBytes.data(), writeBytes.size()); + + DWORD flags = writeBytes.isEmpty() ? AUDCLNT_BUFFERFLAGS_SILENT : 0; + hr = m_renderClient->ReleaseBuffer(writeFramesNum, flags); + if (FAILED(hr)) { + qCWarning(qLcAudioOutput) << "Failed to return buffer" << errorString(hr); + return -1; + } + + return writeSize; +} + +void QWindowsAudioSink::resume() +{ + qCDebug(qLcAudioOutput) << "resume()"; + if (deviceState == QAudio::SuspendedState) { + if (m_pullSource) { + pullSource(); + } else { + deviceStateChange(suspendedInState, QAudio::NoError); + if (remainingPlayTimeUs() > 0) + m_audioClient->Start(); + } + } +} + +void QWindowsAudioSink::suspend() +{ + qCDebug(qLcAudioOutput) << "suspend()"; + if (deviceState == QAudio::ActiveState || deviceState == QAudio::IdleState) { + suspendedInState = deviceState; + deviceStateChange(QAudio::SuspendedState, QAudio::NoError); + } +} + +void QWindowsAudioSink::setVolume(qreal v) +{ + if (qFuzzyCompare(m_volume, v)) + return; + + m_volume = qBound(qreal(0), v, qreal(1)); +} + +void QWindowsAudioSink::stop() { + // TODO: investigate and find a way to drain and stop instead of closing + close(); +} + +void QWindowsAudioSink::reset() +{ + close(); +} + +QT_END_NAMESPACE + +#include "qwindowsaudiosink.moc" diff --git a/src/multimedia/windows/qwindowsaudiosink_p.h b/src/multimedia/windows/qwindowsaudiosink_p.h new file mode 100644 index 000000000..1a98bc296 --- /dev/null +++ b/src/multimedia/windows/qwindowsaudiosink_p.h @@ -0,0 +1,97 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#ifndef QWINDOWSAUDIOOUTPUT_H +#define QWINDOWSAUDIOOUTPUT_H + +#include <QtCore/qdebug.h> +#include <QtCore/qelapsedtimer.h> +#include <QtCore/qiodevice.h> +#include <QtCore/qstring.h> +#include <QtCore/qstringlist.h> +#include <QtCore/qdatetime.h> +#include <QtCore/qmutex.h> +#include <QtCore/qtimer.h> +#include <QtCore/qpointer.h> + +#include <QtMultimedia/qaudio.h> +#include <QtMultimedia/qaudiodevice.h> +#include <private/qaudiosystem_p.h> +#include <qcomptr_p.h> +#include <qwindowsresampler_p.h> + +#include <audioclient.h> +#include <mmdeviceapi.h> + +QT_BEGIN_NAMESPACE + +class QWindowsResampler; + +class QWindowsAudioSink : public QPlatformAudioSink +{ + Q_OBJECT +public: + QWindowsAudioSink(ComPtr<IMMDevice> device, QObject *parent); + ~QWindowsAudioSink(); + + void setFormat(const QAudioFormat& fmt) override; + QAudioFormat format() const override; + QIODevice* start() override; + void start(QIODevice* device) override; + void stop() override; + void reset() override; + void suspend() override; + void resume() override; + qsizetype bytesFree() const override; + void setBufferSize(qsizetype value) override; + qsizetype bufferSize() const override { return m_bufferSize; } + qint64 processedUSecs() const override; + QAudio::Error error() const override { return errorState; } + QAudio::State state() const override { return deviceState; } + void setVolume(qreal) override; + qreal volume() const override { return m_volume; } + +private: + friend class OutputPrivate; + qint64 write(const char *data, qint64 len); + qint64 push(const char *data, qint64 len); + + bool open(); + void close(); + + void deviceStateChange(QAudio::State, QAudio::Error); + + void pullSource(); + qint64 remainingPlayTimeUs(); + + QAudioFormat m_format; + QAudio::Error errorState = QAudio::NoError; + QAudio::State deviceState = QAudio::StoppedState; + QAudio::State suspendedInState = QAudio::SuspendedState; + + qsizetype m_bufferSize = 0; + qreal m_volume = 1.0; + QTimer *m_timer = nullptr; + QScopedPointer<QIODevice> m_pushSource; + QPointer<QIODevice> m_pullSource; + ComPtr<IMMDevice> m_device; + ComPtr<IAudioClient> m_audioClient; + ComPtr<IAudioRenderClient> m_renderClient; + QWindowsResampler m_resampler; +}; + +QT_END_NAMESPACE + + +#endif // QWINDOWSAUDIOOUTPUT_H diff --git a/src/multimedia/windows/qwindowsaudiosource.cpp b/src/multimedia/windows/qwindowsaudiosource.cpp new file mode 100644 index 000000000..85fc454ad --- /dev/null +++ b/src/multimedia/windows/qwindowsaudiosource.cpp @@ -0,0 +1,406 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// INTERNAL USE ONLY: Do NOT use for any other purpose. +// + + +#include "qwindowsaudiosource_p.h" +#include "qwindowsmultimediautils_p.h" +#include "qcomtaskresource_p.h" + +#include <QtCore/QDataStream> +#include <QtCore/qtimer.h> + +#include <private/qaudiohelpers_p.h> + +#include <qloggingcategory.h> +#include <qdebug.h> +#include <audioclient.h> +#include <mmdeviceapi.h> + +QT_BEGIN_NAMESPACE + +static Q_LOGGING_CATEGORY(qLcAudioSource, "qt.multimedia.audiosource") + +using namespace QWindowsMultimediaUtils; + +class OurSink : public QIODevice +{ +public: + OurSink(QWindowsAudioSource& source) : QIODevice(&source), m_audioSource(source) {} + + qint64 bytesAvailable() const override { return m_audioSource.bytesReady(); } + qint64 readData(char* data, qint64 len) override { return m_audioSource.read(data, len); } + qint64 writeData(const char*, qint64) override { return 0; } + +private: + QWindowsAudioSource &m_audioSource; +}; + +QWindowsAudioSource::QWindowsAudioSource(ComPtr<IMMDevice> device, QObject *parent) + : QPlatformAudioSource(parent), + m_timer(new QTimer(this)), + m_device(std::move(device)), + m_ourSink(new OurSink(*this)) +{ + m_ourSink->open(QIODevice::ReadOnly|QIODevice::Unbuffered); + m_timer->setTimerType(Qt::PreciseTimer); + m_timer->setSingleShot(true); + m_timer->callOnTimeout(this, &QWindowsAudioSource::pullCaptureClient); +} + +void QWindowsAudioSource::setVolume(qreal volume) +{ + m_volume = qBound(qreal(0), volume, qreal(1)); +} + +qreal QWindowsAudioSource::volume() const +{ + return m_volume; +} + +QWindowsAudioSource::~QWindowsAudioSource() +{ + stop(); +} + +QAudio::Error QWindowsAudioSource::error() const +{ + return m_errorState; +} + +QAudio::State QWindowsAudioSource::state() const +{ + return m_deviceState; +} + +void QWindowsAudioSource::setFormat(const QAudioFormat& fmt) +{ + if (m_deviceState == QAudio::StoppedState) { + m_format = fmt; + } else { + if (m_format != fmt) { + qWarning() << "Cannot set a new audio format, in the current state (" + << m_deviceState << ")"; + } + } +} + +QAudioFormat QWindowsAudioSource::format() const +{ + return m_format; +} + +void QWindowsAudioSource::deviceStateChange(QAudio::State state, QAudio::Error error) +{ + if (state != m_deviceState) { + bool wasActive = m_deviceState == QAudio::ActiveState || m_deviceState == QAudio::IdleState; + bool isActive = state == QAudio::ActiveState || state == QAudio::IdleState; + + if (isActive && !wasActive) { + m_audioClient->Start(); + qCDebug(qLcAudioSource) << "Audio client started"; + + } else if (wasActive && !isActive) { + m_timer->stop(); + m_audioClient->Stop(); + qCDebug(qLcAudioSource) << "Audio client stopped"; + } + + m_deviceState = state; + emit stateChanged(m_deviceState); + } + + if (error != m_errorState) { + m_errorState = error; + emit errorChanged(error); + } +} + +QByteArray QWindowsAudioSource::readCaptureClientBuffer() +{ + UINT32 actualFrames = 0; + BYTE *data = nullptr; + DWORD flags = 0; + HRESULT hr = m_captureClient->GetBuffer(&data, &actualFrames, &flags, nullptr, nullptr); + if (FAILED(hr)) { + qWarning() << "IAudioCaptureClient::GetBuffer failed" << errorString(hr); + deviceStateChange(QAudio::IdleState, QAudio::IOError); + return {}; + } + + if (actualFrames == 0) + return {}; + + QByteArray out; + if (flags & AUDCLNT_BUFFERFLAGS_SILENT) { + out.resize(m_resampler.outputFormat().bytesForDuration( + m_resampler.inputFormat().framesForDuration(actualFrames)), + 0); + } else { + out = m_resampler.resample( + { data, m_resampler.inputFormat().bytesForFrames(actualFrames) }); + QAudioHelperInternal::qMultiplySamples(m_volume, m_resampler.outputFormat(), out.data(), out.data(), out.size()); + } + + hr = m_captureClient->ReleaseBuffer(actualFrames); + if (FAILED(hr)) { + qWarning() << "IAudioCaptureClient::ReleaseBuffer failed" << errorString(hr); + deviceStateChange(QAudio::IdleState, QAudio::IOError); + return {}; + } + + return out; +} + +void QWindowsAudioSource::schedulePull() +{ + auto allocated = QWindowsAudioUtils::audioClientFramesAllocated(m_audioClient.Get()); + auto inUse = QWindowsAudioUtils::audioClientFramesInUse(m_audioClient.Get()); + + if (!allocated || !inUse) { + deviceStateChange(QAudio::IdleState, QAudio::IOError); + } else { + // Schedule the next audio pull immediately if the audio buffer is more + // than half-full or wait until the audio source fills at least half of it. + if (*inUse > *allocated / 2) { + m_timer->start(0); + } else { + auto timeToHalfBuffer = m_resampler.inputFormat().durationForFrames(*allocated / 2 - *inUse); + m_timer->start(timeToHalfBuffer / 1000); + } + } +} + +void QWindowsAudioSource::pullCaptureClient() +{ + qCDebug(qLcAudioSource) << "Pull captureClient"; + while (true) { + auto out = readCaptureClientBuffer(); + if (out.isEmpty()) + break; + + if (m_clientSink) { + qint64 written = m_clientSink->write(out.data(), out.size()); + if (written != out.size()) + qCDebug(qLcAudioSource) << "Did not write all data to the output"; + + } else { + m_clientBufferResidue += out; + emit m_ourSink->readyRead(); + } + } + + schedulePull(); +} + +void QWindowsAudioSource::start(QIODevice* device) +{ + qCDebug(qLcAudioSource) << "start(ioDevice)"; + if (m_deviceState != QAudio::StoppedState) + close(); + + if (device == nullptr) + return; + + if (!open()) { + m_errorState = QAudio::OpenError; + emit errorChanged(QAudio::OpenError); + return; + } + + m_clientSink = device; + schedulePull(); + deviceStateChange(QAudio::ActiveState, QAudio::NoError); +} + +QIODevice* QWindowsAudioSource::start() +{ + qCDebug(qLcAudioSource) << "start()"; + if (m_deviceState != QAudio::StoppedState) + close(); + + if (!open()) { + m_errorState = QAudio::OpenError; + emit errorChanged(QAudio::OpenError); + return nullptr; + } + + schedulePull(); + deviceStateChange(QAudio::IdleState, QAudio::NoError); + return m_ourSink; +} + +void QWindowsAudioSource::stop() +{ + if (m_deviceState == QAudio::StoppedState) + return; + + close(); +} + +bool QWindowsAudioSource::open() +{ + HRESULT hr = m_device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, + nullptr, (void**)m_audioClient.GetAddressOf()); + if (FAILED(hr)) { + qCWarning(qLcAudioSource) << "Failed to activate audio device" << errorString(hr); + return false; + } + + QComTaskResource<WAVEFORMATEX> pwfx; + hr = m_audioClient->GetMixFormat(pwfx.address()); + if (FAILED(hr)) { + qCWarning(qLcAudioSource) << "Format unsupported" << errorString(hr); + return false; + } + + if (!m_resampler.setup(QWindowsAudioUtils::waveFormatExToFormat(*pwfx), m_format)) { + qCWarning(qLcAudioSource) << "Failed to set up resampler"; + return false; + } + + if (m_bufferSize == 0) + m_bufferSize = m_format.sampleRate() * m_format.bytesPerFrame() / 5; // 200ms + + REFERENCE_TIME requestedDuration = m_format.durationForBytes(m_bufferSize); + + hr = m_audioClient->Initialize(AUDCLNT_SHAREMODE_SHARED, 0, requestedDuration, 0, pwfx.get(), + nullptr); + + if (FAILED(hr)) { + qCWarning(qLcAudioSource) << "Failed to initialize audio client" << errorString(hr); + return false; + } + + auto framesAllocated = QWindowsAudioUtils::audioClientFramesAllocated(m_audioClient.Get()); + if (!framesAllocated) { + qCWarning(qLcAudioSource) << "Failed to get audio client buffer size"; + return false; + } + + m_bufferSize = m_format.bytesForDuration( + m_resampler.inputFormat().durationForFrames(*framesAllocated)); + + hr = m_audioClient->GetService(__uuidof(IAudioCaptureClient), (void**)m_captureClient.GetAddressOf()); + if (FAILED(hr)) { + qCWarning(qLcAudioSource) << "Failed to obtain audio client rendering service" << errorString(hr); + return false; + } + + return true; +} + +void QWindowsAudioSource::close() +{ + qCDebug(qLcAudioSource) << "close()"; + if (m_deviceState == QAudio::StoppedState) + return; + + deviceStateChange(QAudio::StoppedState, QAudio::NoError); + + m_clientBufferResidue.clear(); + m_captureClient.Reset(); + m_audioClient.Reset(); + m_clientSink = nullptr; +} + +qsizetype QWindowsAudioSource::bytesReady() const +{ + if (m_deviceState == QAudio::StoppedState || m_deviceState == QAudio::SuspendedState) + return 0; + + auto frames = QWindowsAudioUtils::audioClientFramesInUse(m_audioClient.Get()); + if (frames) { + auto clientBufferSize = m_resampler.outputFormat().bytesForDuration( + m_resampler.inputFormat().durationForFrames(*frames)); + + return clientBufferSize + m_clientBufferResidue.size(); + + } else { + return 0; + } +} + +qint64 QWindowsAudioSource::read(char* data, qint64 len) +{ + deviceStateChange(QAudio::ActiveState, QAudio::NoError); + schedulePull(); + + if (data == nullptr || len < 0) + return -1; + + auto offset = 0; + if (!m_clientBufferResidue.isEmpty()) { + auto copyLen = qMin(m_clientBufferResidue.size(), len); + memcpy(data, m_clientBufferResidue.data(), copyLen); + len -= copyLen; + offset += copyLen; + } + + m_clientBufferResidue = QByteArray{ m_clientBufferResidue.data() + offset, + m_clientBufferResidue.size() - offset }; + + if (len > 0) { + auto out = readCaptureClientBuffer(); + if (!out.isEmpty()) { + qsizetype copyLen = qMin(out.size(), len); + memcpy(data + offset, out.data(), copyLen); + offset += copyLen; + + m_clientBufferResidue = QByteArray{ out.data() + copyLen, out.size() - copyLen }; + } + } + + return offset; +} + +void QWindowsAudioSource::resume() +{ + qCDebug(qLcAudioSource) << "resume()"; + if (m_deviceState == QAudio::SuspendedState) { + deviceStateChange(QAudio::ActiveState, QAudio::NoError); + pullCaptureClient(); + } +} + +void QWindowsAudioSource::setBufferSize(qsizetype value) +{ + m_bufferSize = value; +} + +qsizetype QWindowsAudioSource::bufferSize() const +{ + return m_bufferSize; +} + +qint64 QWindowsAudioSource::processedUSecs() const +{ + if (m_deviceState == QAudio::StoppedState) + return 0; + + return m_resampler.outputFormat().durationForBytes(m_resampler.totalOutputBytes()); +} + +void QWindowsAudioSource::suspend() +{ + qCDebug(qLcAudioSource) << "suspend"; + if (m_deviceState == QAudio::ActiveState || m_deviceState == QAudio::IdleState) + deviceStateChange(QAudio::SuspendedState, QAudio::NoError); +} + +void QWindowsAudioSource::reset() +{ + stop(); +} + +QT_END_NAMESPACE diff --git a/src/multimedia/windows/qwindowsaudiosource_p.h b/src/multimedia/windows/qwindowsaudiosource_p.h new file mode 100644 index 000000000..68d57bbe1 --- /dev/null +++ b/src/multimedia/windows/qwindowsaudiosource_p.h @@ -0,0 +1,96 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#ifndef QWINDOWSAUDIOINPUT_H +#define QWINDOWSAUDIOINPUT_H + +#include "qwindowsaudioutils_p.h" + +#include <QtCore/qfile.h> +#include <QtCore/qdebug.h> +#include <QtCore/qelapsedtimer.h> +#include <QtCore/qstring.h> +#include <QtCore/qstringlist.h> +#include <QtCore/qdatetime.h> +#include <QtCore/qmutex.h> +#include <QtCore/qbytearray.h> + +#include <QtMultimedia/qaudio.h> +#include <QtMultimedia/qaudiodevice.h> +#include <private/qaudiosystem_p.h> + +#include <qwindowsresampler_p.h> + +struct IMMDevice; +struct IAudioClient; +struct IAudioCaptureClient; + +QT_BEGIN_NAMESPACE +class QTimer; + +class QWindowsAudioSource : public QPlatformAudioSource +{ + Q_OBJECT +public: + QWindowsAudioSource(ComPtr<IMMDevice> device, QObject *parent); + ~QWindowsAudioSource(); + + qint64 read(char* data, qint64 len); + + void setFormat(const QAudioFormat& fmt) override; + QAudioFormat format() const override; + QIODevice* start() override; + void start(QIODevice* device) override; + void stop() override; + void reset() override; + void suspend() override; + void resume() override; + qsizetype bytesReady() const override; + void setBufferSize(qsizetype value) override; + qsizetype bufferSize() const override; + qint64 processedUSecs() const override; + QAudio::Error error() const override; + QAudio::State state() const override; + void setVolume(qreal volume) override; + qreal volume() const override; + +private: + void deviceStateChange(QAudio::State state, QAudio::Error error); + void pullCaptureClient(); + void schedulePull(); + QByteArray readCaptureClientBuffer(); + + QTimer *m_timer = nullptr; + ComPtr<IMMDevice> m_device; + ComPtr<IAudioClient> m_audioClient; + ComPtr<IAudioCaptureClient> m_captureClient; + QWindowsResampler m_resampler; + int m_bufferSize = 0; + qreal m_volume = 1.0; + + QIODevice* m_ourSink = nullptr; + QIODevice* m_clientSink = nullptr; + QAudioFormat m_format; + QAudio::Error m_errorState = QAudio::NoError; + QAudio::State m_deviceState = QAudio::StoppedState; + + QByteArray m_clientBufferResidue; + + bool open(); + void close(); +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/multimedia/windows/qwindowsaudioutils.cpp b/src/multimedia/windows/qwindowsaudioutils.cpp new file mode 100644 index 000000000..dce4a2135 --- /dev/null +++ b/src/multimedia/windows/qwindowsaudioutils.cpp @@ -0,0 +1,208 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwindowsaudioutils_p.h" +#include "qwindowsmediafoundation_p.h" +#include "qdebug.h" +#include "ks.h" +#include "ksmedia.h" + +#include <audioclient.h> + +QT_BEGIN_NAMESPACE + +static QAudioFormat::AudioChannelPosition channelFormatMap[] = + { QAudioFormat::FrontLeft // SPEAKER_FRONT_LEFT (0x1) + , QAudioFormat::FrontRight // SPEAKER_FRONT_RIGHT (0x2) + , QAudioFormat::FrontCenter // SPEAKER_FRONT_CENTER (0x4) + , QAudioFormat::LFE // SPEAKER_LOW_FREQUENCY (0x8) + , QAudioFormat::BackLeft // SPEAKER_BACK_LEFT (0x10) + , QAudioFormat::BackRight // SPEAKER_BACK_RIGHT (0x20) + , QAudioFormat::FrontLeftOfCenter // SPEAKER_FRONT_LEFT_OF_CENTER (0x40) + , QAudioFormat::FrontRightOfCenter// SPEAKER_FRONT_RIGHT_OF_CENTER (0x80) + , QAudioFormat::BackCenter // SPEAKER_BACK_CENTER (0x100) + , QAudioFormat::SideLeft // SPEAKER_SIDE_LEFT (0x200) + , QAudioFormat::SideRight // SPEAKER_SIDE_RIGHT (0x400) + , QAudioFormat::TopCenter // SPEAKER_TOP_CENTER (0x800) + , QAudioFormat::TopFrontLeft // SPEAKER_TOP_FRONT_LEFT (0x1000) + , QAudioFormat::TopFrontCenter // SPEAKER_TOP_FRONT_CENTER (0x2000) + , QAudioFormat::TopFrontRight // SPEAKER_TOP_FRONT_RIGHT (0x4000) + , QAudioFormat::TopBackLeft // SPEAKER_TOP_BACK_LEFT (0x8000) + , QAudioFormat::TopBackCenter // SPEAKER_TOP_BACK_CENTER (0x10000) + , QAudioFormat::TopBackRight // SPEAKER_TOP_BACK_RIGHT (0x20000) + }; + +QAudioFormat::ChannelConfig QWindowsAudioUtils::maskToChannelConfig(UINT32 mask, int count) +{ + quint32 config = 0; + int set = 0; + for (auto c : channelFormatMap) { + if (mask & 1) { + config |= QAudioFormat::channelConfig(c); + ++set; + } + if (set >= count) + break; + mask >>= 1; + } + return QAudioFormat::ChannelConfig(config); +} + +static UINT32 channelConfigToMask(QAudioFormat::ChannelConfig config) +{ + UINT32 mask = 0; + quint32 i = 0; + for (auto c : channelFormatMap) { + if (config & QAudioFormat::channelConfig(c)) + mask |= 1 << i; + ++i; + } + return mask; +} + +bool QWindowsAudioUtils::formatToWaveFormatExtensible(const QAudioFormat &format, WAVEFORMATEXTENSIBLE &wfx) +{ + if (!format.isValid()) + return false; + + wfx.Format.nSamplesPerSec = format.sampleRate(); + wfx.Format.wBitsPerSample = wfx.Samples.wValidBitsPerSample = format.bytesPerSample()*8; + wfx.Format.nChannels = format.channelCount(); + wfx.Format.nBlockAlign = (wfx.Format.wBitsPerSample / 8) * wfx.Format.nChannels; + wfx.Format.nAvgBytesPerSec = wfx.Format.nBlockAlign * wfx.Format.nSamplesPerSec; + wfx.Format.cbSize = 0; + + if (format.sampleFormat() == QAudioFormat::Float) { + wfx.Format.wFormatTag = WAVE_FORMAT_IEEE_FLOAT; + wfx.SubFormat = KSDATAFORMAT_SUBTYPE_IEEE_FLOAT; + } else { + wfx.Format.wFormatTag = WAVE_FORMAT_PCM; + wfx.SubFormat = KSDATAFORMAT_SUBTYPE_PCM; + } + + if (format.channelCount() > 2) { + wfx.Format.wFormatTag = WAVE_FORMAT_EXTENSIBLE; + wfx.Format.cbSize = 22; + wfx.dwChannelMask = format.channelConfig() == QAudioFormat::ChannelConfigUnknown ? KSAUDIO_SPEAKER_DIRECTOUT + : DWORD(format.channelConfig()); + } + + return true; +} + +QAudioFormat QWindowsAudioUtils::waveFormatExToFormat(const WAVEFORMATEX &in) +{ + QAudioFormat out; + out.setSampleRate(in.nSamplesPerSec); + out.setChannelCount(in.nChannels); + if (in.wFormatTag == WAVE_FORMAT_PCM) { + if (in.wBitsPerSample == 8) + out.setSampleFormat(QAudioFormat::UInt8); + else if (in.wBitsPerSample == 16) + out.setSampleFormat(QAudioFormat::Int16); + else if (in.wBitsPerSample == 32) + out.setSampleFormat(QAudioFormat::Int32); + } else if (in.wFormatTag == WAVE_FORMAT_EXTENSIBLE) { + if (in.cbSize >= 22) { + auto wfe = reinterpret_cast<const WAVEFORMATEXTENSIBLE &>(in); + if (wfe.SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT) + out.setSampleFormat(QAudioFormat::Float); + if (qPopulationCount(wfe.dwChannelMask) >= in.nChannels) + out.setChannelConfig(maskToChannelConfig(wfe.dwChannelMask, in.nChannels)); + } + } else if (in.wFormatTag == WAVE_FORMAT_IEEE_FLOAT) { + out.setSampleFormat(QAudioFormat::Float); + } + + return out; +} + +QAudioFormat QWindowsAudioUtils::mediaTypeToFormat(IMFMediaType *mediaType) +{ + QAudioFormat format; + if (!mediaType) + return format; + + UINT32 val = 0; + if (SUCCEEDED(mediaType->GetUINT32(MF_MT_AUDIO_NUM_CHANNELS, &val))) { + format.setChannelCount(int(val)); + } else { + qWarning() << "Could not determine channel count from IMFMediaType"; + return {}; + } + + if (SUCCEEDED(mediaType->GetUINT32(MF_MT_AUDIO_CHANNEL_MASK, &val))) { + if (int(qPopulationCount(val)) >= format.channelCount()) + format.setChannelConfig(maskToChannelConfig(val, format.channelCount())); + } + + if (SUCCEEDED(mediaType->GetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, &val))) { + format.setSampleRate(int(val)); + } + UINT32 bitsPerSample = 0; + mediaType->GetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, &bitsPerSample); + + GUID subType; + if (SUCCEEDED(mediaType->GetGUID(MF_MT_SUBTYPE, &subType))) { + if (subType == MFAudioFormat_Float) { + format.setSampleFormat(QAudioFormat::Float); + } else if (bitsPerSample == 8) { + format.setSampleFormat(QAudioFormat::UInt8); + } else if (bitsPerSample == 16) { + format.setSampleFormat(QAudioFormat::Int16); + } else if (bitsPerSample == 32){ + format.setSampleFormat(QAudioFormat::Int32); + } + } + return format; +} + +ComPtr<IMFMediaType> QWindowsAudioUtils::formatToMediaType(QWindowsMediaFoundation &wmf, const QAudioFormat &format) +{ + ComPtr<IMFMediaType> mediaType; + + if (!format.isValid()) + return mediaType; + + wmf.mfCreateMediaType(mediaType.GetAddressOf()); + + mediaType->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Audio); + if (format.sampleFormat() == QAudioFormat::Float) { + mediaType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_Float); + } else { + mediaType->SetGUID(MF_MT_SUBTYPE, MFAudioFormat_PCM); + } + + mediaType->SetUINT32(MF_MT_AUDIO_NUM_CHANNELS, UINT32(format.channelCount())); + if (format.channelConfig() != QAudioFormat::ChannelConfigUnknown) + mediaType->SetUINT32(MF_MT_AUDIO_CHANNEL_MASK, channelConfigToMask(format.channelConfig())); + mediaType->SetUINT32(MF_MT_AUDIO_SAMPLES_PER_SECOND, UINT32(format.sampleRate())); + auto alignmentBlock = UINT32(format.bytesPerFrame()); + mediaType->SetUINT32(MF_MT_AUDIO_BLOCK_ALIGNMENT, alignmentBlock); + auto avgBytesPerSec = UINT32(format.sampleRate() * format.bytesPerFrame()); + mediaType->SetUINT32(MF_MT_AUDIO_AVG_BYTES_PER_SECOND, avgBytesPerSec); + mediaType->SetUINT32(MF_MT_AUDIO_BITS_PER_SAMPLE, UINT32(format.bytesPerSample()*8)); + mediaType->SetUINT32(MF_MT_ALL_SAMPLES_INDEPENDENT, TRUE); + + return mediaType; +} + +std::optional<quint32> QWindowsAudioUtils::audioClientFramesInUse(IAudioClient *client) +{ + Q_ASSERT(client); + UINT32 framesPadding = 0; + if (SUCCEEDED(client->GetCurrentPadding(&framesPadding))) + return framesPadding; + return {}; +} + +std::optional<quint32> QWindowsAudioUtils::audioClientFramesAllocated(IAudioClient *client) +{ + Q_ASSERT(client); + UINT32 bufferFrameCount = 0; + if (SUCCEEDED(client->GetBufferSize(&bufferFrameCount))) + return bufferFrameCount; + return {}; +} + +QT_END_NAMESPACE diff --git a/src/multimedia/windows/qwindowsaudioutils_p.h b/src/multimedia/windows/qwindowsaudioutils_p.h new file mode 100644 index 000000000..9382f372f --- /dev/null +++ b/src/multimedia/windows/qwindowsaudioutils_p.h @@ -0,0 +1,45 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWINDOWSAUDIOUTILS_H +#define QWINDOWSAUDIOUTILS_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qaudioformat.h> +#include <QtCore/qt_windows.h> +#include <private/qcomptr_p.h> +#include <mmreg.h> + +#include <optional> + +struct IAudioClient; +struct IMFMediaType; + +QT_BEGIN_NAMESPACE + +class QWindowsMediaFoundation; + +namespace QWindowsAudioUtils +{ + bool formatToWaveFormatExtensible(const QAudioFormat &format, WAVEFORMATEXTENSIBLE &wfx); + QAudioFormat waveFormatExToFormat(const WAVEFORMATEX &in); + Q_MULTIMEDIA_EXPORT QAudioFormat mediaTypeToFormat(IMFMediaType *mediaType); + ComPtr<IMFMediaType> formatToMediaType(QWindowsMediaFoundation &, const QAudioFormat &format); + QAudioFormat::ChannelConfig maskToChannelConfig(UINT32 mask, int count); + std::optional<quint32> audioClientFramesInUse(IAudioClient *client); + std::optional<quint32> audioClientFramesAllocated(IAudioClient *client); +} + +QT_END_NAMESPACE + +#endif // QWINDOWSAUDIOUTILS_H diff --git a/src/multimedia/windows/qwindowsmediadevices.cpp b/src/multimedia/windows/qwindowsmediadevices.cpp new file mode 100644 index 000000000..c622a721c --- /dev/null +++ b/src/multimedia/windows/qwindowsmediadevices.cpp @@ -0,0 +1,345 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwindowsmediadevices_p.h" +#include "qmediadevices.h" +#include "qvarlengtharray.h" + +#include "qwindowsaudiosource_p.h" +#include "qwindowsaudiosink_p.h" +#include "qwindowsaudiodevice_p.h" +#include "qcomtaskresource_p.h" + +#include <mmsystem.h> +#include <mmddk.h> +#include <mfobjects.h> +#include <mfidl.h> +#include <mferror.h> +#include <mmdeviceapi.h> +#include <qwindowsmfdefs_p.h> + +#include <QtCore/qmap.h> +#include <private/qcomobject_p.h> +#include <private/qsystemerror_p.h> + +QT_BEGIN_NAMESPACE + +class CMMNotificationClient : public QComObject<IMMNotificationClient> +{ + ComPtr<IMMDeviceEnumerator> m_enumerator; + QWindowsMediaDevices *m_windowsMediaDevices; + QMap<QString, DWORD> m_deviceState; + +public: + CMMNotificationClient(QWindowsMediaDevices *windowsMediaDevices, + ComPtr<IMMDeviceEnumerator> enumerator, + QMap<QString, DWORD> &&deviceState) + : m_enumerator(enumerator), + m_windowsMediaDevices(windowsMediaDevices), + m_deviceState(deviceState) + {} + + HRESULT STDMETHODCALLTYPE OnDefaultDeviceChanged(EDataFlow flow, ERole role, LPCWSTR) override + { + if (role == ERole::eMultimedia) + emitAudioDevicesChanged(flow); + + return S_OK; + } + + HRESULT STDMETHODCALLTYPE OnDeviceAdded(LPCWSTR deviceID) override + { + auto it = m_deviceState.find(QString::fromWCharArray(deviceID)); + if (it == std::end(m_deviceState)) { + m_deviceState.insert(QString::fromWCharArray(deviceID), DEVICE_STATE_ACTIVE); + emitAudioDevicesChanged(deviceID); + } + + return S_OK; + }; + + HRESULT STDMETHODCALLTYPE OnDeviceRemoved(LPCWSTR deviceID) override + { + auto key = QString::fromWCharArray(deviceID); + auto it = m_deviceState.find(key); + if (it != std::end(m_deviceState)) { + if (it.value() == DEVICE_STATE_ACTIVE) + emitAudioDevicesChanged(deviceID); + m_deviceState.remove(key); + } + + return S_OK; + } + + HRESULT STDMETHODCALLTYPE OnDeviceStateChanged(LPCWSTR deviceID, DWORD newState) override + { + if (auto it = m_deviceState.find(QString::fromWCharArray(deviceID)); it != std::end(m_deviceState)) { + // If either the old state or the new state is active emit device change + if ((it.value() == DEVICE_STATE_ACTIVE) != (newState == DEVICE_STATE_ACTIVE)) { + emitAudioDevicesChanged(deviceID); + } + it.value() = newState; + } + + return S_OK; + } + + HRESULT STDMETHODCALLTYPE OnPropertyValueChanged(LPCWSTR, const PROPERTYKEY) override + { + return S_OK; + } + + void emitAudioDevicesChanged(EDataFlow flow) + { + // windowsMediaDevice may be deleted as we are executing the callback + if (flow == EDataFlow::eCapture) { + emit m_windowsMediaDevices->audioInputsChanged(); + } else if (flow == EDataFlow::eRender) { + emit m_windowsMediaDevices->audioOutputsChanged(); + } + } + + void emitAudioDevicesChanged(LPCWSTR deviceID) + { + ComPtr<IMMDevice> device; + ComPtr<IMMEndpoint> endpoint; + EDataFlow flow; + + if (SUCCEEDED(m_enumerator->GetDevice(deviceID, device.GetAddressOf())) + && SUCCEEDED(device->QueryInterface(__uuidof(IMMEndpoint), (void**)endpoint.GetAddressOf())) + && SUCCEEDED(endpoint->GetDataFlow(&flow))) + { + emitAudioDevicesChanged(flow); + } + } + +private: + // Destructor is not public. Caller should call Release. + ~CMMNotificationClient() override = default; +}; + +QWindowsMediaDevices::QWindowsMediaDevices() + : QPlatformMediaDevices() +{ + auto hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, + CLSCTX_INPROC_SERVER,__uuidof(IMMDeviceEnumerator), + (void**)&m_deviceEnumerator); + + if (FAILED(hr)) { + qWarning("Failed to instantiate IMMDeviceEnumerator (%s)." + "Audio device change notification will be disabled", + qPrintable(QSystemError::windowsComString(hr))); + return; + } + + QMap<QString, DWORD> devState; + ComPtr<IMMDeviceCollection> devColl; + UINT count = 0; + + if (SUCCEEDED(m_deviceEnumerator->EnumAudioEndpoints(EDataFlow::eAll, DEVICE_STATEMASK_ALL, devColl.GetAddressOf())) + && SUCCEEDED(devColl->GetCount(&count))) + { + for (UINT i = 0; i < count; i++) { + ComPtr<IMMDevice> device; + DWORD state = 0; + QComTaskResource<WCHAR> id; + + if (SUCCEEDED(devColl->Item(i, device.GetAddressOf())) + && SUCCEEDED(device->GetState(&state)) + && SUCCEEDED(device->GetId(id.address()))) { + devState.insert(QString::fromWCharArray(id.get()), state); + } + } + } + + + m_notificationClient = makeComObject<CMMNotificationClient>(this, m_deviceEnumerator, std::move(devState)); + m_deviceEnumerator->RegisterEndpointNotificationCallback(m_notificationClient.Get()); +} + +QWindowsMediaDevices::~QWindowsMediaDevices() +{ + if (m_deviceEnumerator) { + // Note: Calling UnregisterEndpointNotificationCallback after CoUninitialize + // will abruptly terminate application, preventing remaining destructors from + // being called (QTBUG-120198). + m_deviceEnumerator->UnregisterEndpointNotificationCallback(m_notificationClient.Get()); + } + if (m_warmUpAudioClient) { + HRESULT hr = m_warmUpAudioClient->Stop(); + if (FAILED(hr)) { + qWarning() << "Failed to stop audio engine" << hr; + } + } + + m_deviceEnumerator.Reset(); + m_notificationClient.Reset(); + m_warmUpAudioClient.Reset(); +} + +QList<QAudioDevice> QWindowsMediaDevices::availableDevices(QAudioDevice::Mode mode) const +{ + if (!m_deviceEnumerator) + return {}; + + const auto audioOut = mode == QAudioDevice::Output; + + const auto defaultAudioDeviceID = [this, audioOut]{ + const auto dataFlow = audioOut ? EDataFlow::eRender : EDataFlow::eCapture; + ComPtr<IMMDevice> dev; + QComTaskResource<WCHAR> id; + QString sid; + + if (SUCCEEDED(m_deviceEnumerator->GetDefaultAudioEndpoint(dataFlow, ERole::eMultimedia, dev.GetAddressOf()))) { + if (dev && SUCCEEDED(dev->GetId(id.address()))) { + sid = QString::fromWCharArray(id.get()); + } + } + return sid.toUtf8(); + }(); + + QList<QAudioDevice> devices; + + auto waveDevices = audioOut ? waveOutGetNumDevs() : waveInGetNumDevs(); + + for (auto waveID = 0u; waveID < waveDevices; waveID++) { + auto wave = IntToPtr(waveID); + auto waveMessage = [wave, audioOut](UINT msg, auto p0, auto p1) { + return audioOut ? waveOutMessage((HWAVEOUT)wave, msg, (DWORD_PTR)p0, (DWORD_PTR)p1) + : waveInMessage((HWAVEIN)wave, msg, (DWORD_PTR)p0, (DWORD_PTR)p1); + }; + + size_t len = 0; + if (waveMessage(DRV_QUERYFUNCTIONINSTANCEIDSIZE, &len, 0) != MMSYSERR_NOERROR) + continue; + + QVarLengthArray<WCHAR> id(len); + if (waveMessage(DRV_QUERYFUNCTIONINSTANCEID, id.data(), len) != MMSYSERR_NOERROR) + continue; + + ComPtr<IMMDevice> device; + ComPtr<IPropertyStore> props; + if (FAILED(m_deviceEnumerator->GetDevice(id.data(), device.GetAddressOf())) + || FAILED(device->OpenPropertyStore(STGM_READ, props.GetAddressOf()))) { + continue; + } + + PROPVARIANT varName; + PropVariantInit(&varName); + + if (SUCCEEDED(props->GetValue(QMM_PKEY_Device_FriendlyName, &varName))) { + auto description = QString::fromWCharArray(varName.pwszVal); + auto strID = QString::fromWCharArray(id.data()).toUtf8(); + + auto dev = new QWindowsAudioDeviceInfo(strID, device, waveID, description, mode); + dev->isDefault = strID == defaultAudioDeviceID; + + devices.append(dev->create()); + } + PropVariantClear(&varName); + } + + return devices; +} + +QList<QAudioDevice> QWindowsMediaDevices::audioInputs() const +{ + return availableDevices(QAudioDevice::Input); +} + +QList<QAudioDevice> QWindowsMediaDevices::audioOutputs() const +{ + return availableDevices(QAudioDevice::Output); +} + +QPlatformAudioSource *QWindowsMediaDevices::createAudioSource(const QAudioDevice &deviceInfo, + QObject *parent) +{ + const auto *devInfo = static_cast<const QWindowsAudioDeviceInfo *>(deviceInfo.handle()); + return new QWindowsAudioSource(devInfo->immDev(), parent); +} + +QPlatformAudioSink *QWindowsMediaDevices::createAudioSink(const QAudioDevice &deviceInfo, + QObject *parent) +{ + const auto *devInfo = static_cast<const QWindowsAudioDeviceInfo *>(deviceInfo.handle()); + return new QWindowsAudioSink(devInfo->immDev(), parent); +} + +static bool isPrepareAudioEnabled() +{ + static bool isDisableAudioPrepareSet = false; + static const int disableAudioPrepare = + qEnvironmentVariableIntValue("QT_DISABLE_AUDIO_PREPARE", &isDisableAudioPrepareSet); + + return !isDisableAudioPrepareSet || disableAudioPrepare == 0; +} + +void QWindowsMediaDevices::prepareAudio() +{ + if (!isPrepareAudioEnabled()) + return; + + if (m_isAudioClientWarmedUp.exchange(true)) + return; + + ComPtr<IMMDeviceEnumerator> deviceEnumerator; + HRESULT hr = CoCreateInstance(__uuidof(MMDeviceEnumerator), nullptr, CLSCTX_ALL, + __uuidof(IMMDeviceEnumerator), + reinterpret_cast<void **>(deviceEnumerator.GetAddressOf())); + if (FAILED(hr)) { + qWarning() << "Failed to create device enumerator" << hr; + return; + } + + ComPtr<IMMDevice> device; + hr = deviceEnumerator->GetDefaultAudioEndpoint(eRender, eConsole, device.GetAddressOf()); + if (FAILED(hr)) { + if (hr != E_NOTFOUND) + qWarning() << "Failed to retrieve default audio endpoint" << hr; + return; + } + + hr = device->Activate(__uuidof(IAudioClient3), CLSCTX_ALL, nullptr, + reinterpret_cast<void **>(m_warmUpAudioClient.GetAddressOf())); + if (FAILED(hr)) { + qWarning() << "Failed to activate audio engine" << hr; + return; + } + + QComTaskResource<WAVEFORMATEX> deviceFormat; + UINT32 currentPeriodInFrames = 0; + hr = m_warmUpAudioClient->GetCurrentSharedModeEnginePeriod(deviceFormat.address(), + ¤tPeriodInFrames); + if (FAILED(hr)) { + qWarning() << "Failed to retrieve the current format and periodicity of the audio engine" + << hr; + return; + } + + UINT32 defaultPeriodInFrames = 0; + UINT32 fundamentalPeriodInFrames = 0; + UINT32 minPeriodInFrames = 0; + UINT32 maxPeriodInFrames = 0; + hr = m_warmUpAudioClient->GetSharedModeEnginePeriod(deviceFormat.get(), &defaultPeriodInFrames, + &fundamentalPeriodInFrames, + &minPeriodInFrames, &maxPeriodInFrames); + if (FAILED(hr)) { + qWarning() << "Failed to retrieve the range of periodicities supported by the audio engine" + << hr; + return; + } + + hr = m_warmUpAudioClient->InitializeSharedAudioStream( + AUDCLNT_SHAREMODE_SHARED, minPeriodInFrames, deviceFormat.get(), nullptr); + if (FAILED(hr)) { + qWarning() << "Failed to initialize audio engine stream" << hr; + return; + } + + hr = m_warmUpAudioClient->Start(); + if (FAILED(hr)) + qWarning() << "Failed to start audio engine" << hr; +} + +QT_END_NAMESPACE diff --git a/src/multimedia/windows/qwindowsmediadevices_p.h b/src/multimedia/windows/qwindowsmediadevices_p.h new file mode 100644 index 000000000..c82be8a0d --- /dev/null +++ b/src/multimedia/windows/qwindowsmediadevices_p.h @@ -0,0 +1,65 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWINDOWSMEDIADEVICES_H +#define QWINDOWSMEDIADEVICES_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qplatformmediadevices_p.h> +#include <private/qcomptr_p.h> +#include <QtCore/private/qfunctions_win_p.h> +#include <private/qwindowsmediafoundation_p.h> + +#include <qaudiodevice.h> + +struct IAudioClient3; +struct IMMDeviceEnumerator; + +QT_BEGIN_NAMESPACE + +class QWindowsEngine; +class CMMNotificationClient; + +class QWindowsMediaDevices : public QPlatformMediaDevices +{ +public: + QWindowsMediaDevices(); + virtual ~QWindowsMediaDevices(); + + QList<QAudioDevice> audioInputs() const override; + QList<QAudioDevice> audioOutputs() const override; + QPlatformAudioSource *createAudioSource(const QAudioDevice &deviceInfo, + QObject *parent) override; + QPlatformAudioSink *createAudioSink(const QAudioDevice &deviceInfo, + QObject *parent) override; + + void prepareAudio() override; + +private: + QComHelper m_comRuntime; + QMFRuntimeInit m_wmfRuntime{ QWindowsMediaFoundation::instance() }; + QList<QAudioDevice> availableDevices(QAudioDevice::Mode mode) const; + + ComPtr<IMMDeviceEnumerator> m_deviceEnumerator; + ComPtr<CMMNotificationClient> m_notificationClient; + // The "warm-up" audio client is required to run in the background in order to keep audio engine + // ready for audio output immediately after creating any other subsequent audio client. + ComPtr<IAudioClient3> m_warmUpAudioClient; + std::atomic_bool m_isAudioClientWarmedUp = false; + + friend CMMNotificationClient; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/multimedia/windows/qwindowsmediafoundation.cpp b/src/multimedia/windows/qwindowsmediafoundation.cpp new file mode 100644 index 000000000..7094c9551 --- /dev/null +++ b/src/multimedia/windows/qwindowsmediafoundation.cpp @@ -0,0 +1,70 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwindowsmediafoundation_p.h" +#include <QtCore/qdebug.h> + +QT_BEGIN_NAMESPACE + +namespace { + +Q_GLOBAL_STATIC(QWindowsMediaFoundation, s_wmf); + +template <typename T> +bool setProcAddress(QSystemLibrary &lib, T &f, const char name[]) +{ + f = reinterpret_cast<T>(lib.resolve(name)); + return static_cast<bool>(f); +} + +} // namespace + +QWindowsMediaFoundation *QWindowsMediaFoundation::instance() +{ + if (s_wmf->valid()) + return s_wmf; + + return nullptr; +} + +QWindowsMediaFoundation::QWindowsMediaFoundation() +{ + if (!m_mfplat.load(false)) + return; + + m_valid = setProcAddress(m_mfplat, mfStartup, "MFStartup") + && setProcAddress(m_mfplat, mfShutdown, "MFShutdown") + && setProcAddress(m_mfplat, mfCreateMediaType, "MFCreateMediaType") + && setProcAddress(m_mfplat, mfCreateMemoryBuffer, "MFCreateMemoryBuffer") + && setProcAddress(m_mfplat, mfCreateSample, "MFCreateSample"); + + Q_ASSERT(m_valid); // If it is not valid at this point, we have a programming bug +} + +QWindowsMediaFoundation::~QWindowsMediaFoundation() = default; + +bool QWindowsMediaFoundation::valid() const +{ + return m_valid; +} + +QMFRuntimeInit::QMFRuntimeInit(QWindowsMediaFoundation *wmf) + : m_wmf{ wmf }, m_initResult{ wmf ? m_wmf->mfStartup(MF_VERSION, MFSTARTUP_FULL) : E_FAIL } +{ + if (m_initResult != S_OK) + qErrnoWarning(m_initResult, "Failed to initialize Windows Media Foundation"); +} + +QMFRuntimeInit::~QMFRuntimeInit() +{ + // According to documentation MFShutdown should be called + // also when MFStartup failed. This is wrong. + if (FAILED(m_initResult)) + return; + + const HRESULT hr = m_wmf->mfShutdown(); + if (hr != S_OK) + qErrnoWarning(hr, "Failed to shut down Windows Media Foundation"); +} + +QT_END_NAMESPACE diff --git a/src/multimedia/windows/qwindowsmediafoundation_p.h b/src/multimedia/windows/qwindowsmediafoundation_p.h new file mode 100644 index 000000000..6dabc261e --- /dev/null +++ b/src/multimedia/windows/qwindowsmediafoundation_p.h @@ -0,0 +1,59 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWINDOWSMEDIAFOUNDATION_H +#define QWINDOWSMEDIAFOUNDATION_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qtmultimediaglobal_p.h> +#include <QtCore/private/qsystemlibrary_p.h> +#include <mfapi.h> + +QT_BEGIN_NAMESPACE + +class QWindowsMediaFoundation +{ +public: + static QWindowsMediaFoundation *instance(); + + QWindowsMediaFoundation(); + ~QWindowsMediaFoundation(); + + bool valid() const; + + decltype(&::MFStartup) mfStartup = nullptr; + decltype(&::MFShutdown) mfShutdown = nullptr; + decltype(&::MFCreateMediaType) mfCreateMediaType = nullptr; + decltype(&::MFCreateMemoryBuffer) mfCreateMemoryBuffer = nullptr; + decltype(&::MFCreateSample) mfCreateSample = nullptr; + +private: + QSystemLibrary m_mfplat{ QStringLiteral("Mfplat.dll") }; + bool m_valid = false; +}; + +class QMFRuntimeInit +{ + Q_DISABLE_COPY_MOVE(QMFRuntimeInit) +public: + QMFRuntimeInit(QWindowsMediaFoundation *wmf); + ~QMFRuntimeInit(); + +private: + QWindowsMediaFoundation *m_wmf; + HRESULT m_initResult; +}; + +QT_END_NAMESPACE + +#endif // QWINDOWSMEDIAFOUNDATION_H diff --git a/src/multimedia/windows/qwindowsmfdefs.cpp b/src/multimedia/windows/qwindowsmfdefs.cpp new file mode 100644 index 000000000..ef8c99922 --- /dev/null +++ b/src/multimedia/windows/qwindowsmfdefs.cpp @@ -0,0 +1,26 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwindowsmfdefs_p.h" + +const GUID QMM_MFTranscodeContainerType_ADTS = {0x132fd27d, 0x0f02, 0x43de, {0xa3, 0x01, 0x38, 0xfb, 0xbb, 0xb3, 0x83, 0x4e}}; +const GUID QMM_MFTranscodeContainerType_ASF = {0x430f6f6e, 0xb6bf, 0x4fc1, {0xa0, 0xbd, 0x9e, 0xe4, 0x6e, 0xee, 0x2a, 0xfb}}; +const GUID QMM_MFTranscodeContainerType_AVI = {0x7edfe8af, 0x402f, 0x4d76, {0xa3, 0x3c, 0x61, 0x9f, 0xd1, 0x57, 0xd0, 0xf1}}; +const GUID QMM_MFTranscodeContainerType_FLAC = {0x31344aa3, 0x05a9, 0x42b5, {0x90, 0x1b, 0x8e, 0x9d, 0x42, 0x57, 0xf7, 0x5e}}; +const GUID QMM_MFTranscodeContainerType_MP3 = {0xe438b912, 0x83f1, 0x4de6, {0x9e, 0x3a, 0x9f, 0xfb, 0xc6, 0xdd, 0x24, 0xd1}}; +const GUID QMM_MFTranscodeContainerType_MPEG4 = {0xdc6cd05d, 0xb9d0, 0x40ef, {0xbd, 0x35, 0xfa, 0x62, 0x2c, 0x1a, 0xb2, 0x8a}}; +const GUID QMM_MFTranscodeContainerType_WAVE = {0x64c3453c, 0x0f26, 0x4741, {0xbe, 0x63, 0x87, 0xbd, 0xf8, 0xbb, 0x93, 0x5b}}; + +const GUID QMM_MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID = {0x8ac3587a, 0x4ae7, 0x42d8, {0x99, 0xe0, 0x0a, 0x60, 0x13, 0xee, 0xf9, 0x0f}}; +const GUID QMM_MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_AUDCAP_GUID = {0x14dd9a1c, 0x7cff, 0x41be, {0xb1, 0xb9, 0xba, 0x1a, 0xc6, 0xec, 0xb5, 0x71}}; +const GUID QMM_MF_TRANSCODE_CONTAINERTYPE = {0x150ff23f, 0x4abc, 0x478b, {0xac, 0x4f, 0xe1, 0x91, 0x6f, 0xba, 0x1c, 0xca}}; + +const GUID QMM_MF_SD_STREAM_NAME = {0x4f1b099d, 0xd314, 0x41e5, {0xa7, 0x81, 0x7f, 0xef, 0xaa, 0x4c, 0x50, 0x1f}}; +const GUID QMM_MF_SD_LANGUAGE = {0xaf2180, 0xbdc2, 0x423c, {0xab, 0xca, 0xf5, 0x3, 0x59, 0x3b, 0xc1, 0x21}}; + +const GUID QMM_KSCATEGORY_VIDEO_CAMERA = {0xe5323777, 0xf976, 0x4f5b, {0x9b, 0x55, 0xb9, 0x46, 0x99, 0xc4, 0x6e, 0x44}}; +const GUID QMM_KSCATEGORY_SENSOR_CAMERA = {0x24e552d7, 0x6523, 0x47f7, {0xa6, 0x47, 0xd3, 0x46, 0x5b, 0xf1, 0xf5, 0xca}}; +const GUID QMM_MR_POLICY_VOLUME_SERVICE = {0x1abaa2ac, 0x9d3b, 0x47c6, {0xab, 0x48, 0xc5, 0x95, 0x6, 0xde, 0x78, 0x4d}}; + +const PROPERTYKEY QMM_PKEY_Device_FriendlyName = {{0xa45c254e, 0xdf1c, 0x4efd, {0x80, 0x20, 0x67, 0xd1, 0x46, 0xa8, 0x50, 0xe0}}, 14}; + diff --git a/src/multimedia/windows/qwindowsmfdefs_p.h b/src/multimedia/windows/qwindowsmfdefs_p.h new file mode 100644 index 000000000..4bb90dbe3 --- /dev/null +++ b/src/multimedia/windows/qwindowsmfdefs_p.h @@ -0,0 +1,94 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWINDOWSMFDEFS_P_H +#define QWINDOWSMFDEFS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtmultimediaexports.h> +#include <d3d9.h> +#include <dxva2api.h> +#include <mfidl.h> + +// Stuff that is missing or incorrecty defined in MinGW. + +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MFTranscodeContainerType_ADTS; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MFTranscodeContainerType_ASF; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MFTranscodeContainerType_AVI; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MFTranscodeContainerType_FLAC; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MFTranscodeContainerType_MP3; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MFTranscodeContainerType_MPEG4; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MFTranscodeContainerType_WAVE; + +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_VIDCAP_GUID; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MF_DEVSOURCE_ATTRIBUTE_SOURCE_TYPE_AUDCAP_GUID; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MF_TRANSCODE_CONTAINERTYPE; + +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MF_SD_STREAM_NAME; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MF_SD_LANGUAGE; + +Q_MULTIMEDIA_EXPORT extern const GUID QMM_KSCATEGORY_VIDEO_CAMERA; +Q_MULTIMEDIA_EXPORT extern const GUID QMM_KSCATEGORY_SENSOR_CAMERA; + +Q_MULTIMEDIA_EXPORT extern const GUID QMM_MR_POLICY_VOLUME_SERVICE; + +Q_MULTIMEDIA_EXPORT extern const PROPERTYKEY QMM_PKEY_Device_FriendlyName; + +extern "C" HRESULT WINAPI MFCreateDeviceSource(IMFAttributes *pAttributes, IMFMediaSource **ppSource); + +#define QMM_MFSESSION_GETFULLTOPOLOGY_CURRENT 1 +#define QMM_PRESENTATION_CURRENT_POSITION 0x7fffffffffffffff +#define QMM_WININET_E_CANNOT_CONNECT ((HRESULT)0x80072EFDL) + +#ifndef __IMFVideoProcessor_INTERFACE_DEFINED__ +#define __IMFVideoProcessor_INTERFACE_DEFINED__ +DEFINE_GUID(IID_IMFVideoProcessor, 0x6AB0000C, 0xFECE, 0x4d1f, 0xA2,0xAC, 0xA9,0x57,0x35,0x30,0x65,0x6E); +MIDL_INTERFACE("6AB0000C-FECE-4d1f-A2AC-A9573530656E") +IMFVideoProcessor : public IUnknown +{ + virtual HRESULT STDMETHODCALLTYPE GetAvailableVideoProcessorModes(UINT *, GUID **) = 0; + virtual HRESULT STDMETHODCALLTYPE GetVideoProcessorCaps(LPGUID, DXVA2_VideoProcessorCaps *) = 0; + virtual HRESULT STDMETHODCALLTYPE GetVideoProcessorMode(LPGUID) = 0; + virtual HRESULT STDMETHODCALLTYPE SetVideoProcessorMode(LPGUID) = 0; + virtual HRESULT STDMETHODCALLTYPE GetProcAmpRange(DWORD, DXVA2_ValueRange *) = 0; + virtual HRESULT STDMETHODCALLTYPE GetProcAmpValues(DWORD, DXVA2_ProcAmpValues *) = 0; + virtual HRESULT STDMETHODCALLTYPE SetProcAmpValues(DWORD, DXVA2_ProcAmpValues *) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFilteringRange(DWORD, DXVA2_ValueRange *) = 0; + virtual HRESULT STDMETHODCALLTYPE GetFilteringValue(DWORD, DXVA2_Fixed32 *) = 0; + virtual HRESULT STDMETHODCALLTYPE SetFilteringValue(DWORD, DXVA2_Fixed32 *) = 0; + virtual HRESULT STDMETHODCALLTYPE GetBackgroundColor(COLORREF *) = 0; + virtual HRESULT STDMETHODCALLTYPE SetBackgroundColor(COLORREF) = 0; +}; +#ifdef __CRT_UUID_DECL +__CRT_UUID_DECL(IMFVideoProcessor, 0x6AB0000C, 0xFECE, 0x4d1f, 0xA2,0xAC, 0xA9,0x57,0x35,0x30,0x65,0x6E) +#endif +#endif // __IMFVideoProcessor_INTERFACE_DEFINED__ + +#ifndef __IMFSimpleAudioVolume_INTERFACE_DEFINED__ +#define __IMFSimpleAudioVolume_INTERFACE_DEFINED__ +DEFINE_GUID(IID_IMFSimpleAudioVolume, 0x089EDF13, 0xCF71, 0x4338, 0x8D,0x13, 0x9E,0x56,0x9D,0xBD,0xC3,0x19); +MIDL_INTERFACE("089EDF13-CF71-4338-8D13-9E569DBDC319") +IMFSimpleAudioVolume : public IUnknown +{ + virtual HRESULT STDMETHODCALLTYPE SetMasterVolume(float) = 0; + virtual HRESULT STDMETHODCALLTYPE GetMasterVolume(float *) = 0; + virtual HRESULT STDMETHODCALLTYPE SetMute(BOOL) = 0; + virtual HRESULT STDMETHODCALLTYPE GetMute(BOOL *) = 0; +}; +#ifdef __CRT_UUID_DECL +__CRT_UUID_DECL(IMFSimpleAudioVolume, 0x089EDF13, 0xCF71, 0x4338, 0x8D,0x13, 0x9E,0x56,0x9D,0xBD,0xC3,0x19) +#endif +#endif // __IMFSimpleAudioVolume_INTERFACE_DEFINED__ + +#endif // QWINDOWSMFDEFS_P_H + diff --git a/src/multimedia/windows/qwindowsmultimediautils.cpp b/src/multimedia/windows/qwindowsmultimediautils.cpp new file mode 100644 index 000000000..258cb3e96 --- /dev/null +++ b/src/multimedia/windows/qwindowsmultimediautils.cpp @@ -0,0 +1,215 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#if defined(WINVER) && WINVER < _WIN32_WINNT_WIN10 +# undef WINVER +#endif +#if !defined(WINVER) +# define WINVER _WIN32_WINNT_WIN10 // Enables newer audio formats. +#endif + +#include "qwindowsmultimediautils_p.h" + +#include <initguid.h> +#include <mfapi.h> +#include <mfidl.h> +#include <qwindowsmfdefs_p.h> +#include <system_error> + +QT_BEGIN_NAMESPACE + +QVideoFrameFormat::PixelFormat QWindowsMultimediaUtils::pixelFormatFromMediaSubtype(const GUID &subtype) +{ + if (subtype == MFVideoFormat_ARGB32) +#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN + return QVideoFrameFormat::Format_BGRA8888; +#else + return QVideoFrameFormat::Format_ARGB8888; +#endif + if (subtype == MFVideoFormat_RGB32) +#if Q_BYTE_ORDER == Q_LITTLE_ENDIAN + return QVideoFrameFormat::Format_BGRX8888; +#else + return QVideoFrameFormat::Format_XRGB8888; +#endif + if (subtype == MFVideoFormat_AYUV) + return QVideoFrameFormat::Format_AYUV; + if (subtype == MFVideoFormat_I420) + return QVideoFrameFormat::Format_YUV420P; + if (subtype == MFVideoFormat_UYVY) + return QVideoFrameFormat::Format_UYVY; + if (subtype == MFVideoFormat_YV12) + return QVideoFrameFormat::Format_YV12; + if (subtype == MFVideoFormat_NV12) + return QVideoFrameFormat::Format_NV12; + if (subtype == MFVideoFormat_YUY2) + return QVideoFrameFormat::Format_YUYV; + if (subtype == MFVideoFormat_P010) + return QVideoFrameFormat::Format_P010; + if (subtype == MFVideoFormat_P016) + return QVideoFrameFormat::Format_P016; + if (subtype == MFVideoFormat_L8) + return QVideoFrameFormat::Format_Y8; + if (subtype == MFVideoFormat_L16) + return QVideoFrameFormat::Format_Y16; + if (subtype == MFVideoFormat_MJPG) + return QVideoFrameFormat::Format_Jpeg; + + return QVideoFrameFormat::Format_Invalid; +} + +GUID QWindowsMultimediaUtils::videoFormatForCodec(QMediaFormat::VideoCodec codec) +{ + switch (codec) { + case QMediaFormat::VideoCodec::MPEG1: + return MFVideoFormat_MPG1; + case QMediaFormat::VideoCodec::MPEG2: + return MFVideoFormat_MPEG2; + case QMediaFormat::VideoCodec::MPEG4: + return MFVideoFormat_MP4V; + case QMediaFormat::VideoCodec::H264: + return MFVideoFormat_H264; + case QMediaFormat::VideoCodec::H265: + return MFVideoFormat_H265; + case QMediaFormat::VideoCodec::VP8: + return MFVideoFormat_VP80; + case QMediaFormat::VideoCodec::VP9: + return MFVideoFormat_VP90; + case QMediaFormat::VideoCodec::AV1: + return MFVideoFormat_AV1; + case QMediaFormat::VideoCodec::WMV: + return MFVideoFormat_WMV3; + case QMediaFormat::VideoCodec::MotionJPEG: + return MFVideoFormat_MJPG; + default: + return MFVideoFormat_H264; + } +} + +QMediaFormat::VideoCodec QWindowsMultimediaUtils::codecForVideoFormat(GUID format) +{ + if (format == MFVideoFormat_MPG1) + return QMediaFormat::VideoCodec::MPEG1; + if (format == MFVideoFormat_MPEG2) + return QMediaFormat::VideoCodec::MPEG2; + if (format == MFVideoFormat_MP4V + || format == MFVideoFormat_M4S2 + || format == MFVideoFormat_MP4S + || format == MFVideoFormat_MP43) + return QMediaFormat::VideoCodec::MPEG4; + if (format == MFVideoFormat_H264) + return QMediaFormat::VideoCodec::H264; + if (format == MFVideoFormat_H265) + return QMediaFormat::VideoCodec::H265; + if (format == MFVideoFormat_VP80) + return QMediaFormat::VideoCodec::VP8; + if (format == MFVideoFormat_VP90) + return QMediaFormat::VideoCodec::VP9; + if (format == MFVideoFormat_AV1) + return QMediaFormat::VideoCodec::AV1; + if (format == MFVideoFormat_WMV1 + || format == MFVideoFormat_WMV2 + || format == MFVideoFormat_WMV3) + return QMediaFormat::VideoCodec::WMV; + if (format == MFVideoFormat_MJPG) + return QMediaFormat::VideoCodec::MotionJPEG; + return QMediaFormat::VideoCodec::Unspecified; +} + +GUID QWindowsMultimediaUtils::audioFormatForCodec(QMediaFormat::AudioCodec codec) +{ + switch (codec) { + case QMediaFormat::AudioCodec::MP3: + return MFAudioFormat_MP3; + case QMediaFormat::AudioCodec::AAC: + return MFAudioFormat_AAC; + case QMediaFormat::AudioCodec::ALAC: + return MFAudioFormat_ALAC; + case QMediaFormat::AudioCodec::FLAC: + return MFAudioFormat_FLAC; + case QMediaFormat::AudioCodec::Vorbis: + return MFAudioFormat_Vorbis; + case QMediaFormat::AudioCodec::Wave: + return MFAudioFormat_PCM; + case QMediaFormat::AudioCodec::Opus: + return MFAudioFormat_Opus; + case QMediaFormat::AudioCodec::AC3: + return MFAudioFormat_Dolby_AC3; + case QMediaFormat::AudioCodec::EAC3: + return MFAudioFormat_Dolby_DDPlus; + case QMediaFormat::AudioCodec::WMA: + return MFAudioFormat_WMAudioV9; + default: + return MFAudioFormat_AAC; + } +} + +QMediaFormat::AudioCodec QWindowsMultimediaUtils::codecForAudioFormat(GUID format) +{ + if (format == MFAudioFormat_MP3) + return QMediaFormat::AudioCodec::MP3; + if (format == MFAudioFormat_AAC) + return QMediaFormat::AudioCodec::AAC; + if (format == MFAudioFormat_ALAC) + return QMediaFormat::AudioCodec::ALAC; + if (format == MFAudioFormat_FLAC) + return QMediaFormat::AudioCodec::FLAC; + if (format == MFAudioFormat_Vorbis) + return QMediaFormat::AudioCodec::Vorbis; + if (format == MFAudioFormat_PCM) + return QMediaFormat::AudioCodec::Wave; + if (format == MFAudioFormat_Opus) + return QMediaFormat::AudioCodec::Opus; + if (format == MFAudioFormat_Dolby_AC3) + return QMediaFormat::AudioCodec::AC3; + if (format == MFAudioFormat_Dolby_DDPlus) + return QMediaFormat::AudioCodec::EAC3; + if (format == MFAudioFormat_WMAudioV8 + || format == MFAudioFormat_WMAudioV9 + || format == MFAudioFormat_WMAudio_Lossless) + return QMediaFormat::AudioCodec::WMA; + return QMediaFormat::AudioCodec::Unspecified; +} + +GUID QWindowsMultimediaUtils::containerForVideoFileFormat(QMediaFormat::FileFormat format) +{ + switch (format) { + case QMediaFormat::FileFormat::MPEG4: + return QMM_MFTranscodeContainerType_MPEG4; + case QMediaFormat::FileFormat::WMV: + return QMM_MFTranscodeContainerType_ASF; + case QMediaFormat::FileFormat::AVI: + return QMM_MFTranscodeContainerType_AVI; + default: + return QMM_MFTranscodeContainerType_MPEG4; + } +} + +GUID QWindowsMultimediaUtils::containerForAudioFileFormat(QMediaFormat::FileFormat format) +{ + switch (format) { + case QMediaFormat::FileFormat::MP3: + return QMM_MFTranscodeContainerType_MP3; + case QMediaFormat::FileFormat::AAC: + return QMM_MFTranscodeContainerType_ADTS; + case QMediaFormat::FileFormat::Mpeg4Audio: + return QMM_MFTranscodeContainerType_MPEG4; + case QMediaFormat::FileFormat::WMA: + return QMM_MFTranscodeContainerType_ASF; + case QMediaFormat::FileFormat::FLAC: + return QMM_MFTranscodeContainerType_FLAC; + case QMediaFormat::FileFormat::Wave: + return QMM_MFTranscodeContainerType_WAVE; + default: + return QMM_MFTranscodeContainerType_MPEG4; + } +} + +QString QWindowsMultimediaUtils::errorString(HRESULT hr) +{ + return QStringLiteral("%1 %2") + .arg(quint32(hr), 8, 16) + .arg(QString::fromStdString(std::system_category().message(hr))); +} + +QT_END_NAMESPACE diff --git a/src/multimedia/windows/qwindowsmultimediautils_p.h b/src/multimedia/windows/qwindowsmultimediautils_p.h new file mode 100644 index 000000000..58ecd425f --- /dev/null +++ b/src/multimedia/windows/qwindowsmultimediautils_p.h @@ -0,0 +1,47 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#ifndef QWINDOWSMULTIMEDIATUTILS_P_H +#define QWINDOWSMULTIMEDIATUTILS_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <private/qtmultimediaglobal_p.h> +#include <private/qplatformmediaformatinfo_p.h> +#include <qvideoframeformat.h> +#include <guiddef.h> +#include <qstring.h> + +QT_BEGIN_NAMESPACE + +namespace QWindowsMultimediaUtils { + + Q_MULTIMEDIA_EXPORT QVideoFrameFormat::PixelFormat pixelFormatFromMediaSubtype(const GUID &subtype); + + Q_MULTIMEDIA_EXPORT GUID videoFormatForCodec(QMediaFormat::VideoCodec codec); + + Q_MULTIMEDIA_EXPORT QMediaFormat::VideoCodec codecForVideoFormat(GUID format); + + Q_MULTIMEDIA_EXPORT GUID audioFormatForCodec(QMediaFormat::AudioCodec codec); + + Q_MULTIMEDIA_EXPORT QMediaFormat::AudioCodec codecForAudioFormat(GUID format); + + Q_MULTIMEDIA_EXPORT GUID containerForVideoFileFormat(QMediaFormat::FileFormat format); + + Q_MULTIMEDIA_EXPORT GUID containerForAudioFileFormat(QMediaFormat::FileFormat format); + + Q_MULTIMEDIA_EXPORT QString errorString(HRESULT hr); +} + +QT_END_NAMESPACE + +#endif diff --git a/src/multimedia/windows/qwindowsresampler.cpp b/src/multimedia/windows/qwindowsresampler.cpp new file mode 100644 index 000000000..3c50c0c19 --- /dev/null +++ b/src/multimedia/windows/qwindowsresampler.cpp @@ -0,0 +1,241 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + +#include "qwindowsresampler_p.h" +#include <qwindowsaudioutils_p.h> +#include <qloggingcategory.h> +#include <QUuid> + +#include <wmcodecdsp.h> +#include <mftransform.h> +#include <mferror.h> + +QT_BEGIN_NAMESPACE + +QUuid qIID_IMFTransform(0xbf94c121, 0x5b05, 0x4e6f, 0x80,0x00, 0xba,0x59,0x89,0x61,0x41,0x4d); +QUuid qCLSID_CResamplerMediaObject("f447b69e-1884-4a7e-8055-346f74d6edb3"); + +static Q_LOGGING_CATEGORY(qLcAudioResampler, "qt.multimedia.audioresampler") + +QWindowsResampler::QWindowsResampler() +{ + CoCreateInstance(qCLSID_CResamplerMediaObject, nullptr, CLSCTX_INPROC_SERVER, + qIID_IMFTransform, (LPVOID*)(m_resampler.GetAddressOf())); + if (m_resampler) + m_resampler->AddInputStreams(1, &m_inputStreamID); +} + +QWindowsResampler::~QWindowsResampler() = default; + +quint64 QWindowsResampler::outputBufferSize(quint64 inputBufferSize) const +{ + if (m_inputFormat.isValid() && m_outputFormat.isValid()) + return m_outputFormat.bytesForDuration(m_inputFormat.durationForBytes(inputBufferSize)); + else + return 0; +} + +quint64 QWindowsResampler::inputBufferSize(quint64 outputBufferSize) const +{ + if (m_inputFormat.isValid() && m_outputFormat.isValid()) + return m_inputFormat.bytesForDuration(m_outputFormat.durationForBytes(outputBufferSize)); + else + return 0; +} + +HRESULT QWindowsResampler::processInput(const QByteArrayView &in) +{ + ComPtr<IMFSample> sample; + HRESULT hr = m_wmf->mfCreateSample(sample.GetAddressOf()); + if (FAILED(hr)) + return hr; + + ComPtr<IMFMediaBuffer> buffer; + hr = m_wmf->mfCreateMemoryBuffer(in.size(), buffer.GetAddressOf()); + if (FAILED(hr)) + return hr; + + BYTE *data = nullptr; + DWORD maxLen = 0; + DWORD currentLen = 0; + hr = buffer->Lock(&data, &maxLen, ¤tLen); + if (FAILED(hr)) + return hr; + + memcpy(data, in.data(), in.size()); + + hr = buffer->Unlock(); + if (FAILED(hr)) + return hr; + + hr = buffer->SetCurrentLength(in.size()); + if (FAILED(hr)) + return hr; + + hr = sample->AddBuffer(buffer.Get()); + if (FAILED(hr)) + return hr; + + return m_resampler->ProcessInput(m_inputStreamID, sample.Get(), 0); +} + +HRESULT QWindowsResampler::processOutput(QByteArray &out) +{ + ComPtr<IMFSample> sample; + ComPtr<IMFMediaBuffer> buffer; + + if (m_resamplerNeedsSampleBuffer) { + HRESULT hr = m_wmf->mfCreateSample(sample.GetAddressOf()); + if (FAILED(hr)) + return hr; + + auto expectedOutputSize = outputBufferSize(m_totalInputBytes) - m_totalOutputBytes; + hr = m_wmf->mfCreateMemoryBuffer(expectedOutputSize, buffer.GetAddressOf()); + if (FAILED(hr)) + return hr; + + hr = sample->AddBuffer(buffer.Get()); + if (FAILED(hr)) + return hr; + } + + HRESULT hr = S_OK; + + MFT_OUTPUT_DATA_BUFFER outputDataBuffer; + outputDataBuffer.dwStreamID = 0; + do { + outputDataBuffer.pEvents = nullptr; + outputDataBuffer.dwStatus = 0; + outputDataBuffer.pSample = m_resamplerNeedsSampleBuffer ? sample.Get() : nullptr; + DWORD status = 0; + hr = m_resampler->ProcessOutput(0, 1, &outputDataBuffer, &status); + if (SUCCEEDED(hr)) { + ComPtr<IMFMediaBuffer> outputBuffer; + outputDataBuffer.pSample->ConvertToContiguousBuffer(outputBuffer.GetAddressOf()); + DWORD len = 0; + BYTE *data = nullptr; + hr = outputBuffer->Lock(&data, nullptr, &len); + if (SUCCEEDED(hr)) + out.push_back(QByteArray(reinterpret_cast<char *>(data), len)); + outputBuffer->Unlock(); + } + } while (SUCCEEDED(hr)); + + if (hr == MF_E_TRANSFORM_NEED_MORE_INPUT) + hr = S_OK; + + return hr; +} + +QByteArray QWindowsResampler::resample(const QByteArrayView &in) +{ + m_totalInputBytes += in.size(); + + if (m_inputFormat == m_outputFormat) { + m_totalOutputBytes += in.size(); + return {in.data(), in.size()}; + + } else { + Q_ASSERT(m_resampler && m_wmf); + + QByteArray out; + HRESULT hr = processInput(in); + if (SUCCEEDED(hr)) + hr = processOutput(out); + + if (FAILED(hr)) + qCWarning(qLcAudioResampler) << "Resampling failed" << hr; + + m_totalOutputBytes += out.size(); + return out; + } +} + +QByteArray QWindowsResampler::resample(IMFSample *sample) +{ + Q_ASSERT(sample); + + DWORD totalLength = 0; + HRESULT hr = sample->GetTotalLength(&totalLength); + if (FAILED(hr)) + return {}; + + m_totalInputBytes += totalLength; + + QByteArray out; + + if (m_inputFormat == m_outputFormat) { + ComPtr<IMFMediaBuffer> outputBuffer; + sample->ConvertToContiguousBuffer(outputBuffer.GetAddressOf()); + DWORD len = 0; + BYTE *data = nullptr; + hr = outputBuffer->Lock(&data, nullptr, &len); + if (SUCCEEDED(hr)) + out.push_back(QByteArray(reinterpret_cast<char *>(data), len)); + outputBuffer->Unlock(); + + } else { + Q_ASSERT(m_resampler && m_wmf); + + hr = m_resampler->ProcessInput(m_inputStreamID, sample, 0); + if (SUCCEEDED(hr)) + hr = processOutput(out); + + if (FAILED(hr)) + qCWarning(qLcAudioResampler) << "Resampling failed" << hr; + } + + m_totalOutputBytes += out.size(); + + return out; +} + +bool QWindowsResampler::setup(const QAudioFormat &fin, const QAudioFormat &fout) +{ + qCDebug(qLcAudioResampler) << "Setup audio resampler" << fin << "->" << fout; + + m_totalInputBytes = 0; + m_totalOutputBytes = 0; + + if (fin == fout) { + qCDebug(qLcAudioResampler) << "Pass through mode"; + m_inputFormat = fin; + m_outputFormat = fout; + return true; + } + + if (!m_resampler || !m_wmf) + return false; + + ComPtr<IMFMediaType> min = QWindowsAudioUtils::formatToMediaType(*m_wmf, fin); + ComPtr<IMFMediaType> mout = QWindowsAudioUtils::formatToMediaType(*m_wmf, fout); + + HRESULT hr = m_resampler->SetInputType(m_inputStreamID, min.Get(), 0); + if (FAILED(hr)) { + qCWarning(qLcAudioResampler) << "Failed to set input type" << hr; + return false; + } + + hr = m_resampler->SetOutputType(0, mout.Get(), 0); + if (FAILED(hr)) { + qCWarning(qLcAudioResampler) << "Failed to set output type" << hr; + return false; + } + + MFT_OUTPUT_STREAM_INFO streamInfo; + hr = m_resampler->GetOutputStreamInfo(0, &streamInfo); + if (FAILED(hr)) { + qCWarning(qLcAudioResampler) << "Could not obtain stream info" << hr; + return false; + } + + m_resamplerNeedsSampleBuffer = (streamInfo.dwFlags + & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES)) == 0; + + m_inputFormat = fin; + m_outputFormat = fout; + + return true; +} + +QT_END_NAMESPACE diff --git a/src/multimedia/windows/qwindowsresampler_p.h b/src/multimedia/windows/qwindowsresampler_p.h new file mode 100644 index 000000000..5885bffa6 --- /dev/null +++ b/src/multimedia/windows/qwindowsresampler_p.h @@ -0,0 +1,75 @@ +// Copyright (C) 2021 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-2.0-only OR GPL-3.0-only + + +#ifndef QT_QWINDOWSRESAMPLER_H +#define QT_QWINDOWSRESAMPLER_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qbytearray.h> +#include <qbytearrayview.h> +#include <qaudioformat.h> +#include <private/qcomptr_p.h> +#include <private/qwindowsmediafoundation_p.h> +#include <qt_windows.h> +#include <mftransform.h> +#include <QtCore/private/qfunctions_win_p.h> + +struct IMFSample; +struct IMFTransform; + +QT_BEGIN_NAMESPACE + +class QWindowsMediaFoundation; + +class Q_MULTIMEDIA_EXPORT QWindowsResampler +{ +public: + QWindowsResampler(); + ~QWindowsResampler(); + + bool setup(const QAudioFormat &in, const QAudioFormat &out); + + QByteArray resample(const QByteArrayView &in); + QByteArray resample(IMFSample *sample); + + QAudioFormat inputFormat() const { return m_inputFormat; } + QAudioFormat outputFormat() const { return m_outputFormat; } + + quint64 outputBufferSize(quint64 inputBufferSize) const; + quint64 inputBufferSize(quint64 outputBufferSize) const; + + quint64 totalInputBytes() const { return m_totalInputBytes; } + quint64 totalOutputBytes() const { return m_totalOutputBytes; } + +private: + HRESULT processInput(const QByteArrayView &in); + HRESULT processOutput(QByteArray &out); + + QComHelper m_comRuntime; + QWindowsMediaFoundation *m_wmf{ QWindowsMediaFoundation::instance() }; + QMFRuntimeInit m_wmfRuntime{ m_wmf }; + ComPtr<IMFTransform> m_resampler; + + bool m_resamplerNeedsSampleBuffer = false; + quint64 m_totalInputBytes = 0; + quint64 m_totalOutputBytes = 0; + QAudioFormat m_inputFormat; + QAudioFormat m_outputFormat; + + DWORD m_inputStreamID = 0; +}; + +QT_END_NAMESPACE + +#endif // QT_QWINDOWSRESAMPLER_H |