diff options
Diffstat (limited to 'src/spatialaudio')
26 files changed, 3746 insertions, 0 deletions
diff --git a/src/spatialaudio/CMakeLists.txt b/src/spatialaudio/CMakeLists.txt new file mode 100644 index 000000000..d0d005e1e --- /dev/null +++ b/src/spatialaudio/CMakeLists.txt @@ -0,0 +1,29 @@ +# Copyright (C) 2022 The Qt Company Ltd. +# SPDX-License-Identifier: BSD-3-Clause + +qt_internal_add_module(SpatialAudio + SOURCES + qambisonicdecoder.cpp qambisonicdecoder_p.h qambisonicdecoderdata_p.h + qaudioengine.cpp qaudioengine.h qaudioengine_p.h + qaudiolistener.cpp qaudiolistener.h + qaudioroom.cpp qaudioroom.h qaudioroom_p.h + qspatialsound.cpp qspatialsound.h qspatialsound_p.h + qambientsound.cpp qambientsound.h qambientsound_p.h + qtspatialaudioglobal.h qtspatialaudioglobal_p.h + INCLUDE_DIRECTORIES + "../3rdparty/resonance-audio/resonance_audio" + "../3rdparty/resonance-audio" + "../resonance-audio" + "../3rdparty/eigen" + LIBRARIES + Qt::MultimediaPrivate + Qt::BundledResonanceAudio + PUBLIC_LIBRARIES + Qt::Multimedia + GENERATE_CPP_EXPORTS +) + + +qt_internal_add_docs(SpatialAudio + doc/qtspatialaudio.qdocconf +) diff --git a/src/spatialaudio/doc/qtspatialaudio.qdocconf b/src/spatialaudio/doc/qtspatialaudio.qdocconf new file mode 100644 index 000000000..3c8916907 --- /dev/null +++ b/src/spatialaudio/doc/qtspatialaudio.qdocconf @@ -0,0 +1,62 @@ +include($QT_INSTALL_DOCS/global/qt-module-defaults.qdocconf) + +project = QtSpatialAudio +description = Qt Spatial Audio Documentation +version = $QT_VERSION +buildversion = "Technology Preview" + +moduleheader = QtSpatialAudio +includepaths += . + +examplesinstallpath = spatialaudio + +# The following parameters are for creating a qhp file, the qhelpgenerator +# program can convert the qhp file into a qch file which can be opened in +# Qt Assistant and/or Qt Creator. + +# Defines the name of the project. You cannot use operators (+, =, -) in +# the name. Properties for this project are set using a qhp.<projectname>.property +# format. +qhp.projects = QtSpatialAudio +qhp.QtSpatialAudio.file = qtspatialaudio.qhp +qhp.QtSpatialAudio.namespace = org.qt-project.qtspatialaudio.$QT_VERSION_TAG +qhp.QtSpatialAudio.indexTitle = Qt Spatial Audio +qhp.QtSpatialAudio.virtualFolder = qtspatialaudio + +# For listing child nodes in Qt Creator or Assistant. +qhp.QtSpatialAudio.subprojects = classes qmltypes examples + +qhp.QtSpatialAudio.subprojects.classes.title = Qt Spatial Audio Classes +qhp.QtSpatialAudio.subprojects.classes.indexTitle = Qt Spatial Audio C++ Classes +qhp.QtSpatialAudio.subprojects.classes.selectors = module:QtSpatialAudio +qhp.QtSpatialAudio.subprojects.classes.sortPages = true + +qhp.QtSpatialAudio.subprojects.qmltypes.title = QML Types +qhp.QtSpatialAudio.subprojects.qmltypes.indexTitle = Qt Spatial Audio QML Types +qhp.QtSpatialAudio.subprojects.qmltypes.selectors = qmlclass +qhp.QtSpatialAudio.subprojects.qmltypes.sortPages = true + +qhp.QtSpatialAudio.subprojects.examples.title = Examples +qhp.QtSpatialAudio.subprojects.examples.indexTitle = Qt Spatial Audio Examples +qhp.QtSpatialAudio.subprojects.examples.selectors = doc:example +qhp.QtSpatialAudio.subprojects.examples.sortPages = true + +exampledirs += ../../../examples/spatialaudio \ + snippets + +headerdirs += .. \ + ../../spatialaudioquick3d + +imagedirs += src/images \ + +sourcedirs += .. \ + ../../spatialaudioquick3d + +depends = qtcore qtdoc qtgui qtquick qtqml qtnetwork qmake qtcmake qtquickcontrols qtquick3d qtmultimedia + +# Ignore \since commands for versions earlier than 6.3 +ignoresince = 6.4 + +navigation.landingpage = "Qt Spatial Audio" +navigation.cppclassespage = "Qt Spatial Audio C++ Classes" +navigation.qmltypespage = "Qt Spatial Audio QML Types" diff --git a/src/spatialaudio/doc/src/qtspatialaudio-cpp.qdoc b/src/spatialaudio/doc/src/qtspatialaudio-cpp.qdoc new file mode 100644 index 000000000..aff743e82 --- /dev/null +++ b/src/spatialaudio/doc/src/qtspatialaudio-cpp.qdoc @@ -0,0 +1,33 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GFDL-1.3-no-invariants-only + +/*! + \module QtSpatialAudio + \title Qt Spatial Audio Module C++ Classes + \ingroup modules + \qtvariable spatialaudio + \qtcmakepackage SpatialAudio + + \brief The \l {Qt Spatial Audio} module provides functionality for 3D audio. + + \include module-use.qdocinc using qt module + + \code + find_package(Qt6 REQUIRED COMPONENTS SpatialAudio) + target_link_libraries(my_project PRIVATE Qt6::SpatialAudio) + \endcode +*/ + +/*! + \page qtspatialaudio-modules.html + \title Qt Spatial Audio C++ Classes + \brief Provides C++ class for spatial audio use cases. + + The C++ classes provide the same functionality as the Qt Quick3D spatial audio + module. + + \section1 Classes + + \generatelist {classesbymodule QtSpatialAudio} + +*/ diff --git a/src/spatialaudio/doc/src/qtspatialaudio-examples.qdoc b/src/spatialaudio/doc/src/qtspatialaudio-examples.qdoc new file mode 100644 index 000000000..b2ceb118c --- /dev/null +++ b/src/spatialaudio/doc/src/qtspatialaudio-examples.qdoc @@ -0,0 +1,15 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GFDL-1.3-no-invariants-only + +/*! + \group spatialaudio_examples + \ingroup multimedia_examples + \title Qt Spatial Audio Examples + \brief Demonstrates the spatial audio functionality provided by Qt. + + The \l{Qt Spatial Audio} module provides cross-platform capabilities to + add support for spatial audio to Qt based applications. + + The examples listed below show some typical use cases where audio sources are + located in 3D space adding virtual rooms around the source. +*/ diff --git a/src/spatialaudio/doc/src/qtspatialaudio-index.qdoc b/src/spatialaudio/doc/src/qtspatialaudio-index.qdoc new file mode 100644 index 000000000..784273b54 --- /dev/null +++ b/src/spatialaudio/doc/src/qtspatialaudio-index.qdoc @@ -0,0 +1,104 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GFDL-1.3-no-invariants-only + +/*! + \page qtspatialaudio-index.html + \title Qt Spatial Audio + \brief The Qt Spatial Audio module provides APIs for modeling sound source + and their surrounds in 3D space + + Qt Spatial Audio is an add-on module that provides a rich set of QML types + and C++ classes to implement sound fields in 3D space. It contains an easy to use + API for positing a listener in space, adding localized sound sources around the + listener and emulating virtual rooms with reverb and reflections. + + \section1 Getting started + + If you are new to Qt Spatial Audio, the QML types can be + \l{qtqml import syntax}{imported} into an application using the following + statement in your \c {.qml} file. + + \qml + import QtQuick3D.SpatialAudio + \endqml + + To link against the C++ libraries, add the following to your project's + \c CMakeLists.txt file. Substitute \c my_project with the name of your + project. + + \code + find_package(Qt6 REQUIRED COMPONENTS SpatialAudio) + target_link_libraries(my_project PRIVATE Qt6::SpatialAudio) + \endcode + + \l{Spatial Audio Overview} provides a more detailed description about how + to use the different classes listed below. + + \section1 QML Types + + The following table outlines some important QML types. + + \table + \header + \li Type + \li Description + \row + \li \l{AudioEngine} + \li The engine doing the processing of the audio scene + \row + \li \l {SpatialSound} + \li A sound source located in 3D space. + \row + \li \l {AmbientSound} + \li A location independent stereo sound track. + \row + \li \l {AudioRoom} + \li Defines a room that generates audio reverb and reflections. + \endtable + + \section1 C++ Classes + + The following table outlines some important C++ Classes + + \table + \header + \li Class + \li Description + \row + \li \l{QAudioEngine} + \li The engine doing the processing of the audio scene + \row + \li \l {QSpatialSound} + \li A sound source located in 3D space. + \row + \li \l {QAmbientSound} + \li A location independent stereo sound track. + \row + \li \l {QAudioRoom} + \li Defines a room that generates audio reverb and reflections. + \endtable + + \section1 Licenses and Attributions + + The Qt Spatial Audio module is available under commercial licenses from + \l{The Qt Company}. + In addition, it is available under free software licenses. These free software + licenses are + \l{GNU Lesser General Public License, version 3}, or + the \l{GNU General Public License, version 3}. + See \l{Qt Licensing} for further details. + + Note that Qt Spatial Audio is not available under the \l{GNU General Public License, version 2}. + + Furthermore, Qt Spatial Audio in Qt \QtVersion contains third party + modules under the following permissive licenses: + + \generatelist{groupsbymodule attributions-qtspatialaudio} + + \section1 Reference and Examples + \list + \li \l{Qt Spatial Audio QML Types}{QML Types} + \li \l{Qt Spatial Audio C++ Classes}{C++ Classes} + \li \l{Qt Spatial Audio Examples}{Examples} + \endlist +*/ diff --git a/src/spatialaudio/doc/src/qtspatialaudio-qml-types.qdoc b/src/spatialaudio/doc/src/qtspatialaudio-qml-types.qdoc new file mode 100644 index 000000000..fbd1853d3 --- /dev/null +++ b/src/spatialaudio/doc/src/qtspatialaudio-qml-types.qdoc @@ -0,0 +1,26 @@ +// Copyright (C) 2015 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GFDL-1.3-no-invariants-only + +/*! +\qmlmodule QtQuick3D.SpatialAudio +\title Qt Spatial Audio QML Types +\ingroup qmlmodules +\brief Provides QML types for spatial audio. + +The QML types for \l{Qt Spatial Audio} support the full functionality of the +C++ API. + +\section1 QML Types + +Qt Spatial Audio QML types are designed to be used together with \l{Qt Quick 3D}. They +can be imported into your application using the following import statement in your +.qml file: + +\qml +import QtQuick3D.SpatialAudio +\endqml + +\generatelist qmltypesbymodule QtQuick3D.SpatialAudio + +\noautolist +*/ diff --git a/src/spatialaudio/doc/src/spatialaudiooverview.qdoc b/src/spatialaudio/doc/src/spatialaudiooverview.qdoc new file mode 100644 index 000000000..62b95871e --- /dev/null +++ b/src/spatialaudio/doc/src/spatialaudiooverview.qdoc @@ -0,0 +1,64 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR GFDL-1.3-no-invariants-only + +/*! +\page spatialaudiooverview.html +\title Spatial Audio Overview +\brief Support for spatial audio. +\ingroup explanations-graphicsandmultimedia + +The Qt Spatial Audio API provides a number of classes that allow the creation of +three dimensional sound scene. It is defined by objects located in 3D space +that emit sound and surrounding geometry that can be modelled using +one or several rooms. Finally a listener can be placed into this +sound scene at a specified position and orientation. + +There are both C++ and QML APIs that can be used. + +\section1 Creating a sound scene + +To create the sound scene, one first instantiates a \l QAudioEngine. This engine +processes input sound data and geometries to create a realistic +representation of the sound scene as it would be experienced by a person placed +at a specific location inside the scene. + +The \l QAudioEngine::OutputMode property can be used to optimize the output either +for headphones using binaural (virtual 3D) rendering or for a stereo or surround speaker +configuration. + +The output device can be selected using \l QAudioEngine::outputDevice property. + +Once the engine is set up, we can place various sound objects into the scene by creating +\l QSpatialSound objects and specifying a url to a sound file using the \l +QSpatialSound::source property. + +\l QAudioListener can be used to define the position and orientation of a person +listening to the sound scene. At max one listener per engine can be used. If no listener +is specified, the engine assumes that the listener is at the origin of the coordinate system +facing into a positive z direction, with positive y pointing upwards. + +In addition to sound sources and a listener, you can define a geometry that influences how the +sound is being experienced by the listener through a set of \l QAudioRoom objects. Rooms +are rectangular and support a wide variety of materials for each wall giving a different experience +with different sound reflections and reverb. Room effects will get applied if the listener is +located inside one of the rooms. If he is inside multiple rooms, the room with the smallest +geometrical volume will take precedence. + +If you need some stereo overlay that is independent of the position and orientation of +the listener (such as background music or a voice-over), you can use +\l QAmbientSound to create the sound overlay. + +For a small QWidget based example showcasing one audio source that can be moved around in a room, have +a look at the \l {audiopanning}{Spacial Audio Panning Example}. + +\section1 Reference Documentation + +\section2 C++ Classes + +\annotatedlist spatialaudio + +\section2 QML Types + +\annotatedlist quick3d_spatialaudio + +*/ diff --git a/src/spatialaudio/qambientsound.cpp b/src/spatialaudio/qambientsound.cpp new file mode 100644 index 000000000..cdf61b918 --- /dev/null +++ b/src/spatialaudio/qambientsound.cpp @@ -0,0 +1,283 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#include "qambientsound.h" +#include "qambientsound_p.h" +#include "qaudioengine_p.h" +#include "resonance_audio.h" +#include <qaudiosink.h> +#include <qurl.h> +#include <qdebug.h> +#include <qaudiodecoder.h> + +QT_BEGIN_NAMESPACE + +void QAmbientSoundPrivate::load() +{ + decoder.reset(new QAudioDecoder); + buffers.clear(); + currentBuffer = 0; + sourceDeviceFile.reset(nullptr); + bufPos = 0; + m_playing = false; + m_loading = true; + auto *ep = QAudioEnginePrivate::get(engine); + QAudioFormat f; + f.setSampleFormat(QAudioFormat::Float); + f.setSampleRate(ep->sampleRate); + f.setChannelConfig(nchannels == 2 ? QAudioFormat::ChannelConfigStereo : QAudioFormat::ChannelConfigMono); + decoder->setAudioFormat(f); + if (url.scheme().compare(u"qrc", Qt::CaseInsensitive) == 0) { + auto qrcFile = std::make_unique<QFile>(u':' + url.path()); + if (!qrcFile->open(QFile::ReadOnly)) + return; + sourceDeviceFile = std::move(qrcFile); + decoder->setSourceDevice(sourceDeviceFile.get()); + } else { + decoder->setSource(url); + } + connect(decoder.get(), &QAudioDecoder::bufferReady, this, &QAmbientSoundPrivate::bufferReady); + connect(decoder.get(), &QAudioDecoder::finished, this, &QAmbientSoundPrivate::finished); + decoder->start(); +} + +void QAmbientSoundPrivate::getBuffer(float *buf, int nframes, int channels) +{ + Q_ASSERT(channels == nchannels); + QMutexLocker l(&mutex); + if (!m_playing || currentBuffer >= buffers.size()) { + memset(buf, 0, channels * nframes * sizeof(float)); + } else { + int frames = nframes; + float *ff = buf; + while (frames) { + if (currentBuffer < buffers.size()) { + const QAudioBuffer &b = buffers.at(currentBuffer); + // qDebug() << s << b.format().sampleRate() << b.format().channelCount() << b.format().sampleFormat(); + auto *f = b.constData<float>() + bufPos*nchannels; + int toCopy = qMin(b.frameCount() - bufPos, frames); + memcpy(ff, f, toCopy*sizeof(float)*nchannels); + ff += toCopy*nchannels; + frames -= toCopy; + bufPos += toCopy; + Q_ASSERT(bufPos <= b.frameCount()); + if (bufPos == b.frameCount()) { + ++currentBuffer; + bufPos = 0; + } + } else { + // no more data available + if (m_loading) + qDebug() << "underrun" << frames << "frames when loading" << url; + memset(ff, 0, frames * channels * sizeof(float)); + ff += frames * channels; + frames = 0; + } + if (!m_loading) { + if (currentBuffer == buffers.size()) { + currentBuffer = 0; + ++m_currentLoop; + } + if (m_loops > 0 && m_currentLoop >= m_loops) { + m_playing = false; + m_currentLoop = 0; + } + } + } + Q_ASSERT(ff - buf == channels*nframes); + } +} + +void QAmbientSoundPrivate::bufferReady() +{ + QMutexLocker l(&mutex); + auto b = decoder->read(); + // qDebug() << "read buffer" << b.format() << b.startTime() << b.duration(); + buffers.append(b); + if (m_autoPlay) + m_playing = true; +} + +void QAmbientSoundPrivate::finished() +{ + m_loading = false; +} + +/*! + \class QAmbientSound + \inmodule QtSpatialAudio + \ingroup spatialaudio + \ingroup multimedia_audio + + \brief A stereo overlay sound. + + QAmbientSound represents a position and orientation independent sound. + It's commonly used for background sounds (e.g. music) that is supposed to be independent + of the listeners position and orientation. + */ + +/*! + Creates a stereo sound source for \a engine. + */ +QAmbientSound::QAmbientSound(QAudioEngine *engine) + : d(new QAmbientSoundPrivate(this)) +{ + setEngine(engine); +} + +QAmbientSound::~QAmbientSound() +{ + setEngine(nullptr); + delete d; +} + +/*! + \property QAmbientSound::volume + + Defines the volume of the sound. + + Values between 0 and 1 will attenuate the sound, while values above 1 + provide an additional gain boost. + */ +void QAmbientSound::setVolume(float volume) +{ + if (d->volume == volume) + return; + d->volume = volume; + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSourceVolume(d->sourceId, d->volume); + emit volumeChanged(); +} + +float QAmbientSound::volume() const +{ + return d->volume; +} + +void QAmbientSound::setSource(const QUrl &url) +{ + if (d->url == url) + return; + d->url = url; + + d->load(); + emit sourceChanged(); +} + +/*! + \property QAmbientSound::source + + The source file for the sound to be played. + */ +QUrl QAmbientSound::source() const +{ + return d->url; +} +/*! + \enum QAmbientSound::Loops + + Lets you control the playback loop using the following values: + + \value Infinite Loops infinitely + \value Once Stops playback after running once +*/ +/*! + \property QAmbientSound::loops + + Determines how many times the sound is played before the player stops. + Set to QAmbientSound::Infinite to play the current sound in + a loop forever. + + The default value is \c 1. + */ +int QAmbientSound::loops() const +{ + return d->m_loops.loadRelaxed(); +} + +void QAmbientSound::setLoops(int loops) +{ + int oldLoops = d->m_loops.fetchAndStoreRelaxed(loops); + if (oldLoops != loops) + emit loopsChanged(); +} + +/*! + \property QAmbientSound::autoPlay + + Determines whether the sound should automatically start playing when a source + gets specified. + + The default value is \c true. + */ +bool QAmbientSound::autoPlay() const +{ + return d->m_autoPlay.loadRelaxed(); +} + +void QAmbientSound::setAutoPlay(bool autoPlay) +{ + bool old = d->m_autoPlay.fetchAndStoreRelaxed(autoPlay); + if (old != autoPlay) + emit autoPlayChanged(); +} + +/*! + Starts playing back the sound. Does nothing if the sound is already playing. + */ +void QAmbientSound::play() +{ + d->play(); +} + +/*! + Pauses sound playback. Calling play() will continue playback. + */ +void QAmbientSound::pause() +{ + d->pause(); +} + +/*! + Stops sound playback and resets the current position and current loop count to 0. + Calling play() will start playback at the beginning of the sound file. + */ +void QAmbientSound::stop() +{ + d->stop(); +} + +/*! + \internal + */ +void QAmbientSound::setEngine(QAudioEngine *engine) +{ + if (d->engine == engine) + return; + + // Remove self from old engine (if necessary) + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->removeStereoSound(this); + + d->engine = engine; + + // Add self to new engine if necessary + ep = QAudioEnginePrivate::get(d->engine); + if (ep) { + ep->addStereoSound(this); + ep->resonanceAudio->api->SetSourceVolume(d->sourceId, d->volume); + } +} + +/*! + Returns the engine associated with this sound. + */ +QAudioEngine *QAmbientSound::engine() const +{ + return d->engine; +} + +QT_END_NAMESPACE + +#include "moc_qambientsound.cpp" diff --git a/src/spatialaudio/qambientsound.h b/src/spatialaudio/qambientsound.h new file mode 100644 index 000000000..f10159251 --- /dev/null +++ b/src/spatialaudio/qambientsound.h @@ -0,0 +1,68 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#ifndef QAMBIENTSOUND_H +#define QAMBIENTSOUND_H + +#include <QtSpatialAudio/qtspatialaudioglobal.h> +#include <QtMultimedia/qtmultimediaglobal.h> +#include <QtCore/QUrl> +#include <QtCore/QObject> + +QT_BEGIN_NAMESPACE + +class QAudioEngine; +class QAmbientSoundPrivate; + +class Q_SPATIALAUDIO_EXPORT QAmbientSound : public QObject +{ + Q_OBJECT + Q_PROPERTY(QUrl source READ source WRITE setSource NOTIFY sourceChanged) + Q_PROPERTY(float volume READ volume WRITE setVolume NOTIFY volumeChanged) + Q_PROPERTY(int loops READ loops WRITE setLoops NOTIFY loopsChanged) + Q_PROPERTY(bool autoPlay READ autoPlay WRITE setAutoPlay NOTIFY autoPlayChanged) + +public: + explicit QAmbientSound(QAudioEngine *engine); + ~QAmbientSound(); + + void setSource(const QUrl &url); + QUrl source() const; + + enum Loops + { + Infinite = -1, + Once = 1 + }; + Q_ENUM(Loops) + + int loops() const; + void setLoops(int loops); + + bool autoPlay() const; + void setAutoPlay(bool autoPlay); + + void setVolume(float volume); + float volume() const; + + QAudioEngine *engine() const; + +Q_SIGNALS: + void sourceChanged(); + void loopsChanged(); + void autoPlayChanged(); + void volumeChanged(); + +public Q_SLOTS: + void play(); + void pause(); + void stop(); + +private: + void setEngine(QAudioEngine *engine); + friend class QAmbientSoundPrivate; + QAmbientSoundPrivate *d = nullptr; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qambientsound_p.h b/src/spatialaudio/qambientsound_p.h new file mode 100644 index 000000000..d26404f43 --- /dev/null +++ b/src/spatialaudio/qambientsound_p.h @@ -0,0 +1,84 @@ +// Copyright (C) 2024 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only + +#ifndef QAMBIENTSOUND_P_H +#define QAMBIENTSOUND_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtspatialaudioglobal_p.h> +#include <qmutex.h> +#include <qurl.h> +#include <qfile.h> +#include <qaudiodecoder.h> +#include <qaudiobuffer.h> + +QT_BEGIN_NAMESPACE + +class QAudioEngine; + +class QAmbientSoundPrivate : public QObject +{ +public: + QAmbientSoundPrivate(QObject *parent, int nchannels = 2) + : QObject(parent) + , nchannels(nchannels) + {} + + template<typename T> + static QAmbientSoundPrivate *get(T *soundSource) { return soundSource ? soundSource->d : nullptr; } + + QUrl url; + float volume = 1.; + int nchannels = 2; + std::unique_ptr<QAudioDecoder> decoder; + std::unique_ptr<QFile> sourceDeviceFile; + QAudioEngine *engine = nullptr; + + QMutex mutex; + int currentBuffer = 0; + int bufPos = 0; + int m_currentLoop = 0; + QList<QAudioBuffer> buffers; + int sourceId = -1; // kInvalidSourceId + + QAtomicInteger<bool> m_autoPlay = true; + QAtomicInteger<bool> m_playing = false; + QAtomicInt m_loops = 1; + bool m_loading = false; + + void play() { + m_playing = true; + } + void pause() { + m_playing = false; + } + void stop() { + QMutexLocker locker(&mutex); + m_playing = false; + currentBuffer = 0; + bufPos = 0; + m_currentLoop = 0; + } + + void load(); + void getBuffer(float *buf, int frames, int channels); + +private Q_SLOTS: + void bufferReady(); + void finished(); + +}; + +QT_END_NAMESPACE + +#endif // QAMBIENTSOUND_P_H diff --git a/src/spatialaudio/qambisonicdecoder.cpp b/src/spatialaudio/qambisonicdecoder.cpp new file mode 100644 index 000000000..4404b11b0 --- /dev/null +++ b/src/spatialaudio/qambisonicdecoder.cpp @@ -0,0 +1,314 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#include "qambisonicdecoder_p.h" + +#include "qambisonicdecoderdata_p.h" +#include <cmath> +#include <qdebug.h> + +QT_BEGIN_NAMESPACE + +// Ambisonic decoding is described in detail in https://ambisonics.dreamhosters.com/BLaH3.pdf. +// We're using a phase matched band splitting filter to split the ambisonic signal into a low +// and high frequency component and apply matrix conversions to those components individually +// as described in the document. +// +// We are currently not using a near field compensation filter, something that could potentially +// improve sound quality further. +// +// For mono and stereo decoding, we use a simpler algorithm to avoid artificially dampening signals +// coming from the back, as we do not have any speakers in that direction and the calculations +// through matlab would give us audible 'holes'. + +struct QAmbisonicDecoderData +{ + QAudioFormat::ChannelConfig config; + const float *lf[3]; + const float *hf[3]; + const float *reverb; +}; + +const float reverb_x_0[] = +{ + 1.f, 0.f, // L + 0.f, 1.f, // R + .7f, .7f, // C + 1.f, 0.f, // Ls + 0.f, 1.f, // Rs + 1.f, 0.f, // Lb + 0.f, 1.f, // Rb +}; + +const float reverb_x_1[] = +{ + 1.f, 0.f, // L + 0.f, 1.f, // R + .7f, .7f, // C + .0f, .0f, // LFE + 1.f, 0.f, // Ls + 0.f, 1.f, // Rs + 1.f, 0.f, // Lb + 0.f, 1.f, // Rb +}; + +static const QAmbisonicDecoderData decoderMap[] = +{ + { QAudioFormat::ChannelConfigSurround5Dot0, + { decoderMatrix_5dot0_1_lf, decoderMatrix_5dot0_2_lf, decoderMatrix_5dot0_3_lf }, + { decoderMatrix_5dot0_1_hf, decoderMatrix_5dot0_2_hf, decoderMatrix_5dot0_3_hf }, + reverb_x_0 + }, + { QAudioFormat::ChannelConfigSurround5Dot1, + { decoderMatrix_5dot1_1_lf, decoderMatrix_5dot1_2_lf, decoderMatrix_5dot1_3_lf }, + { decoderMatrix_5dot1_1_hf, decoderMatrix_5dot1_2_hf, decoderMatrix_5dot1_3_hf }, + reverb_x_1 + }, + { QAudioFormat::ChannelConfigSurround7Dot0, + { decoderMatrix_7dot0_1_lf, decoderMatrix_7dot0_2_lf, decoderMatrix_7dot0_3_lf }, + { decoderMatrix_7dot0_1_hf, decoderMatrix_7dot0_2_hf, decoderMatrix_7dot0_3_hf }, + reverb_x_0 + }, + { QAudioFormat::ChannelConfigSurround7Dot1, + { decoderMatrix_7dot1_1_lf, decoderMatrix_7dot1_2_lf, decoderMatrix_7dot1_3_lf }, + { decoderMatrix_7dot1_1_hf, decoderMatrix_7dot1_2_hf, decoderMatrix_7dot1_3_hf }, + reverb_x_1 + } +}; + +// Implements a split second order IIR filter +// The audio data is split into a phase synced low and high frequency part +// This allows us to apply different factors to both parts for better sound +// localization when converting from ambisonic formats +// +// Details are described in https://ambisonics.dreamhosters.com/BLaH3.pdf, Appendix A.2. +class QAmbisonicDecoderFilter +{ +public: + QAmbisonicDecoderFilter() = default; + void configure(float sampleRate, float cutoffFrequency = 380) + { + double k = tan(M_PI*cutoffFrequency/sampleRate); + a1 = float(2.*(k*k - 1.)/(k*k + 2*k + 1.)); + a2 = float((k*k - 2*k + 1.)/(k*k + 2*k + 1.)); + + b0_lf = float(k*k/(k*k + 2*k + 1)); + b1_lf = 2.f*b0_lf; + + b0_hf = float(1./(k*k + 2*k + 1)); + b1_hf = -2.f*b0_hf; + } + + struct Output + { + float lf; + float hf; + }; + + Output next(float x) + { + float r_lf = x*b0_lf + + prevX[0]*b1_lf + + prevX[1]*b0_lf - + prevR_lf[0]*a1 - + prevR_lf[1]*a2; + float r_hf = x*b0_hf + + prevX[0]*b1_hf + + prevX[1]*b0_hf - + prevR_hf[0]*a1 - + prevR_hf[1]*a2; + prevX[1] = prevX[0]; + prevX[0] = x; + prevR_lf[1] = prevR_lf[0]; + prevR_lf[0] = r_lf; + prevR_hf[1] = prevR_hf[0]; + prevR_hf[0] = r_hf; + return { r_lf, r_hf }; + } + +private: + float a1 = 0.; + float a2 = 0.; + + float b0_hf = 0.; + float b1_hf = 0.; + + float b0_lf = 0.; + float b1_lf = 0.; + + float prevX[2] = {}; + float prevR_lf[2] = {}; + float prevR_hf[2] = {}; +}; + + +QAmbisonicDecoder::QAmbisonicDecoder(AmbisonicLevel ambisonicLevel, const QAudioFormat &format) + : level(ambisonicLevel) +{ + Q_ASSERT(level > 0 && level <= 3); + inputChannels = (level+1)*(level+1); + outputChannels = format.channelCount(); + + channelConfig = format.channelConfig(); + if (channelConfig == QAudioFormat::ChannelConfigUnknown) + channelConfig = format.defaultChannelConfigForChannelCount(format.channelCount()); + + if (channelConfig == QAudioFormat::ChannelConfigMono || + channelConfig == QAudioFormat::ChannelConfigStereo || + channelConfig == QAudioFormat::ChannelConfig2Dot1 || + channelConfig == QAudioFormat::ChannelConfig3Dot0 || + channelConfig == QAudioFormat::ChannelConfig3Dot1) { + // these are non surround configs and handled manually to avoid + // audible holes for sounds coming from behing + // + // We use a simpler decoding process here, only taking first order + // ambisonics into account + // + // Left and right channels get 50% W and 50% X + // Center gets 50% W and 50% Y + // LFE gets 50% W + simpleDecoderFactors = new float[4*outputChannels]; + float *r = new float[2*outputChannels]; // reverb output is in stereo + float *f = simpleDecoderFactors; + reverbFactors = r; + if (channelConfig & QAudioFormat::channelConfig(QAudioFormat::FrontLeft)) { + f[0] = 0.5f; f[1] = 0.5f; f[2] = 0.; f[3] = 0.f; + f += 4; + r[0] = 1.; r[1] = 0.; + r += 2; + } + if (channelConfig & QAudioFormat::channelConfig(QAudioFormat::FrontRight)) { + f[0] = 0.5f; f[1] = -0.5f; f[2] = 0.; f[3] = 0.f; + f += 4; + r[0] = 0.; r[1] = 1.; + r += 2; + } + if (channelConfig & QAudioFormat::channelConfig(QAudioFormat::FrontCenter)) { + f[0] = 0.5f; f[1] = -0.f; f[2] = 0.; f[3] = 0.5f; + f += 4; + r[0] = .5; r[1] = .5; + r += 2; + } + if (channelConfig & QAudioFormat::channelConfig(QAudioFormat::LFE)) { + f[0] = 0.5f; f[1] = -0.f; f[2] = 0.; f[3] = 0.0f; + f += 4; + r[0] = 0.; r[1] = 0.; + r += 2; + } + Q_UNUSED(f); + Q_UNUSED(r); + Q_ASSERT((f - simpleDecoderFactors) == 4*outputChannels); + Q_ASSERT((r - reverbFactors) == 2*outputChannels); + + return; + } + + for (const auto &d : decoderMap) { + if (d.config == channelConfig) { + decoderData = &d; + reverbFactors = decoderData->reverb; + break; + } + } + if (!decoderData) { + // can't handle this, + outputChannels = 0; + return; + } + + filters = new QAmbisonicDecoderFilter[inputChannels]; + for (int i = 0; i < inputChannels; ++i) + filters[i].configure(format.sampleRate()); +} + +QAmbisonicDecoder::~QAmbisonicDecoder() +{ + if (simpleDecoderFactors) { + delete simpleDecoderFactors; + delete reverbFactors; + } +} + +void QAmbisonicDecoder::processBuffer(const float *input[], float *output, int nSamples) +{ + float *o = output; + memset(o, 0, nSamples*outputChannels*sizeof(float)); + + if (simpleDecoderFactors) { + for (int i = 0; i < nSamples; ++i) { + for (int j = 0; j < 4; ++j) { + for (int k = 0; k < outputChannels; ++k) + o[k] += simpleDecoderFactors[k*4 + j]*input[j][i]; + } + o += outputChannels; + } + return; + } + + const float *matrix_hi = decoderData->hf[level - 1]; + const float *matrix_lo = decoderData->lf[level - 1]; + for (int i = 0; i < nSamples; ++i) { + QAmbisonicDecoderFilter::Output buf[maxAmbisonicChannels]; + for (int j = 0; j < inputChannels; ++j) + buf[j] = filters[j].next(input[j][i]); + for (int j = 0; j < inputChannels; ++j) { + for (int k = 0; k < outputChannels; ++k) + o[k] += matrix_lo[k*inputChannels + j]*buf[j].lf + matrix_hi[k*inputChannels + j]*buf[j].hf; + } + o += outputChannels; + } +} + +void QAmbisonicDecoder::processBuffer(const float *input[], short *output, int nSamples) +{ + const float *reverb[] = { nullptr, nullptr }; + return processBufferWithReverb(input, reverb, output, nSamples); +} + +void QAmbisonicDecoder::processBufferWithReverb(const float *input[], const float *reverb[2], short *output, int nSamples) +{ + if (simpleDecoderFactors) { + for (int i = 0; i < nSamples; ++i) { + float o[4] = {}; + for (int k = 0; k < outputChannels; ++k) { + for (int j = 0; j < 4; ++j) + o[k] += simpleDecoderFactors[k*4 + j]*input[j][i]; + } + if (reverb[0]) { + for (int k = 0; k < outputChannels; ++k) { + o[k] += reverb[0][i]*reverbFactors[2*k] + reverb[1][i]*reverbFactors[2*k+1]; + } + } + + for (int k = 0; k < outputChannels; ++k) + output[k] = static_cast<short>(o[k]*32768.); + output += outputChannels; + } + return; + } + + // qDebug() << "XXX" << inputChannels << outputChannels; + const float *matrix_hi = decoderData->hf[level - 1]; + const float *matrix_lo = decoderData->lf[level - 1]; + for (int i = 0; i < nSamples; ++i) { + QAmbisonicDecoderFilter::Output buf[maxAmbisonicChannels]; + for (int j = 0; j < inputChannels; ++j) + buf[j] = filters[j].next(input[j][i]); + float o[32] = {}; // we can't support more than 32 channels from our API + for (int j = 0; j < inputChannels; ++j) { + for (int k = 0; k < outputChannels; ++k) + o[k] += matrix_lo[k*inputChannels + j]*buf[j].lf + matrix_hi[k*inputChannels + j]*buf[j].hf; + } + if (reverb[0]) { + for (int k = 0; k < outputChannels; ++k) { + o[k] += reverb[0][i]*reverbFactors[2*k] + reverb[1][i]*reverbFactors[2*k+1]; + } + } + for (int k = 0; k < outputChannels; ++k) + output[k] = static_cast<short>(o[k]*32768.); + output += outputChannels; + } + +} + +QT_END_NAMESPACE + diff --git a/src/spatialaudio/qambisonicdecoder_p.h b/src/spatialaudio/qambisonicdecoder_p.h new file mode 100644 index 000000000..9ac60c2a7 --- /dev/null +++ b/src/spatialaudio/qambisonicdecoder_p.h @@ -0,0 +1,69 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#ifndef QAMBISONICDECODER_P_H +#define QAMBISONICDECODER_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtspatialaudioglobal_p.h> +#include <qaudioformat.h> + +QT_BEGIN_NAMESPACE + +struct QAmbisonicDecoderData; +class QAmbisonicDecoderFilter; + +class QAmbisonicDecoder +{ +public: + enum AmbisonicLevel + { + AmbisonicLevel1 = 1, + LowQuality = AmbisonicLevel1, + AmbisonicLevel2 = 2, + MediumQuality = AmbisonicLevel2, + AmbisonicLevel3 = 3, + HighQuality = AmbisonicLevel3 + }; + QAmbisonicDecoder(AmbisonicLevel ambisonicLevel, const QAudioFormat &format); + ~QAmbisonicDecoder(); + + bool hasValidConfig() const { return outputChannels > 0; } + + int nInputChannels() const { return inputChannels; } + int nOutputChannels() const { return outputChannels; } + + int outputSize(int nSamples) const { return outputChannels * nSamples; } + + // input is planar, output interleaved + void processBuffer(const float *input[], float *output, int nSamples); + void processBuffer(const float *input[], short *output, int nSamples); + + void processBufferWithReverb(const float *input[], const float *reverb[2], short *output, int nSamples); + + static constexpr int maxAmbisonicChannels = 16; + static constexpr int maxAmbisonicLevel = 3; +private: + QAudioFormat::ChannelConfig channelConfig; + AmbisonicLevel level = AmbisonicLevel1; + int inputChannels = 0; + int outputChannels = 0; + const QAmbisonicDecoderData *decoderData = nullptr; + QAmbisonicDecoderFilter *filters = nullptr; + float *simpleDecoderFactors = nullptr; + const float *reverbFactors = nullptr; +}; + + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qambisonicdecoderdata_p.h b/src/spatialaudio/qambisonicdecoderdata_p.h new file mode 100644 index 000000000..dfee9213a --- /dev/null +++ b/src/spatialaudio/qambisonicdecoderdata_p.h @@ -0,0 +1,279 @@ +// Copyright (C) 2016 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#ifndef QAMBISONICDECODERDATA_P_H +#define QAMBISONICDECODERDATA_P_H + +#include <qtspatialaudioglobal_p.h> + +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +// This file is generated by the matlab/octave file adt_generate_qt.m +// using the Ambisonic Decoder Toolbox (https://bitbucket.org/ambidecodertoolbox/adt/src/master/) + + +QT_BEGIN_NAMESPACE + +// Decoder matrix for 5dot0, ambisonic level 1 +static constexpr float decoderMatrix_5dot0_1_lf[5*4] = { +0.255580f, 0.430877f, 0.000000f, 0.386458f, // L +0.255573f, -0.430877f, 0.000000f, 0.386450f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, // C +0.552170f, 0.623932f, 0.000000f, -0.628578f, // Ls +0.552175f, -0.623939f, 0.000000f, -0.628571f, // Rs +}; + +// Decoder matrix for 5dot0, ambisonic level 1 +static constexpr float decoderMatrix_5dot0_1_hf[5*4] = { +0.361445f, 0.351810f, 0.000000f, 0.315542f, // L +0.361435f, -0.351809f, 0.000000f, 0.315535f, // R +0.191780f, 0.000000f, 0.000000f, 0.268870f, // C +0.780886f, 0.509439f, 0.000000f, -0.513232f, // Ls +0.780893f, -0.509444f, 0.000000f, -0.513226f, // Rs +}; + +// Decoder matrix for 5dot0, ambisonic level 2 +static constexpr float decoderMatrix_5dot0_2_lf[5*9] = { +0.255580f, 0.430877f, 0.000000f, 0.386458f, 0.157509f, 0.000000f, -0.095304f, 0.000000f, -0.013008f, // L +0.255573f, -0.430877f, 0.000000f, 0.386450f, -0.157507f, 0.000000f, -0.095343f, 0.000000f, -0.013009f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, // C +0.552170f, 0.623932f, 0.000000f, -0.628578f, -0.041978f, 0.000000f, -0.139072f, 0.000000f, -0.030798f, // Ls +0.552175f, -0.623939f, 0.000000f, -0.628571f, 0.041979f, 0.000000f, -0.139039f, 0.000000f, -0.030796f, // Rs +}; + +// Decoder matrix for 5dot0, ambisonic level 2 +static constexpr float decoderMatrix_5dot0_2_hf[5*9] = { +0.404108f, 0.527714f, 0.000000f, 0.473313f, 0.099617f, 0.000000f, -0.060276f, 0.000000f, -0.008227f, // L +0.404097f, -0.527714f, 0.000000f, 0.473303f, -0.099616f, 0.000000f, -0.060300f, 0.000000f, -0.008228f, // R +0.214417f, 0.000000f, 0.000000f, 0.403305f, 0.000000f, 0.000000f, -0.035484f, 0.000000f, 0.070209f, // C +0.873057f, 0.764158f, 0.000000f, -0.769848f, -0.026549f, 0.000000f, -0.087957f, 0.000000f, -0.019478f, // Ls +0.873065f, -0.764166f, 0.000000f, -0.769839f, 0.026550f, 0.000000f, -0.087936f, 0.000000f, -0.019477f, // Rs +}; + +// Decoder matrix for 5dot0, ambisonic level 3 +static constexpr float decoderMatrix_5dot0_3_lf[5*16] = { +0.255580f, 0.430877f, 0.000000f, 0.386458f, 0.157509f, 0.000000f, -0.095304f, 0.000000f, -0.013008f, 0.013422f, 0.000000f, 0.030238f, 0.000000f, 0.025660f, 0.000000f, -0.014215f, // L +0.255573f, -0.430877f, 0.000000f, 0.386450f, -0.157507f, 0.000000f, -0.095343f, 0.000000f, -0.013009f, -0.013422f, 0.000000f, -0.030227f, 0.000000f, 0.025649f, 0.000000f, -0.014214f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018999f, 0.000000f, 0.020478f, // C +0.552170f, 0.623932f, 0.000000f, -0.628578f, -0.041978f, 0.000000f, -0.139072f, 0.000000f, -0.030798f, -0.012642f, 0.000000f, 0.049794f, 0.000000f, -0.073727f, 0.000000f, -0.001768f, // Ls +0.552175f, -0.623939f, 0.000000f, -0.628571f, 0.041979f, 0.000000f, -0.139039f, 0.000000f, -0.030796f, 0.012642f, 0.000000f, -0.049793f, 0.000000f, -0.073721f, 0.000000f, -0.001769f, // Rs +}; + +// Decoder matrix for 5dot0, ambisonic level 3 +static constexpr float decoderMatrix_5dot0_3_hf[5*16] = { +0.426355f, 0.618969f, 0.000000f, 0.555160f, 0.160893f, 0.000000f, -0.097352f, 0.000000f, -0.013287f, 0.006823f, 0.000000f, 0.015372f, 0.000000f, 0.013045f, 0.000000f, -0.007226f, // L +0.426343f, -0.618969f, 0.000000f, 0.555149f, -0.160891f, 0.000000f, -0.097392f, 0.000000f, -0.013289f, -0.006823f, 0.000000f, -0.015367f, 0.000000f, 0.013039f, 0.000000f, -0.007226f, // R +0.226221f, 0.000000f, 0.000000f, 0.473046f, 0.000000f, 0.000000f, -0.057310f, 0.000000f, 0.113395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.009658f, 0.000000f, 0.010410f, // C +0.921121f, 0.896300f, 0.000000f, -0.902974f, -0.042880f, 0.000000f, -0.142060f, 0.000000f, -0.031459f, -0.006427f, 0.000000f, 0.025314f, 0.000000f, -0.037481f, 0.000000f, -0.000899f, // Ls +0.921130f, -0.896310f, 0.000000f, -0.902963f, 0.042881f, 0.000000f, -0.142027f, 0.000000f, -0.031457f, 0.006427f, 0.000000f, -0.025313f, 0.000000f, -0.037478f, 0.000000f, -0.000899f, // Rs +}; + +// Decoder matrix for 5dot1, ambisonic level 1 +static constexpr float decoderMatrix_5dot1_1_lf[6*4] = { +0.255580f, 0.430877f, 0.000000f, 0.386458f, // L +0.255573f, -0.430877f, 0.000000f, 0.386450f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, // C +0.5f, 0.0f, 0.0f, 0.0f, // LFE +0.552170f, 0.623932f, 0.000000f, -0.628578f, // Ls +0.552175f, -0.623939f, 0.000000f, -0.628571f, // Rs +}; + +// Decoder matrix for 5dot1, ambisonic level 1 +static constexpr float decoderMatrix_5dot1_1_hf[6*4] = { +0.361445f, 0.351810f, 0.000000f, 0.315542f, // L +0.361435f, -0.351809f, 0.000000f, 0.315535f, // R +0.191780f, 0.000000f, 0.000000f, 0.268870f, // C +0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.780886f, 0.509439f, 0.000000f, -0.513232f, // Ls +0.780893f, -0.509444f, 0.000000f, -0.513226f, // Rs +}; + +// Decoder matrix for 5dot1, ambisonic level 2 +static constexpr float decoderMatrix_5dot1_2_lf[6*9] = { +0.255580f, 0.430877f, 0.000000f, 0.386458f, 0.157509f, 0.000000f, -0.095304f, 0.000000f, -0.013008f, // L +0.255573f, -0.430877f, 0.000000f, 0.386450f, -0.157507f, 0.000000f, -0.095343f, 0.000000f, -0.013009f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, // C +0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.552170f, 0.623932f, 0.000000f, -0.628578f, -0.041978f, 0.000000f, -0.139072f, 0.000000f, -0.030798f, // Ls +0.552175f, -0.623939f, 0.000000f, -0.628571f, 0.041979f, 0.000000f, -0.139039f, 0.000000f, -0.030796f, // Rs +}; + +// Decoder matrix for 5dot1, ambisonic level 2 +static constexpr float decoderMatrix_5dot1_2_hf[6*9] = { +0.404108f, 0.527714f, 0.000000f, 0.473313f, 0.099617f, 0.000000f, -0.060276f, 0.000000f, -0.008227f, // L +0.404097f, -0.527714f, 0.000000f, 0.473303f, -0.099616f, 0.000000f, -0.060300f, 0.000000f, -0.008228f, // R +0.214417f, 0.000000f, 0.000000f, 0.403305f, 0.000000f, 0.000000f, -0.035484f, 0.000000f, 0.070209f, // C +0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.873057f, 0.764158f, 0.000000f, -0.769848f, -0.026549f, 0.000000f, -0.087957f, 0.000000f, -0.019478f, // Ls +0.873065f, -0.764166f, 0.000000f, -0.769839f, 0.026550f, 0.000000f, -0.087936f, 0.000000f, -0.019477f, // Rs +}; + +// Decoder matrix for 5dot1, ambisonic level 3 +static constexpr float decoderMatrix_5dot1_3_lf[6*16] = { +0.255580f, 0.430877f, 0.000000f, 0.386458f, 0.157509f, 0.000000f, -0.095304f, 0.000000f, -0.013008f, 0.013422f, 0.000000f, 0.030238f, 0.000000f, 0.025660f, 0.000000f, -0.014215f, // L +0.255573f, -0.430877f, 0.000000f, 0.386450f, -0.157507f, 0.000000f, -0.095343f, 0.000000f, -0.013009f, -0.013422f, 0.000000f, -0.030227f, 0.000000f, 0.025649f, 0.000000f, -0.014214f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018999f, 0.000000f, 0.020478f, // C +0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.552170f, 0.623932f, 0.000000f, -0.628578f, -0.041978f, 0.000000f, -0.139072f, 0.000000f, -0.030798f, -0.012642f, 0.000000f, 0.049794f, 0.000000f, -0.073727f, 0.000000f, -0.001768f, // Ls +0.552175f, -0.623939f, 0.000000f, -0.628571f, 0.041979f, 0.000000f, -0.139039f, 0.000000f, -0.030796f, 0.012642f, 0.000000f, -0.049793f, 0.000000f, -0.073721f, 0.000000f, -0.001769f, // Rs +}; + +// Decoder matrix for 5dot1, ambisonic level 3 +static constexpr float decoderMatrix_5dot1_3_hf[6*16] = { +0.426355f, 0.618969f, 0.000000f, 0.555160f, 0.160893f, 0.000000f, -0.097352f, 0.000000f, -0.013287f, 0.006823f, 0.000000f, 0.015372f, 0.000000f, 0.013045f, 0.000000f, -0.007226f, // L +0.426343f, -0.618969f, 0.000000f, 0.555149f, -0.160891f, 0.000000f, -0.097392f, 0.000000f, -0.013289f, -0.006823f, 0.000000f, -0.015367f, 0.000000f, 0.013039f, 0.000000f, -0.007226f, // R +0.226221f, 0.000000f, 0.000000f, 0.473046f, 0.000000f, 0.000000f, -0.057310f, 0.000000f, 0.113395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.009658f, 0.000000f, 0.010410f, // C +0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.921121f, 0.896300f, 0.000000f, -0.902974f, -0.042880f, 0.000000f, -0.142060f, 0.000000f, -0.031459f, -0.006427f, 0.000000f, 0.025314f, 0.000000f, -0.037481f, 0.000000f, -0.000899f, // Ls +0.921130f, -0.896310f, 0.000000f, -0.902963f, 0.042881f, 0.000000f, -0.142027f, 0.000000f, -0.031457f, 0.006427f, 0.000000f, -0.025313f, 0.000000f, -0.037478f, 0.000000f, -0.000899f, // Rs +}; + +// Decoder matrix for 7dot0, ambisonic level 1 +static constexpr float decoderMatrix_7dot0_1_lf[7*4] = { +0.205900f, 0.314866f, 0.000000f, 0.366133f, // L +0.205883f, -0.314871f, 0.000000f, 0.366126f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, // C +0.276200f, 0.619758f, 0.000000f, 0.000000f, // Ls +0.276228f, -0.619768f, 0.000000f, 0.000000f, // Rs +0.276222f, 0.309884f, 0.000000f, -0.536733f, // Lb +0.276196f, -0.309893f, 0.000000f, -0.536723f, // Rb +}; + +// Decoder matrix for 7dot0, ambisonic level 1 +static constexpr float decoderMatrix_7dot0_1_hf[7*4] = { +0.291186f, 0.257087f, 0.000000f, 0.298946f, // L +0.291162f, -0.257091f, 0.000000f, 0.298940f, // R +0.191780f, 0.000000f, 0.000000f, 0.268870f, // C +0.390605f, 0.506030f, 0.000000f, 0.000000f, // Ls +0.390645f, -0.506039f, 0.000000f, 0.000000f, // Rs +0.390637f, 0.253019f, 0.000000f, -0.438240f, // Lb +0.390600f, -0.253026f, 0.000000f, -0.438232f, // Rb +}; + +// Decoder matrix for 7dot0, ambisonic level 2 +static constexpr float decoderMatrix_7dot0_2_lf[7*9] = { +0.205900f, 0.314866f, 0.000000f, 0.366133f, 0.144868f, 0.000000f, -0.081538f, 0.000000f, 0.023353f, // L +0.205883f, -0.314871f, 0.000000f, 0.366126f, -0.144867f, 0.000000f, -0.081652f, 0.000000f, 0.023347f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, // C +0.276200f, 0.619758f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106989f, 0.000000f, -0.163267f, // Ls +0.276228f, -0.619768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106856f, 0.000000f, -0.163267f, // Rs +0.276222f, 0.309884f, 0.000000f, -0.536733f, -0.141395f, 0.000000f, -0.106873f, 0.000000f, 0.081634f, // Lb +0.276196f, -0.309893f, 0.000000f, -0.536723f, 0.141394f, 0.000000f, -0.107038f, 0.000000f, 0.081628f, // Rb +}; + +// Decoder matrix for 7dot0, ambisonic level 2 +static constexpr float decoderMatrix_7dot0_2_hf[7*9] = { +0.325556f, 0.385630f, 0.000000f, 0.448419f, 0.091623f, 0.000000f, -0.051569f, 0.000000f, 0.014770f, // L +0.325529f, -0.385637f, 0.000000f, 0.448410f, -0.091622f, 0.000000f, -0.051641f, 0.000000f, 0.014766f, // R +0.214417f, 0.000000f, 0.000000f, 0.403305f, 0.000000f, 0.000000f, -0.035484f, 0.000000f, 0.070209f, // C +0.436710f, 0.759045f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.067666f, 0.000000f, -0.103259f, // Ls +0.436755f, -0.759058f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.067582f, 0.000000f, -0.103259f, // Rs +0.436746f, 0.379529f, 0.000000f, -0.657361f, -0.089426f, 0.000000f, -0.067593f, 0.000000f, 0.051630f, // Lb +0.436704f, -0.379539f, 0.000000f, -0.657348f, 0.089426f, 0.000000f, -0.067697f, 0.000000f, 0.051626f, // Rb +}; + +// Decoder matrix for 7dot0, ambisonic level 3 +static constexpr float decoderMatrix_7dot0_3_lf[7*16] = { +0.205900f, 0.314866f, 0.000000f, 0.366133f, 0.144868f, 0.000000f, -0.081538f, 0.000000f, 0.023353f, 0.019489f, 0.000000f, 0.019857f, 0.000000f, 0.022683f, 0.000000f, -0.011066f, // L +0.205883f, -0.314871f, 0.000000f, 0.366126f, -0.144867f, 0.000000f, -0.081652f, 0.000000f, 0.023347f, -0.019488f, 0.000000f, -0.019830f, 0.000000f, 0.022669f, 0.000000f, -0.011066f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018999f, 0.000000f, 0.020478f, // C +0.276200f, 0.619758f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106989f, 0.000000f, -0.163267f, -0.018501f, 0.000000f, 0.040168f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Ls +0.276228f, -0.619768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106856f, 0.000000f, -0.163267f, 0.018501f, 0.000000f, -0.040194f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Rs +0.276222f, 0.309884f, 0.000000f, -0.536733f, -0.141395f, 0.000000f, -0.106873f, 0.000000f, 0.081634f, 0.018501f, 0.000000f, 0.020094f, 0.000000f, -0.034809f, 0.000000f, 0.000000f, // Lb +0.276196f, -0.309893f, 0.000000f, -0.536723f, 0.141394f, 0.000000f, -0.107038f, 0.000000f, 0.081628f, -0.018500f, 0.000000f, -0.020079f, 0.000000f, -0.034779f, 0.000000f, 0.000000f, // Rb +}; + +// Decoder matrix for 7dot0, ambisonic level 3 +static constexpr float decoderMatrix_7dot0_3_hf[7*16] = { +0.343479f, 0.452315f, 0.000000f, 0.525962f, 0.147981f, 0.000000f, -0.083290f, 0.000000f, 0.023855f, 0.009908f, 0.000000f, 0.010095f, 0.000000f, 0.011532f, 0.000000f, -0.005626f, // L +0.343450f, -0.452323f, 0.000000f, 0.525952f, -0.147980f, 0.000000f, -0.083406f, 0.000000f, 0.023849f, -0.009907f, 0.000000f, -0.010081f, 0.000000f, 0.011524f, 0.000000f, -0.005626f, // R +0.226221f, 0.000000f, 0.000000f, 0.473046f, 0.000000f, 0.000000f, -0.057310f, 0.000000f, 0.113395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.009658f, 0.000000f, 0.010410f, // C +0.460752f, 0.890303f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.109288f, 0.000000f, -0.166775f, -0.009405f, 0.000000f, 0.020420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Ls +0.460799f, -0.890318f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.109152f, 0.000000f, -0.166775f, 0.009405f, 0.000000f, -0.020434f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Rs +0.460790f, 0.445159f, 0.000000f, -0.771035f, -0.144433f, 0.000000f, -0.109170f, 0.000000f, 0.083387f, 0.009406f, 0.000000f, 0.010215f, 0.000000f, -0.017696f, 0.000000f, 0.000000f, // Lb +0.460745f, -0.445171f, 0.000000f, -0.771020f, 0.144432f, 0.000000f, -0.109338f, 0.000000f, 0.083382f, -0.009405f, 0.000000f, -0.010207f, 0.000000f, -0.017681f, 0.000000f, 0.000000f, // Rb +}; + +// Decoder matrix for 7dot1, ambisonic level 1 +static constexpr float decoderMatrix_7dot1_1_lf[8*4] = { +0.205900f, 0.314866f, 0.000000f, 0.366133f, // L +0.205883f, -0.314871f, 0.000000f, 0.366126f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, // C +0.5f, 0.0f, 0.0f, 0.0f, // LFE +0.276200f, 0.619758f, 0.000000f, 0.000000f, // Ls +0.276228f, -0.619768f, 0.000000f, 0.000000f, // Rs +0.276222f, 0.309884f, 0.000000f, -0.536733f, // Lb +0.276196f, -0.309893f, 0.000000f, -0.536723f, // Rb +}; + +// Decoder matrix for 7dot1, ambisonic level 1 +static constexpr float decoderMatrix_7dot1_1_hf[8*4] = { +0.291186f, 0.257087f, 0.000000f, 0.298946f, // L +0.291162f, -0.257091f, 0.000000f, 0.298940f, // R +0.191780f, 0.000000f, 0.000000f, 0.268870f, // C +0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.390605f, 0.506030f, 0.000000f, 0.000000f, // Ls +0.390645f, -0.506039f, 0.000000f, 0.000000f, // Rs +0.390637f, 0.253019f, 0.000000f, -0.438240f, // Lb +0.390600f, -0.253026f, 0.000000f, -0.438232f, // Rb +}; + +// Decoder matrix for 7dot1, ambisonic level 2 +static constexpr float decoderMatrix_7dot1_2_lf[8*9] = { +0.205900f, 0.314866f, 0.000000f, 0.366133f, 0.144868f, 0.000000f, -0.081538f, 0.000000f, 0.023353f, // L +0.205883f, -0.314871f, 0.000000f, 0.366126f, -0.144867f, 0.000000f, -0.081652f, 0.000000f, 0.023347f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, // C +0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.276200f, 0.619758f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106989f, 0.000000f, -0.163267f, // Ls +0.276228f, -0.619768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106856f, 0.000000f, -0.163267f, // Rs +0.276222f, 0.309884f, 0.000000f, -0.536733f, -0.141395f, 0.000000f, -0.106873f, 0.000000f, 0.081634f, // Lb +0.276196f, -0.309893f, 0.000000f, -0.536723f, 0.141394f, 0.000000f, -0.107038f, 0.000000f, 0.081628f, // Rb +}; + +// Decoder matrix for 7dot1, ambisonic level 2 +static constexpr float decoderMatrix_7dot1_2_hf[8*9] = { +0.325556f, 0.385630f, 0.000000f, 0.448419f, 0.091623f, 0.000000f, -0.051569f, 0.000000f, 0.014770f, // L +0.325529f, -0.385637f, 0.000000f, 0.448410f, -0.091622f, 0.000000f, -0.051641f, 0.000000f, 0.014766f, // R +0.214417f, 0.000000f, 0.000000f, 0.403305f, 0.000000f, 0.000000f, -0.035484f, 0.000000f, 0.070209f, // C +0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.436710f, 0.759045f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.067666f, 0.000000f, -0.103259f, // Ls +0.436755f, -0.759058f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.067582f, 0.000000f, -0.103259f, // Rs +0.436746f, 0.379529f, 0.000000f, -0.657361f, -0.089426f, 0.000000f, -0.067593f, 0.000000f, 0.051630f, // Lb +0.436704f, -0.379539f, 0.000000f, -0.657348f, 0.089426f, 0.000000f, -0.067697f, 0.000000f, 0.051626f, // Rb +}; + +// Decoder matrix for 7dot1, ambisonic level 3 +static constexpr float decoderMatrix_7dot1_3_lf[8*16] = { +0.205900f, 0.314866f, 0.000000f, 0.366133f, 0.144868f, 0.000000f, -0.081538f, 0.000000f, 0.023353f, 0.019489f, 0.000000f, 0.019857f, 0.000000f, 0.022683f, 0.000000f, -0.011066f, // L +0.205883f, -0.314871f, 0.000000f, 0.366126f, -0.144867f, 0.000000f, -0.081652f, 0.000000f, 0.023347f, -0.019488f, 0.000000f, -0.019830f, 0.000000f, 0.022669f, 0.000000f, -0.011066f, // R +0.135609f, 0.000000f, 0.000000f, 0.329297f, 0.000000f, 0.000000f, -0.056105f, 0.000000f, 0.111010f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.018999f, 0.000000f, 0.020478f, // C +0.5f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.276200f, 0.619758f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106989f, 0.000000f, -0.163267f, -0.018501f, 0.000000f, 0.040168f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Ls +0.276228f, -0.619768f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.106856f, 0.000000f, -0.163267f, 0.018501f, 0.000000f, -0.040194f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Rs +0.276222f, 0.309884f, 0.000000f, -0.536733f, -0.141395f, 0.000000f, -0.106873f, 0.000000f, 0.081634f, 0.018501f, 0.000000f, 0.020094f, 0.000000f, -0.034809f, 0.000000f, 0.000000f, // Lb +0.276196f, -0.309893f, 0.000000f, -0.536723f, 0.141394f, 0.000000f, -0.107038f, 0.000000f, 0.081628f, -0.018500f, 0.000000f, -0.020079f, 0.000000f, -0.034779f, 0.000000f, 0.000000f, // Rb +}; + +// Decoder matrix for 7dot1, ambisonic level 3 +static constexpr float decoderMatrix_7dot1_3_hf[8*16] = { +0.343479f, 0.452315f, 0.000000f, 0.525962f, 0.147981f, 0.000000f, -0.083290f, 0.000000f, 0.023855f, 0.009908f, 0.000000f, 0.010095f, 0.000000f, 0.011532f, 0.000000f, -0.005626f, // L +0.343450f, -0.452323f, 0.000000f, 0.525952f, -0.147980f, 0.000000f, -0.083406f, 0.000000f, 0.023849f, -0.009907f, 0.000000f, -0.010081f, 0.000000f, 0.011524f, 0.000000f, -0.005626f, // R +0.226221f, 0.000000f, 0.000000f, 0.473046f, 0.000000f, 0.000000f, -0.057310f, 0.000000f, 0.113395f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, 0.009658f, 0.000000f, 0.010410f, // C +0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, // LFE +0.460752f, 0.890303f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.109288f, 0.000000f, -0.166775f, -0.009405f, 0.000000f, 0.020420f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Ls +0.460799f, -0.890318f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, -0.109152f, 0.000000f, -0.166775f, 0.009405f, 0.000000f, -0.020434f, 0.000000f, 0.000000f, 0.000000f, 0.000000f, // Rs +0.460790f, 0.445159f, 0.000000f, -0.771035f, -0.144433f, 0.000000f, -0.109170f, 0.000000f, 0.083387f, 0.009406f, 0.000000f, 0.010215f, 0.000000f, -0.017696f, 0.000000f, 0.000000f, // Lb +0.460745f, -0.445171f, 0.000000f, -0.771020f, 0.144432f, 0.000000f, -0.109338f, 0.000000f, 0.083382f, -0.009405f, 0.000000f, -0.010207f, 0.000000f, -0.017681f, 0.000000f, 0.000000f, // Rb +}; + +QT_END_NAMESPACE + +#endif + diff --git a/src/spatialaudio/qaudioengine.cpp b/src/spatialaudio/qaudioengine.cpp new file mode 100644 index 000000000..82228f72f --- /dev/null +++ b/src/spatialaudio/qaudioengine.cpp @@ -0,0 +1,602 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#include <qaudioengine_p.h> +#include <qambientsound_p.h> +#include <qspatialsound_p.h> +#include <qambientsound.h> +#include <qaudioroom_p.h> +#include <qaudiolistener.h> +#include <resonance_audio.h> +#include <qambisonicdecoder_p.h> +#include <qaudiodecoder.h> +#include <qmediadevices.h> +#include <qiodevice.h> +#include <qaudiosink.h> +#include <qdebug.h> +#include <qelapsedtimer.h> + +#include <QFile> + +QT_BEGIN_NAMESPACE + +// We'd like to have short buffer times, so the sound adjusts itself to changes +// quickly, but times below 100ms seem to give stuttering on macOS. +// It might be possible to set this value lower on other OSes. +const int bufferTimeMs = 100; + +// This class lives in the audioThread, but pulls data from QAudioEnginePrivate +// which lives in the mainThread. +class QAudioOutputStream : public QIODevice +{ + Q_OBJECT +public: + explicit QAudioOutputStream(QAudioEnginePrivate *d) + : d(d) + { + open(QIODevice::ReadOnly); + } + ~QAudioOutputStream(); + + qint64 readData(char *data, qint64 len) override; + + qint64 writeData(const char *, qint64) override; + + qint64 size() const override { return 0; } + qint64 bytesAvailable() const override { + return std::numeric_limits<qint64>::max(); + } + bool isSequential() const override { + return true; + } + bool atEnd() const override { + return false; + } + qint64 pos() const override { + return m_pos; + } + + Q_INVOKABLE void startOutput() { + d->mutex.lock(); + Q_ASSERT(!sink); + QAudioFormat format; + auto channelConfig = d->outputMode == QAudioEngine::Surround ? + d->device.channelConfiguration() : QAudioFormat::ChannelConfigStereo; + if (channelConfig != QAudioFormat::ChannelConfigUnknown) + format.setChannelConfig(channelConfig); + else + format.setChannelCount(d->device.preferredFormat().channelCount()); + format.setSampleRate(d->sampleRate); + format.setSampleFormat(QAudioFormat::Int16); + ambisonicDecoder.reset(new QAmbisonicDecoder(QAmbisonicDecoder::HighQuality, format)); + sink.reset(new QAudioSink(d->device, format)); + const qsizetype bufferSize = format.bytesForDuration(bufferTimeMs * 1000); + sink->setBufferSize(bufferSize); + d->mutex.unlock(); + // It is important to unlock the mutex before starting the sink, as the sink will + // call readData() in the audio thread, which will try to lock the mutex (again) + sink->start(this); + } + + Q_INVOKABLE void stopOutput() { + sink->stop(); + sink.reset(); + ambisonicDecoder.reset(); + } + + Q_INVOKABLE void restartOutput() { + stopOutput(); + startOutput(); + } + + void setPaused(bool paused) { + if (paused) + sink->suspend(); + else + sink->resume(); + } + +private: + qint64 m_pos = 0; + QAudioEnginePrivate *d = nullptr; + std::unique_ptr<QAudioSink> sink; + std::unique_ptr<QAmbisonicDecoder> ambisonicDecoder; +}; + + +QAudioOutputStream::~QAudioOutputStream() +{ +} + +qint64 QAudioOutputStream::writeData(const char *, qint64) +{ + return 0; +} + +qint64 QAudioOutputStream::readData(char *data, qint64 len) +{ + if (d->paused.loadRelaxed()) + return 0; + + QMutexLocker l(&d->mutex); + d->updateRooms(); + + int nChannels = ambisonicDecoder ? ambisonicDecoder->nOutputChannels() : 2; + if (len < nChannels*int(sizeof(float))*QAudioEnginePrivate::bufferSize) + return 0; + + short *fd = (short *)data; + qint64 frames = len / nChannels / sizeof(short); + bool ok = true; + while (frames >= qint64(QAudioEnginePrivate::bufferSize)) { + // Fill input buffers + for (auto *source : std::as_const(d->sources)) { + auto *sp = QSpatialSoundPrivate::get(source); + if (!sp) + continue; + float buf[QAudioEnginePrivate::bufferSize]; + sp->getBuffer(buf, QAudioEnginePrivate::bufferSize, 1); + d->resonanceAudio->api->SetInterleavedBuffer(sp->sourceId, buf, 1, QAudioEnginePrivate::bufferSize); + } + for (auto *source : std::as_const(d->stereoSources)) { + auto *sp = QAmbientSoundPrivate::get(source); + if (!sp) + continue; + float buf[2*QAudioEnginePrivate::bufferSize]; + sp->getBuffer(buf, QAudioEnginePrivate::bufferSize, 2); + d->resonanceAudio->api->SetInterleavedBuffer(sp->sourceId, buf, 2, QAudioEnginePrivate::bufferSize); + } + + if (ambisonicDecoder && d->outputMode == QAudioEngine::Surround) { + const float *channels[QAmbisonicDecoder::maxAmbisonicChannels]; + const float *reverbBuffers[2]; + int nSamples = d->resonanceAudio->getAmbisonicOutput(channels, reverbBuffers, ambisonicDecoder->nInputChannels()); + Q_ASSERT(ambisonicDecoder->nOutputChannels() <= 8); + ambisonicDecoder->processBufferWithReverb(channels, reverbBuffers, fd, nSamples); + } else { + ok = d->resonanceAudio->api->FillInterleavedOutputBuffer(2, QAudioEnginePrivate::bufferSize, fd); + if (!ok) { + // If we get here, it means that resonanceAudio did not actually fill the buffer. + // Sometimes this is expected, for example if resonanceAudio does not have any sources. + // In this case we just fill the buffer with silence. + if (d->sources.isEmpty() && d->stereoSources.isEmpty()) { + memset(fd, 0, nChannels * QAudioEnginePrivate::bufferSize * sizeof(short)); + } else { + // If we get here, it means that something unexpected happened, so bail. + qWarning() << " Reading failed!"; + break; + } + } + } + fd += nChannels*QAudioEnginePrivate::bufferSize; + frames -= QAudioEnginePrivate::bufferSize; + } + const int bytesProcessed = ((char *)fd - data); + m_pos += bytesProcessed; + return bytesProcessed; +} + + +QAudioEnginePrivate::QAudioEnginePrivate() +{ + device = QMediaDevices::defaultAudioOutput(); +} + +QAudioEnginePrivate::~QAudioEnginePrivate() +{ + delete resonanceAudio; +} + +void QAudioEnginePrivate::addSpatialSound(QSpatialSound *sound) +{ + QMutexLocker l(&mutex); + QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound); + + sd->sourceId = resonanceAudio->api->CreateSoundObjectSource(vraudio::kBinauralHighQuality); + sources.append(sound); +} + +void QAudioEnginePrivate::removeSpatialSound(QSpatialSound *sound) +{ + QMutexLocker l(&mutex); + QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound); + + resonanceAudio->api->DestroySource(sd->sourceId); + sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId; + sources.removeOne(sound); +} + +void QAudioEnginePrivate::addStereoSound(QAmbientSound *sound) +{ + QMutexLocker l(&mutex); + QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound); + + sd->sourceId = resonanceAudio->api->CreateStereoSource(2); + stereoSources.append(sound); +} + +void QAudioEnginePrivate::removeStereoSound(QAmbientSound *sound) +{ + QMutexLocker l(&mutex); + QAmbientSoundPrivate *sd = QAmbientSoundPrivate::get(sound); + + resonanceAudio->api->DestroySource(sd->sourceId); + sd->sourceId = vraudio::ResonanceAudioApi::kInvalidSourceId; + stereoSources.removeOne(sound); +} + +void QAudioEnginePrivate::addRoom(QAudioRoom *room) +{ + QMutexLocker l(&mutex); + rooms.append(room); +} + +void QAudioEnginePrivate::removeRoom(QAudioRoom *room) +{ + QMutexLocker l(&mutex); + rooms.removeOne(room); +} + +// This method is called from the audio thread +void QAudioEnginePrivate::updateRooms() +{ + if (!roomEffectsEnabled) + return; + + bool needUpdate = listenerPositionDirty; + listenerPositionDirty = false; + + bool roomDirty = false; + for (const auto &room : rooms) { + auto *rd = QAudioRoomPrivate::get(room); + if (rd->dirty) { + roomDirty = true; + rd->update(); + needUpdate = true; + } + } + + if (!needUpdate) + return; + + QVector3D listenerPos = listenerPosition(); + float roomVolume = float(qInf()); + QAudioRoom *room = nullptr; + // Find the smallest room that contains the listener and apply its room effects + for (auto *r : std::as_const(rooms)) { + QVector3D dim2 = r->dimensions()/2.; + float vol = dim2.x()*dim2.y()*dim2.z(); + if (vol > roomVolume) + continue; + QVector3D dist = r->position() - listenerPos; + // transform into room coordinates + dist = r->rotation().rotatedVector(dist); + if (qAbs(dist.x()) <= dim2.x() && + qAbs(dist.y()) <= dim2.y() && + qAbs(dist.z()) <= dim2.z()) { + room = r; + roomVolume = vol; + } + } + if (room != currentRoom) + roomDirty = true; + const bool previousRoom = currentRoom; + currentRoom = room; + + if (!roomDirty) + return; + + // apply room to engine + if (!currentRoom) { + resonanceAudio->api->EnableRoomEffects(false); + return; + } + if (!previousRoom) + resonanceAudio->api->EnableRoomEffects(true); + + QAudioRoomPrivate *rp = QAudioRoomPrivate::get(room); + resonanceAudio->api->SetReflectionProperties(rp->reflections); + resonanceAudio->api->SetReverbProperties(rp->reverb); + + // update room effects for all sound sources + for (auto *s : std::as_const(sources)) { + auto *sp = QSpatialSoundPrivate::get(s); + if (!sp) + continue; + sp->updateRoomEffects(); + } +} + +QVector3D QAudioEnginePrivate::listenerPosition() const +{ + return listener ? listener->position() : QVector3D(); +} + + +/*! + \class QAudioEngine + \inmodule QtSpatialAudio + \ingroup spatialaudio + \ingroup multimedia_audio + + \brief QAudioEngine manages a three dimensional sound field. + + You can use an instance of QAudioEngine to manage a sound field in + three dimensions. A sound field is defined by several QSpatialSound + objects that define a sound at a specified location in 3D space. You can also + add stereo overlays using QAmbientSound. + + You can use QAudioListener to define the position of the person listening + to the sound field relative to the sound sources. Sound sources will be less audible + if the listener is further away from source. They will also get mapped to the corresponding + loudspeakers depending on the direction between listener and source. + + QAudioEngine offers two output modes. The first mode renders the sound field to a set of + speakers, either a stereo speaker pair or a surround configuration. The second mode provides + an immersive 3D sound experience when using headphones. + + Perception of sound localization is driven mainly by two factors. The first factor is timing + differences of the sound waves between left and right ear. The second factor comes from various + ways how sounds coming from different direcations create different types of reflections from our + ears and heads. See https://en.wikipedia.org/wiki/Sound_localization for more details. + + The spatial audio engine emulates those timing differences and reflections through + Head related transfer functions (HRTF, see + https://en.wikipedia.org/wiki/Head-related_transfer_function). The functions used emulates those + effects for an average persons ears and head. It provides a good and immersive 3D sound localization + experience for most persons when using headphones. + + The engine is rather versatile allowing you to define room properties and reverb settings to emulate + different types of rooms. + + Sound sources can also be occluded dampening the sound coming from those sources. + + The audio engine uses a coordinate system that is in centimeters by default. The axes are aligned with the + typical coordinate system used in 3D. Positive x points to the right, positive y points up and positive z points + backwards. + +*/ + +/*! + \fn QAudioEngine::QAudioEngine() + \fn QAudioEngine::QAudioEngine(QObject *parent) + \fn QAudioEngine::QAudioEngine(int sampleRate, QObject *parent = nullptr) + + Constructs a spatial audio engine with \a parent, if any. + + The engine will operate with a sample rate given by \a sampleRate. The + default sample rate, if none is provided, is 44100 (44.1kHz). + + Sound content that is not provided at that sample rate will automatically + get resampled to \a sampleRate when being processed by the engine. The + default sample rate is fine in most cases, but you can define a different + rate if most of your sound files are sampled with a different rate, and + avoid some CPU overhead for resampling. + */ +QAudioEngine::QAudioEngine(int sampleRate, QObject *parent) + : QObject(parent) + , d(new QAudioEnginePrivate) +{ + d->sampleRate = sampleRate; + d->resonanceAudio = new vraudio::ResonanceAudio(2, QAudioEnginePrivate::bufferSize, d->sampleRate); +} + +/*! + Destroys the spatial audio engine. + */ +QAudioEngine::~QAudioEngine() +{ + stop(); + delete d; +} + +/*! \enum QAudioEngine::OutputMode + \value Surround Map the sounds to the loudspeaker configuration of the output device. + This is normally a stereo or surround speaker setup. + \value Stereo Map the sounds to the stereo loudspeaker configuration of the output device. + This will ignore any additional speakers and only use the left and right channels + to create a stero rendering of the sound field. + \value Headphone Use Headphone spatialization to create a 3D audio effect when listening + to the sound field through headphones +*/ + +/*! + \property QAudioEngine::outputMode + + Sets or retrieves the current output mode of the engine. + + \sa QAudioEngine::OutputMode + */ +void QAudioEngine::setOutputMode(OutputMode mode) +{ + if (d->outputMode == mode) + return; + d->outputMode = mode; + if (d->resonanceAudio->api) + d->resonanceAudio->api->SetStereoSpeakerMode(mode != Headphone); + + QMetaObject::invokeMethod(d->outputStream.get(), "restartOutput", Qt::BlockingQueuedConnection); + + emit outputModeChanged(); +} + +QAudioEngine::OutputMode QAudioEngine::outputMode() const +{ + return d->outputMode; +} + +/*! + Returns the sample rate the engine has been configured with. + */ +int QAudioEngine::sampleRate() const +{ + return d->sampleRate; +} + +/*! + \property QAudioEngine::outputDevice + + Sets or returns the device that is being used for playing the sound field. + */ +void QAudioEngine::setOutputDevice(const QAudioDevice &device) +{ + if (d->device == device) + return; + if (d->resonanceAudio->api) { + qWarning() << "Changing device on a running engine not implemented"; + return; + } + d->device = device; + emit outputDeviceChanged(); +} + +QAudioDevice QAudioEngine::outputDevice() const +{ + return d->device; +} + +/*! + \property QAudioEngine::masterVolume + + Sets or returns volume being used to render the sound field. + */ +void QAudioEngine::setMasterVolume(float volume) +{ + if (d->masterVolume == volume) + return; + d->masterVolume = volume; + d->resonanceAudio->api->SetMasterVolume(volume); + emit masterVolumeChanged(); +} + +float QAudioEngine::masterVolume() const +{ + return d->masterVolume; +} + +/*! + Starts the engine. + */ +void QAudioEngine::start() +{ + if (d->outputStream) + // already started + return; + + d->resonanceAudio->api->SetStereoSpeakerMode(d->outputMode != Headphone); + d->resonanceAudio->api->SetMasterVolume(d->masterVolume); + + d->outputStream.reset(new QAudioOutputStream(d)); + d->outputStream->moveToThread(&d->audioThread); + d->audioThread.start(QThread::TimeCriticalPriority); + + QMetaObject::invokeMethod(d->outputStream.get(), "startOutput"); +} + +/*! + Stops the engine. + */ +void QAudioEngine::stop() +{ + QMetaObject::invokeMethod(d->outputStream.get(), "stopOutput", Qt::BlockingQueuedConnection); + d->outputStream.reset(); + d->audioThread.exit(0); + d->audioThread.wait(); + delete d->resonanceAudio->api; + d->resonanceAudio->api = nullptr; +} + +/*! + \property QAudioEngine::paused + + Pauses the spatial audio engine. + */ +void QAudioEngine::setPaused(bool paused) +{ + bool old = d->paused.fetchAndStoreRelaxed(paused); + if (old != paused) { + if (d->outputStream) + d->outputStream->setPaused(paused); + emit pausedChanged(); + } +} + +bool QAudioEngine::paused() const +{ + return d->paused.loadRelaxed(); +} + +/*! + Enables room effects such as echos and reverb. + + Enables room effects if \a enabled is true. + Room effects will only apply if you create one or more \l QAudioRoom objects + and the listener is inside at least one of the rooms. If the listener is inside + multiple rooms, the room with the smallest volume will be used. + */ +void QAudioEngine::setRoomEffectsEnabled(bool enabled) +{ + if (d->roomEffectsEnabled == enabled) + return; + d->roomEffectsEnabled = enabled; + d->resonanceAudio->roomEffectsEnabled = enabled; +} + +/*! + Returns true if room effects are enabled. + */ +bool QAudioEngine::roomEffectsEnabled() const +{ + return d->roomEffectsEnabled; +} + +/*! + \property QAudioEngine::distanceScale + + Defines the scale of the coordinate system being used by the spatial audio engine. + By default, all units are in centimeters, in line with the default units being + used by Qt Quick 3D. + + Set the distance scale to QAudioEngine::DistanceScaleMeter to get units in meters. +*/ +void QAudioEngine::setDistanceScale(float scale) +{ + // multiply with 100, to get the conversion to meters that resonance audio uses + scale /= 100.f; + if (scale <= 0.0f) { + qWarning() << "QAudioEngine: Invalid distance scale."; + return; + } + if (scale == d->distanceScale) + return; + d->distanceScale = scale; + emit distanceScaleChanged(); +} + +float QAudioEngine::distanceScale() const +{ + return d->distanceScale*100.f; +} + +/*! + \fn void QAudioEngine::pause() + + Pauses playback. +*/ +/*! + \fn void QAudioEngine::resume() + + Resumes playback. +*/ +/*! + \variable QAudioEngine::DistanceScaleCentimeter + \internal +*/ +/*! + \variable QAudioEngine::DistanceScaleMeter + \internal +*/ + +QT_END_NAMESPACE + +#include "moc_qaudioengine.cpp" +#include "qaudioengine.moc" diff --git a/src/spatialaudio/qaudioengine.h b/src/spatialaudio/qaudioengine.h new file mode 100644 index 000000000..13de2638a --- /dev/null +++ b/src/spatialaudio/qaudioengine.h @@ -0,0 +1,80 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only + +#ifndef QAUDIOENGINE_H +#define QAUDIOENGINE_H + +#include <QtSpatialAudio/qtspatialaudioglobal.h> +#include <QtCore/qobject.h> + +QT_BEGIN_NAMESPACE + +class QAudioEnginePrivate; +class QAudioDevice; + +class Q_SPATIALAUDIO_EXPORT QAudioEngine : public QObject +{ + Q_OBJECT + Q_PROPERTY(OutputMode outputMode READ outputMode WRITE setOutputMode NOTIFY outputModeChanged) + Q_PROPERTY(QAudioDevice outputDevice READ outputDevice WRITE setOutputDevice NOTIFY outputDeviceChanged) + Q_PROPERTY(float masterVolume READ masterVolume WRITE setMasterVolume NOTIFY masterVolumeChanged) + Q_PROPERTY(bool paused READ paused WRITE setPaused NOTIFY pausedChanged) + Q_PROPERTY(float distanceScale READ distanceScale WRITE setDistanceScale NOTIFY distanceScaleChanged) +public: + QAudioEngine() : QAudioEngine(nullptr) {}; + explicit QAudioEngine(QObject *parent) : QAudioEngine(44100, parent) {} + explicit QAudioEngine(int sampleRate, QObject *parent = nullptr); + ~QAudioEngine(); + + enum OutputMode { + Surround, + Stereo, + Headphone + }; + Q_ENUM(OutputMode) + + void setOutputMode(OutputMode mode); + OutputMode outputMode() const; + + int sampleRate() const; + + void setOutputDevice(const QAudioDevice &device); + QAudioDevice outputDevice() const; + + void setMasterVolume(float volume); + float masterVolume() const; + + void setPaused(bool paused); + bool paused() const; + + void setRoomEffectsEnabled(bool enabled); + bool roomEffectsEnabled() const; + + static constexpr float DistanceScaleCentimeter = 1.f; + static constexpr float DistanceScaleMeter = 100.f; + + void setDistanceScale(float scale); + float distanceScale() const; + +Q_SIGNALS: + void outputModeChanged(); + void outputDeviceChanged(); + void masterVolumeChanged(); + void pausedChanged(); + void distanceScaleChanged(); + +public Q_SLOTS: + void start(); + void stop(); + + void pause() { setPaused(true); } + void resume() { setPaused(false); } + +private: + friend class QAudioEnginePrivate; + QAudioEnginePrivate *d; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qaudioengine_p.h b/src/spatialaudio/qaudioengine_p.h new file mode 100644 index 000000000..59eaa4540 --- /dev/null +++ b/src/spatialaudio/qaudioengine_p.h @@ -0,0 +1,92 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only + +#ifndef QAUDIOENGINE_P_H +#define QAUDIOENGINE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtspatialaudioglobal_p.h> +#include <qaudioengine.h> +#include <qaudiodevice.h> +#include <qaudiodecoder.h> +#include <qthread.h> +#include <qmutex.h> +#include <qurl.h> +#include <qaudiobuffer.h> +#include <qvector3d.h> +#include <qfile.h> + +namespace vraudio { +class ResonanceAudio; +} + +QT_BEGIN_NAMESPACE + +class QSpatialSound; +class QAmbientSound; +class QAudioSink; +class QAudioOutputStream; +class QAmbisonicDecoder; +class QAudioDecoder; +class QAudioRoom; +class QAudioListener; + +class QAudioEnginePrivate +{ +public: + static QAudioEnginePrivate *get(QAudioEngine *engine) { return engine ? engine->d : nullptr; } + + static constexpr int bufferSize = 128; + + QAudioEnginePrivate(); + ~QAudioEnginePrivate(); + vraudio::ResonanceAudio *resonanceAudio = nullptr; + int sampleRate = 44100; + float masterVolume = 1.; + QAudioEngine::OutputMode outputMode = QAudioEngine::Surround; + bool roomEffectsEnabled = true; + + // Resonance Audio uses meters internally, while Qt Quick 3D and our API uses cm by default. + // To make things independent from the scale setting, we store all distances in meters internally + // and convert in the setters and getters. + float distanceScale = 0.01f; + + QMutex mutex; + QAudioDevice device; + QAtomicInteger<bool> paused = false; + + QThread audioThread; + std::unique_ptr<QAudioOutputStream> outputStream; + + QAudioListener *listener = nullptr; + QList<QSpatialSound *> sources; + QList<QAmbientSound *> stereoSources; + QList<QAudioRoom *> rooms; + mutable bool listenerPositionDirty = true; + QAudioRoom *currentRoom = nullptr; + + void addSpatialSound(QSpatialSound *sound); + void removeSpatialSound(QSpatialSound *sound); + void addStereoSound(QAmbientSound *sound); + void removeStereoSound(QAmbientSound *sound); + + void addRoom(QAudioRoom *room); + void removeRoom(QAudioRoom *room); + void updateRooms(); + + QVector3D listenerPosition() const; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qaudiolistener.cpp b/src/spatialaudio/qaudiolistener.cpp new file mode 100644 index 000000000..ed4dbd58e --- /dev/null +++ b/src/spatialaudio/qaudiolistener.cpp @@ -0,0 +1,134 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#include "qaudiolistener.h" +#include "qaudioengine_p.h" +#include "resonance_audio.h" +#include <qaudiosink.h> +#include <qurl.h> +#include <qdebug.h> +#include <qaudiodecoder.h> + +QT_BEGIN_NAMESPACE + +class QAudioListenerPrivate +{ +public: + QAudioEngine *engine = nullptr; + QVector3D pos; + QQuaternion rotation; +}; + +/*! + \class QAudioListener + \inmodule QtSpatialAudio + \ingroup spatialaudio + \ingroup multimedia_audio + + \brief Defines the position and orientation of the person listening to a sound field + defined by QAudioEngine. + + A QAudioEngine can have exactly one listener that defines the position and orientation + of the person listening to the sound field. + */ + +/*! + Creates a listener for the spatial audio engine for \a engine. + */ +QAudioListener::QAudioListener(QAudioEngine *engine) + : d(new QAudioListenerPrivate) +{ + setEngine(engine); +} + +/*! + Destroys the listener. + */ +QAudioListener::~QAudioListener() +{ + // Unregister this listener from the engine + setEngine(nullptr); + delete d; +} + +/*! + Sets the listener's position in 3D space to \a pos. Units are in centimeters + by default. + + \sa QAudioEngine::distanceScale + */ +void QAudioListener::setPosition(QVector3D pos) +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + if (!ep) + return; + pos *= ep->distanceScale; + if (d->pos == pos) + return; + + d->pos = pos; + if (ep && ep->resonanceAudio->api) { + ep->resonanceAudio->api->SetHeadPosition(pos.x(), pos.y(), pos.z()); + ep->listenerPositionDirty = true; + } +} + +/*! + Returns the current position of the listener. + */ +QVector3D QAudioListener::position() const +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + if (!ep) + return QVector3D(); + return d->pos/ep->distanceScale; +} + +/*! + Sets the listener's orientation in 3D space to \a q. + */ +void QAudioListener::setRotation(const QQuaternion &q) +{ + d->rotation = q; + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep && ep->resonanceAudio->api) + ep->resonanceAudio->api->SetHeadRotation(d->rotation.x(), d->rotation.y(), d->rotation.z(), d->rotation.scalar()); +} + +/*! + Returns the listener's orientation in 3D space. + */ +QQuaternion QAudioListener::rotation() const +{ + return d->rotation; +} + +/*! + \internal + */ +void QAudioListener::setEngine(QAudioEngine *engine) +{ + if (d->engine) { + auto *ed = QAudioEnginePrivate::get(d->engine); + ed->listener = nullptr; + } + d->engine = engine; + if (d->engine) { + auto *ed = QAudioEnginePrivate::get(d->engine); + if (ed->listener) { + qWarning() << "Ignoring attempt to add a second listener to the spatial audio engine."; + d->engine = nullptr; + return; + } + ed->listener = this; + } +} + +/*! + Returns the engine associated with this listener. + */ +QAudioEngine *QAudioListener::engine() const +{ + return d->engine; +} + +QT_END_NAMESPACE diff --git a/src/spatialaudio/qaudiolistener.h b/src/spatialaudio/qaudiolistener.h new file mode 100644 index 000000000..daa088b6d --- /dev/null +++ b/src/spatialaudio/qaudiolistener.h @@ -0,0 +1,37 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#ifndef QLISTENER_H +#define QLISTENER_H + +#include <QtSpatialAudio/qtspatialaudioglobal.h> +#include <QtCore/QObject> +#include <QtMultimedia/qaudioformat.h> +#include <QtGui/qvector3d.h> +#include <QtGui/qquaternion.h> + +QT_BEGIN_NAMESPACE + +class QAudioEngine; + +class QAudioListenerPrivate; +class Q_SPATIALAUDIO_EXPORT QAudioListener : public QObject +{ +public: + explicit QAudioListener(QAudioEngine *engine); + ~QAudioListener(); + + void setPosition(QVector3D pos); + QVector3D position() const; + void setRotation(const QQuaternion &q); + QQuaternion rotation() const; + + QAudioEngine *engine() const; + +private: + void setEngine(QAudioEngine *engine); + QAudioListenerPrivate *d = nullptr; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qaudioroom.cpp b/src/spatialaudio/qaudioroom.cpp new file mode 100644 index 000000000..3187abd10 --- /dev/null +++ b/src/spatialaudio/qaudioroom.cpp @@ -0,0 +1,399 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#include <qaudioroom_p.h> + +QT_BEGIN_NAMESPACE + +namespace { +inline QVector3D toVector(const float *f) +{ + return QVector3D(f[0], f[1], f[2]); +} + +inline void toFloats(const QVector3D &v, float *f) +{ + f[0] = v.x(); + f[1] = v.y(); + f[2] = v.z(); +} + +inline QQuaternion toQuaternion(const float *f) +{ + // resonance audio puts the scalar component last + return QQuaternion(f[3], f[0], f[1], f[2]); +} + +inline void toFloats(const QQuaternion &q, float *f) +{ + f[0] = q.x(); + f[1] = q.y(); + f[2] = q.z(); + f[3] = q.scalar(); +} + +// Default values for occlusion and dampening of different wall materials. +// These values are used as defaults if a wall is only defined by a material +// and define how sound passes through the wall. +// We define both occlusion and dampening constants to be able to tune the +// sound. Dampening only reduces the level of the sound without affecting its +// tone, while occlusion will dampen higher frequencies more than lower ones +struct { + float occlusion; + float dampening; +} occlusionAndDampening[] = { + { 0.f, 1.f }, // Transparent, + { 0.f, .1f }, // AcousticCeilingTiles, + { 2.f, .4f }, // BrickBare, + { 2.f, .4f }, // BrickPainted, + { 4.f, 1.f }, // ConcreteBlockCoarse, + { 4.f, 1.f }, // ConcreteBlockPainted, + { .7f, .7f }, // CurtainHeavy, + { .5f, .5f }, // FiberGlassInsulation, + { .2f, .3f }, // GlassThin, + { .5f, .2f }, // GlassThick, + { 7.f, 1.f }, // Grass, + { 4.f, 1.f }, // LinoleumOnConcrete, + { 4.f, 1.f }, // Marble, + { 0.f, .2f }, // Metal, + { 4.f, 1.f }, // ParquetOnConcrete, + { 2.f, .4f }, // PlasterRough, + { 2.f, .4f }, // PlasterSmooth, + { 1.5f, .2f }, // PlywoodPanel, + { 4.f, 1.f }, // PolishedConcreteOrTile, + { 4.f, 1.f }, // Sheetrock, + { 4.f, 1.f }, // WaterOrIceSurface, + { 1.f, .3f }, // WoodCeiling, + { 1.f, .3f }, // WoodPanel, + { 0.f, .0f }, // UniformMaterial, +}; + +} + +// make sure the wall definitions agree with resonance audio + +static_assert(QAudioRoom::LeftWall == 0); +static_assert(QAudioRoom::RightWall == 1); +static_assert(QAudioRoom::Floor == 2); +static_assert(QAudioRoom::Ceiling == 3); +static_assert(QAudioRoom::FrontWall == 4); +static_assert(QAudioRoom::BackWall == 5); + +float QAudioRoomPrivate::wallOcclusion(QAudioRoom::Wall wall) const +{ + return m_wallOcclusion[wall] < 0 ? occlusionAndDampening[roomProperties.material_names[wall]].occlusion : m_wallOcclusion[wall]; +} + +float QAudioRoomPrivate::wallDampening(QAudioRoom::Wall wall) const +{ + return m_wallDampening[wall] < 0 ? occlusionAndDampening[roomProperties.material_names[wall]].dampening : m_wallDampening[wall]; +} + +void QAudioRoomPrivate::update() +{ + if (!dirty) + return; + reflections = vraudio::ComputeReflectionProperties(roomProperties); + reverb = vraudio::ComputeReverbProperties(roomProperties); + dirty = false; +} + + +/*! + \class QAudioRoom + \inmodule QtSpatialAudio + \ingroup spatialaudio + \ingroup multimedia_audio + + Defines a room for the spatial audio engine. + + If the listener is inside a room, first order sound reflections and reverb + matching the rooms properties will get applied to the sound field. + + A room is always square and defined by its center position, its orientation and dimensions. + Each of the 6 walls of the room can be made of different materials that will contribute + to the computed reflections and reverb that the listener will experience while being inside + the room. + + If multiple rooms cover the same position, the engine will use the room with the smallest + volume. + */ + +/*! + Constructs a QAudioRoom for \a engine. + */ +QAudioRoom::QAudioRoom(QAudioEngine *engine) + : d(new QAudioRoomPrivate) +{ + Q_ASSERT(engine); + d->engine = engine; + auto *ep = QAudioEnginePrivate::get(engine); + ep->addRoom(this); +} + +/*! + Destroys the room. + */ +QAudioRoom::~QAudioRoom() +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->removeRoom(this); + delete d; +} + +/*! + \enum QAudioRoom::Material + + Defines different materials that can be applied to the different walls of the room. + + \value Transparent The side of the room is open and won't contribute to reflections or reverb. + \value AcousticCeilingTiles Acoustic tiles that suppress most reflections and reverb. + \value BrickBare Bare brick wall. + \value BrickPainted Painted brick wall. + \value ConcreteBlockCoarse Raw concrete wall + \value ConcreteBlockPainted Painted concrete wall + \value CurtainHeavy Heavy curtain. Will mostly reflect low frequencies + \value FiberGlassInsulation Fiber glass insulation. Only reflects very low frequencies + \value GlassThin Thin glass wall + \value GlassThick Thick glass wall + \value Grass Grass + \value LinoleumOnConcrete Linoleum floor + \value Marble Marble floor + \value Metal Metal + \value ParquetOnConcrete Parquet wooden floor on concrete + \value PlasterRough Rough plaster + \value PlasterSmooth Smooth plaster + \value PlywoodPanel Plywodden panel + \value PolishedConcreteOrTile Polished concrete or tiles + \value Sheetrock Rock + \value WaterOrIceSurface Water or ice + \value WoodCeiling Wooden ceiling + \value WoodPanel Wooden panel + \value UniformMaterial Artificial material giving uniform reflections on all frequencies +*/ + +/*! + \enum QAudioRoom::Wall + + An enum defining the 6 walls of the room + + \value LeftWall Left wall (negative x) + \value RightWall Right wall (positive x) + \value Floor Bottom wall (negative y) + \value Ceiling Top wall (positive y) + \value FrontWall Front wall (negative z) + \value BackWall Back wall (positive z) +*/ + + +/*! + \property QAudioRoom::position + + Defines the position of the center of the room in 3D space. Units are in centimeters + by default. + + \sa dimensions, QAudioEngine::distanceScale + */ +void QAudioRoom::setPosition(QVector3D pos) +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + pos *= ep->distanceScale; + if (toVector(d->roomProperties.position) == pos) + return; + toFloats(pos, d->roomProperties.position); + d->dirty = true; + emit positionChanged(); +} + +QVector3D QAudioRoom::position() const +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + auto pos = toVector(d->roomProperties.position); + pos /= ep->distanceScale; + return pos; +} + +/*! + \property QAudioRoom::dimensions + + Defines the dimensions of the room in 3D space. Units are in centimeters + by default. + + \sa position, QAudioEngine::distanceScale + */ +void QAudioRoom::setDimensions(QVector3D dim) +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + dim *= ep->distanceScale; + if (toVector(d->roomProperties.dimensions) == dim) + return; + toFloats(dim, d->roomProperties.dimensions); + d->dirty = true; + emit dimensionsChanged(); +} + +QVector3D QAudioRoom::dimensions() const +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + auto dim = toVector(d->roomProperties.dimensions); + dim /= ep->distanceScale; + return dim; +} + +/*! + \property QAudioRoom::rotation + + Defines the orientation of the room in 3D space. + */ +void QAudioRoom::setRotation(const QQuaternion &q) +{ + if (toQuaternion(d->roomProperties.rotation) == q) + return; + toFloats(q, d->roomProperties.rotation); + d->dirty = true; + emit rotationChanged(); +} + +QQuaternion QAudioRoom::rotation() const +{ + return toQuaternion(d->roomProperties.rotation); +} + +/*! + \fn void QAudioRoom::wallsChanged() + + Signals when the wall material changes. +*/ +/*! + Sets \a wall to \a material. + + Different wall materials have different reflection and reverb properties + that influence the sound of the room. + + \sa wallMaterial(), Material, QAudioRoom::Wall + */ +void QAudioRoom::setWallMaterial(Wall wall, Material material) +{ + static_assert(vraudio::kUniform == int(UniformMaterial)); + static_assert(vraudio::kTransparent == int(Transparent)); + + if (d->roomProperties.material_names[int(wall)] == int(material)) + return; + d->roomProperties.material_names[int(wall)] = vraudio::MaterialName(int(material)); + d->dirty = true; + emit wallsChanged(); +} + +/*! + returns the material being used for \a wall. + + \sa setWallMaterial(), Material, QAudioRoom::Wall + */ +QAudioRoom::Material QAudioRoom::wallMaterial(Wall wall) const +{ + return Material(d->roomProperties.material_names[int(wall)]); +} + +/*! + \property QAudioRoom::reflectionGain + + A gain factor for reflections generated in this room. A value + from 0 to 1 will dampen reflections, while a value larger than 1 + will apply a gain to reflections, making them louder. + + The default is 1, a factor of 0 disables reflections. Negative + values are mapped to 0. + */ +void QAudioRoom::setReflectionGain(float factor) +{ + if (factor < 0.) + factor = 0.; + if (d->roomProperties.reflection_scalar == factor) + return; + d->roomProperties.reflection_scalar = factor; + d->dirty = true; + emit reflectionGainChanged(); +} + +float QAudioRoom::reflectionGain() const +{ + return d->roomProperties.reflection_scalar; +} + +/*! + \property QAudioRoom::reverbGain + + A gain factor for reverb generated in this room. A value + from 0 to 1 will dampen reverb, while a value larger than 1 + will apply a gain to the reverb, making it louder. + + The default is 1, a factor of 0 disables reverb. Negative + values are mapped to 0. + */ +void QAudioRoom::setReverbGain(float factor) +{ + if (factor < 0) + factor = 0; + if (d->roomProperties.reverb_gain == factor) + return; + d->roomProperties.reverb_gain = factor; + d->dirty = true; + emit reverbGainChanged(); +} + +float QAudioRoom::reverbGain() const +{ + return d->roomProperties.reverb_gain; +} + +/*! + \property QAudioRoom::reverbTime + + A factor to be applies to all reverb timings generated for this room. + Larger values will lead to longer reverb timings, making the room sound + larger. + + The default is 1. Negative values are mapped to 0. + */ +void QAudioRoom::setReverbTime(float factor) +{ + if (factor < 0) + factor = 0; + if (d->roomProperties.reverb_time == factor) + return; + d->roomProperties.reverb_time = factor; + d->dirty = true; + emit reverbTimeChanged(); +} + +float QAudioRoom::reverbTime() const +{ + return d->roomProperties.reverb_time; +} + +/*! + \property QAudioRoom::reverbBrightness + + A brightness factor to be applied to the generated reverb. + A positive value will increase reverb for higher frequencies and + dampen lower frequencies, a negative value does the reverse. + + The default is 0. + */ +void QAudioRoom::setReverbBrightness(float factor) +{ + if (d->roomProperties.reverb_brightness == factor) + return; + d->roomProperties.reverb_brightness = factor; + d->dirty = true; + emit reverbBrightnessChanged(); +} + +float QAudioRoom::reverbBrightness() const +{ + return d->roomProperties.reverb_brightness; +} + +QT_END_NAMESPACE + +#include "moc_qaudioroom.cpp" diff --git a/src/spatialaudio/qaudioroom.h b/src/spatialaudio/qaudioroom.h new file mode 100644 index 000000000..eb2b88cbc --- /dev/null +++ b/src/spatialaudio/qaudioroom.h @@ -0,0 +1,107 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only + +#ifndef QAUDIOROOM_H +#define QAUDIOROOM_H + +#include <QtSpatialAudio/qtspatialaudioglobal.h> +#include <QtCore/qobject.h> +#include <QtGui/qvector3d.h> + +QT_BEGIN_NAMESPACE + +class QAudioEngine; +class QAudioRoomPrivate; + +class Q_SPATIALAUDIO_EXPORT QAudioRoom : public QObject +{ + Q_OBJECT + Q_PROPERTY(QVector3D position READ position WRITE setPosition NOTIFY positionChanged) + Q_PROPERTY(QVector3D dimensions READ dimensions WRITE setDimensions NOTIFY dimensionsChanged) + Q_PROPERTY(QQuaternion rotation READ rotation WRITE setRotation NOTIFY rotationChanged) + Q_PROPERTY(float reflectionGain READ reflectionGain WRITE setReflectionGain NOTIFY reflectionGainChanged) + Q_PROPERTY(float reverbGain READ reverbGain WRITE setReverbGain NOTIFY reverbGainChanged) + Q_PROPERTY(float reverbTime READ reverbTime WRITE setReverbTime NOTIFY reverbTimeChanged) + Q_PROPERTY(float reverbBrightness READ reverbBrightness WRITE setReverbBrightness NOTIFY reverbBrightnessChanged) +public: + explicit QAudioRoom(QAudioEngine *engine); + ~QAudioRoom(); + + enum Material { + Transparent, + AcousticCeilingTiles, + BrickBare, + BrickPainted, + ConcreteBlockCoarse, + ConcreteBlockPainted, + CurtainHeavy, + FiberGlassInsulation, + GlassThin, + GlassThick, + Grass, + LinoleumOnConcrete, + Marble, + Metal, + ParquetOnConcrete, + PlasterRough, + PlasterSmooth, + PlywoodPanel, + PolishedConcreteOrTile, + Sheetrock, + WaterOrIceSurface, + WoodCeiling, + WoodPanel, + UniformMaterial, + }; + + enum Wall { + LeftWall, + RightWall, + Floor, + Ceiling, + FrontWall, + BackWall + }; + + void setPosition(QVector3D pos); + QVector3D position() const; + + void setDimensions(QVector3D dim); + QVector3D dimensions() const; + + void setRotation(const QQuaternion &q); + QQuaternion rotation() const; + + void setWallMaterial(Wall wall, Material material); + Material wallMaterial(Wall wall) const; + + void setReflectionGain(float factor); + float reflectionGain() const; + + void setReverbGain(float factor); + float reverbGain() const; + + void setReverbTime(float factor); + float reverbTime() const; + + void setReverbBrightness(float factor); + float reverbBrightness() const; + +Q_SIGNALS: + void positionChanged(); + void dimensionsChanged(); + void rotationChanged(); + void wallsChanged(); + void reflectionGainChanged(); + void reverbGainChanged(); + void reverbTimeChanged(); + void reverbBrightnessChanged(); + +private: + friend class QAudioRoomPrivate; + QAudioRoomPrivate *d; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qaudioroom_p.h b/src/spatialaudio/qaudioroom_p.h new file mode 100644 index 000000000..b320dc9fc --- /dev/null +++ b/src/spatialaudio/qaudioroom_p.h @@ -0,0 +1,50 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#ifndef QAUDIOROOM_P_H +#define QAUDIOROOM_P_H + +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qtspatialaudioglobal_p.h> +#include <qaudioroom.h> +#include <qaudioengine_p.h> +#include <QtGui/qquaternion.h> + +#include <resonance_audio.h> +#include "platforms/common/room_effects_utils.h" +#include "platforms/common/room_properties.h" + +QT_BEGIN_NAMESPACE + +class QAudioRoomPrivate +{ +public: + static QAudioRoomPrivate *get(const QAudioRoom *r) { return r->d; } + + QAudioEngine *engine = nullptr; + vraudio::RoomProperties roomProperties; + bool dirty = true; + + vraudio::ReverbProperties reverb; + vraudio::ReflectionProperties reflections; + + float m_wallOcclusion[6] = { -1.f, -1.f, -1.f, -1.f, -1.f, -1.f }; + float m_wallDampening[6] = { -1.f, -1.f, -1.f, -1.f, -1.f, -1.f }; + + float wallOcclusion(QAudioRoom::Wall wall) const; + float wallDampening(QAudioRoom::Wall wall) const; + + void update(); +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qspatialsound.cpp b/src/spatialaudio/qspatialsound.cpp new file mode 100644 index 000000000..84512556c --- /dev/null +++ b/src/spatialaudio/qspatialsound.cpp @@ -0,0 +1,595 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#include "qaudioroom_p.h" +#include "qspatialsound_p.h" +#include "qaudiolistener.h" +#include "qaudioengine_p.h" +#include "resonance_audio.h" +#include <qaudiosink.h> +#include <qurl.h> +#include <qdebug.h> +#include <qaudiodecoder.h> + +QT_BEGIN_NAMESPACE + +/*! + \class QSpatialSound + \inmodule QtSpatialAudio + \ingroup spatialaudio + \ingroup multimedia_audio + + \brief A sound object in 3D space. + + QSpatialSound represents an audible object in 3D space. You can define + its position and orientation in space, set the sound it is playing and define a + volume for the object. + + The object can have different attenuation behavior, emit sound mainly in one direction + or spherically, and behave as if occluded by some other object. + */ + +/*! + Creates a spatial sound source for \a engine. The object can be placed in + 3D space and will be louder the closer to the listener it is. + */ +QSpatialSound::QSpatialSound(QAudioEngine *engine) + : d(new QSpatialSoundPrivate(this)) +{ + setEngine(engine); +} + +/*! + Destroys the sound source. + */ +QSpatialSound::~QSpatialSound() +{ + setEngine(nullptr); +} + +/*! + \property QSpatialSound::position + + Defines the position of the sound source in 3D space. Units are in centimeters + by default. + + \sa QAudioEngine::distanceScale + */ +void QSpatialSound::setPosition(QVector3D pos) +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + pos *= ep->distanceScale; + d->pos = pos; + if (ep) + ep->resonanceAudio->api->SetSourcePosition(d->sourceId, pos.x(), pos.y(), pos.z()); + emit positionChanged(); +} + +QVector3D QSpatialSound::position() const +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + return d->pos/ep->distanceScale; +} + +/*! + \property QSpatialSound::rotation + + Defines the orientation of the sound source in 3D space. + */ +void QSpatialSound::setRotation(const QQuaternion &q) +{ + d->rotation = q; + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSourceRotation(d->sourceId, q.x(), q.y(), q.z(), q.scalar()); + emit rotationChanged(); +} + +QQuaternion QSpatialSound::rotation() const +{ + return d->rotation; +} + +/*! + \property QSpatialSound::volume + + Defines the volume of the sound. + + Values between 0 and 1 will attenuate the sound, while values above 1 + provide an additional gain boost. + */ +void QSpatialSound::setVolume(float volume) +{ + if (d->volume == volume) + return; + d->volume = volume; + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSourceVolume(d->sourceId, d->volume*d->wallDampening); + emit volumeChanged(); +} + +float QSpatialSound::volume() const +{ + return d->volume; +} + +/*! + \enum QSpatialSound::DistanceModel + + Defines how the volume of the sound scales with distance to the listener. + + \value Logarithmic Volume decreases logarithmically with distance. + \value Linear Volume decreases linearly with distance. + \value ManualAttenuation Attenuation is defined manually using the + \l manualAttenuation property. +*/ + +/*! + \property QSpatialSound::distanceModel + + Defines distance model for this sound source. The volume starts scaling down + from \l size to \l distanceCutoff. The volume is constant for distances smaller + than size and zero for distances larger than the cutoff distance. + + \sa QSpatialSound::DistanceModel + */ +void QSpatialSound::setDistanceModel(DistanceModel model) +{ + if (d->distanceModel == model) + return; + d->distanceModel = model; + + d->updateDistanceModel(); + emit distanceModelChanged(); +} + +void QSpatialSoundPrivate::updateDistanceModel() +{ + if (!engine || sourceId < 0) + return; + auto *ep = QAudioEnginePrivate::get(engine); + + vraudio::DistanceRolloffModel dm = vraudio::kLogarithmic; + switch (distanceModel) { + case QSpatialSound::DistanceModel::Linear: + dm = vraudio::kLinear; + break; + case QSpatialSound::DistanceModel::ManualAttenuation: + dm = vraudio::kNone; + break; + default: + break; + } + + ep->resonanceAudio->api->SetSourceDistanceModel(sourceId, dm, size, distanceCutoff); +} + +void QSpatialSoundPrivate::updateRoomEffects() +{ + if (!engine || sourceId < 0) + return; + auto *ep = QAudioEnginePrivate::get(engine); + if (!ep->currentRoom) + return; + auto *rp = QAudioRoomPrivate::get(ep->currentRoom); + if (!rp) + return; + + QVector3D roomDim2 = ep->currentRoom->dimensions()/2.; + QVector3D roomPos = ep->currentRoom->position(); + QQuaternion roomRot = ep->currentRoom->rotation(); + QVector3D dist = pos - roomPos; + // transform into room coordinates + dist = roomRot.rotatedVector(dist); + if (qAbs(dist.x()) <= roomDim2.x() && + qAbs(dist.y()) <= roomDim2.y() && + qAbs(dist.z()) <= roomDim2.z()) { + // Source is inside room, apply + ep->resonanceAudio->api->SetSourceRoomEffectsGain(sourceId, 1); + wallDampening = 1.; + wallOcclusion = 0.; + } else { + // ### calculate room occlusion and dampening + // This is a bit of heuristics on top of the heuristic dampening/occlusion numbers for walls + // + // We basically cast a ray from the listener through the walls. If walls have different characteristics + // and we get close to a corner, we try to use some averaging to avoid abrupt changes + auto relativeListenerPos = ep->listenerPosition() - roomPos; + relativeListenerPos = roomRot.rotatedVector(relativeListenerPos); + + auto direction = dist.normalized(); + enum { + X, Y, Z + }; + // Very rough approximation, use the size of the source plus twice the size of our head. + // One could probably improve upon this. + const float transitionDistance = size + 0.4; + QAudioRoom::Wall walls[3]; + walls[X] = direction.x() > 0 ? QAudioRoom::RightWall : QAudioRoom::LeftWall; + walls[Y] = direction.y() > 0 ? QAudioRoom::FrontWall : QAudioRoom::BackWall; + walls[Z] = direction.z() > 0 ? QAudioRoom::Ceiling : QAudioRoom::Floor; + float factors[3] = { 0., 0., 0. }; + bool foundWall = false; + if (direction.x() != 0) { + float sign = direction.x() > 0 ? 1.f : -1.f; + float dx = sign * roomDim2.x() - relativeListenerPos.x(); + QVector3D intersection = relativeListenerPos + direction*dx/direction.x(); + float dy = roomDim2.y() - qAbs(intersection.y()); + float dz = roomDim2.z() - qAbs(intersection.z()); + if (dy > 0 && dz > 0) { +// qDebug() << "Hit with wall X" << walls[0] << dy << dz; + // Ray is hitting this wall + factors[Y] = qMax(0.f, 1.f/3.f - dy/transitionDistance); + factors[Z] = qMax(0.f, 1.f/3.f - dz/transitionDistance); + factors[X] = 1.f - factors[Y] - factors[Z]; + foundWall = true; + } + } + if (!foundWall && direction.y() != 0) { + float sign = direction.y() > 0 ? 1.f : -1.f; + float dy = sign * roomDim2.y() - relativeListenerPos.y(); + QVector3D intersection = relativeListenerPos + direction*dy/direction.y(); + float dx = roomDim2.x() - qAbs(intersection.x()); + float dz = roomDim2.z() - qAbs(intersection.z()); + if (dx > 0 && dz > 0) { + // Ray is hitting this wall +// qDebug() << "Hit with wall Y" << walls[1] << dx << dy; + factors[X] = qMax(0.f, 1.f/3.f - dx/transitionDistance); + factors[Z] = qMax(0.f, 1.f/3.f - dz/transitionDistance); + factors[Y] = 1.f - factors[X] - factors[Z]; + foundWall = true; + } + } + if (!foundWall) { + Q_ASSERT(direction.z() != 0); + float sign = direction.z() > 0 ? 1.f : -1.f; + float dz = sign * roomDim2.z() - relativeListenerPos.z(); + QVector3D intersection = relativeListenerPos + direction*dz/direction.z(); + float dx = roomDim2.x() - qAbs(intersection.x()); + float dy = roomDim2.y() - qAbs(intersection.y()); + if (dx > 0 && dy > 0) { + // Ray is hitting this wall +// qDebug() << "Hit with wall Z" << walls[2]; + factors[X] = qMax(0.f, 1.f/3.f - dx/transitionDistance); + factors[Y] = qMax(0.f, 1.f/3.f - dy/transitionDistance); + factors[Z] = 1.f - factors[X] - factors[Y]; + foundWall = true; + } + } + wallDampening = 0; + wallOcclusion = 0; + for (int i = 0; i < 3; ++i) { + wallDampening += factors[i]*rp->wallDampening(walls[i]); + wallOcclusion += factors[i]*rp->wallOcclusion(walls[i]); + } + +// qDebug() << "intersection with wall" << walls[0] << walls[1] << walls[2] << factors[0] << factors[1] << factors[2] << wallDampening << wallOcclusion; + ep->resonanceAudio->api->SetSourceRoomEffectsGain(sourceId, 0); + } + ep->resonanceAudio->api->SetSoundObjectOcclusionIntensity(sourceId, occlusionIntensity + wallOcclusion); + ep->resonanceAudio->api->SetSourceVolume(sourceId, volume*wallDampening); +} + +QSpatialSound::DistanceModel QSpatialSound::distanceModel() const +{ + return d->distanceModel; +} + +/*! + \property QSpatialSound::size + + Defines the size of the sound source. If the listener is closer to the sound + object than the size, volume will stay constant. The size is also used to for + occlusion calculations, where large sources can be partially occluded by a wall. + */ +void QSpatialSound::setSize(float size) +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + size *= ep->distanceScale; + if (d->size == size) + return; + d->size = size; + + d->updateDistanceModel(); + emit sizeChanged(); +} + +float QSpatialSound::size() const +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + return d->size/ep->distanceScale; +} + +/*! + \property QSpatialSound::distanceCutoff + + Defines a distance beyond which sound coming from the source will cutoff. + If the listener is further away from the sound object than the cutoff + distance it won't be audible anymore. + */ +void QSpatialSound::setDistanceCutoff(float cutoff) +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + cutoff *= ep->distanceScale; + if (d->distanceCutoff == cutoff) + return; + d->distanceCutoff = cutoff; + + d->updateDistanceModel(); + emit distanceCutoffChanged(); +} + +float QSpatialSound::distanceCutoff() const +{ + auto *ep = QAudioEnginePrivate::get(d->engine); + return d->distanceCutoff/ep->distanceScale; +} + +/*! + \property QSpatialSound::manualAttenuation + + Defines a manual attenuation factor if \l distanceModel is set to + QSpatialSound::DistanceModel::ManualAttenuation. + */ +void QSpatialSound::setManualAttenuation(float attenuation) +{ + if (d->manualAttenuation == attenuation) + return; + d->manualAttenuation = attenuation; + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSourceDistanceAttenuation(d->sourceId, d->manualAttenuation); + emit manualAttenuationChanged(); +} + +float QSpatialSound::manualAttenuation() const +{ + return d->manualAttenuation; +} + +/*! + \property QSpatialSound::occlusionIntensity + + Defines how much the object is occluded. 0 implies the object is + not occluded at all, 1 implies the sound source is fully occluded by + another object. + + A fully occluded object will still be audible, but especially higher + frequencies will be dampened. In addition, the object will still + participate in generating reverb and reflections in the room. + + Values larger than 1 are possible to further dampen the direct + sound coming from the source. + + The default is 0. + */ +void QSpatialSound::setOcclusionIntensity(float occlusion) +{ + if (d->occlusionIntensity == occlusion) + return; + d->occlusionIntensity = occlusion; + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSoundObjectOcclusionIntensity(d->sourceId, d->occlusionIntensity + d->wallOcclusion); + emit occlusionIntensityChanged(); +} + +float QSpatialSound::occlusionIntensity() const +{ + return d->occlusionIntensity; +} + +/*! + \property QSpatialSound::directivity + + Defines the directivity of the sound source. A value of 0 implies that the sound is + emitted equally in all directions, while a value of 1 implies that the source mainly + emits sound in the forward direction. + + Valid values are between 0 and 1, the default is 0. + */ +void QSpatialSound::setDirectivity(float alpha) +{ + alpha = qBound(0., alpha, 1.); + if (alpha == d->directivity) + return; + d->directivity = alpha; + + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSoundObjectDirectivity(d->sourceId, d->directivity, d->directivityOrder); + + emit directivityChanged(); +} + +float QSpatialSound::directivity() const +{ + return d->directivity; +} + +/*! + \property QSpatialSound::directivityOrder + + Defines the order of the directivity of the sound source. A higher order + implies a sharper localization of the sound cone. + + The minimum value and default for this property is 1. + */ +void QSpatialSound::setDirectivityOrder(float order) +{ + order = qMax(order, 1.); + if (order == d->directivityOrder) + return; + d->directivityOrder = order; + + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSoundObjectDirectivity(d->sourceId, d->directivity, d->directivityOrder); + + emit directivityChanged(); +} + +float QSpatialSound::directivityOrder() const +{ + return d->directivityOrder; +} + +/*! + \property QSpatialSound::nearFieldGain + + Defines the near field gain for the sound source. Valid values are between 0 and 1. + A near field gain of 1 will raise the volume of the sound signal by approx 20 dB for + distances very close to the listener. + */ +void QSpatialSound::setNearFieldGain(float gain) +{ + gain = qBound(0., gain, 1.); + if (gain == d->nearFieldGain) + return; + d->nearFieldGain = gain; + + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->resonanceAudio->api->SetSoundObjectNearFieldEffectGain(d->sourceId, d->nearFieldGain*9.f); + + emit nearFieldGainChanged(); + +} + +float QSpatialSound::nearFieldGain() const +{ + return d->nearFieldGain; +} + +/*! + \property QSpatialSound::source + + The source file for the sound to be played. + */ +void QSpatialSound::setSource(const QUrl &url) +{ + if (d->url == url) + return; + d->url = url; + + d->load(); + emit sourceChanged(); +} + +QUrl QSpatialSound::source() const +{ + return d->url; +} + +/*! + \enum QSpatialSound::Loops + + Lets you control the sound playback loop using the following values: + + \value Infinite Playback infinitely + \value Once Playback once +*/ +/*! + \property QSpatialSound::loops + + Determines how many times the sound is played before the player stops. + Set to QSpatialSound::Infinite to play the current sound in a loop forever. + + The default value is \c 1. + */ +int QSpatialSound::loops() const +{ + return d->m_loops.loadRelaxed(); +} + +void QSpatialSound::setLoops(int loops) +{ + int oldLoops = d->m_loops.fetchAndStoreRelaxed(loops); + if (oldLoops != loops) + emit loopsChanged(); +} + +/*! + \property QSpatialSound::autoPlay + + Determines whether the sound should automatically start playing when a source + gets specified. + + The default value is \c true. + */ +bool QSpatialSound::autoPlay() const +{ + return d->m_autoPlay.loadRelaxed(); +} + +void QSpatialSound::setAutoPlay(bool autoPlay) +{ + bool old = d->m_autoPlay.fetchAndStoreRelaxed(autoPlay); + if (old != autoPlay) + emit autoPlayChanged(); +} + +/*! + Starts playing back the sound. Does nothing if the sound is already playing. + */ +void QSpatialSound::play() +{ + d->play(); +} + +/*! + Pauses sound playback. Calling play() will continue playback. + */ +void QSpatialSound::pause() +{ + d->pause(); +} + +/*! + Stops sound playback and resets the current position and current loop count to 0. + Calling play() will start playback at the beginning of the sound file. + */ +void QSpatialSound::stop() +{ + d->stop(); +} + +/*! + \internal + */ +void QSpatialSound::setEngine(QAudioEngine *engine) +{ + if (d->engine == engine) + return; + + // Remove self from old engine (if necessary) + auto *ep = QAudioEnginePrivate::get(d->engine); + if (ep) + ep->removeSpatialSound(this); + + d->engine = engine; + + // Add self to new engine if necessary + ep = QAudioEnginePrivate::get(d->engine); + if (ep) { + ep->addSpatialSound(this); + ep->resonanceAudio->api->SetSourcePosition(d->sourceId, d->pos.x(), d->pos.y(), d->pos.z()); + ep->resonanceAudio->api->SetSourceRotation(d->sourceId, d->rotation.x(), d->rotation.y(), d->rotation.z(), d->rotation.scalar()); + ep->resonanceAudio->api->SetSourceVolume(d->sourceId, d->volume); + ep->resonanceAudio->api->SetSoundObjectDirectivity(d->sourceId, d->directivity, d->directivityOrder); + ep->resonanceAudio->api->SetSoundObjectNearFieldEffectGain(d->sourceId, d->nearFieldGain); + d->updateDistanceModel(); + } +} + +/*! + Returns the engine associated with this listener. + */ +QAudioEngine *QSpatialSound::engine() const +{ + return d->engine; +} + +QT_END_NAMESPACE + +#include "moc_qspatialsound.cpp" diff --git a/src/spatialaudio/qspatialsound.h b/src/spatialaudio/qspatialsound.h new file mode 100644 index 000000000..d171e0526 --- /dev/null +++ b/src/spatialaudio/qspatialsound.h @@ -0,0 +1,127 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only +#ifndef QSPATIALSOUND_H +#define QSPATIALSOUND_H + +#include <QtSpatialAudio/qtspatialaudioglobal.h> +#include <QtCore/QObject> +#include <QtGui/qvector3d.h> +#include <QtGui/qquaternion.h> + +QT_BEGIN_NAMESPACE + +class QAudioEngine; +class QAmbientSoundPrivate; + +class QSpatialSoundPrivate; +class Q_SPATIALAUDIO_EXPORT QSpatialSound : public QObject +{ + Q_OBJECT + Q_PROPERTY(QUrl source READ source WRITE setSource NOTIFY sourceChanged) + Q_PROPERTY(QVector3D position READ position WRITE setPosition NOTIFY positionChanged) + Q_PROPERTY(QQuaternion rotation READ rotation WRITE setRotation NOTIFY rotationChanged) + Q_PROPERTY(float volume READ volume WRITE setVolume NOTIFY volumeChanged) + Q_PROPERTY(DistanceModel distanceModel READ distanceModel WRITE setDistanceModel NOTIFY distanceModelChanged) + Q_PROPERTY(float size READ size WRITE setSize NOTIFY sizeChanged) + Q_PROPERTY(float distanceCutoff READ distanceCutoff WRITE setDistanceCutoff NOTIFY distanceCutoffChanged) + Q_PROPERTY(float manualAttenuation READ manualAttenuation WRITE setManualAttenuation NOTIFY manualAttenuationChanged) + Q_PROPERTY(float occlusionIntensity READ occlusionIntensity WRITE setOcclusionIntensity NOTIFY occlusionIntensityChanged) + Q_PROPERTY(float directivity READ directivity WRITE setDirectivity NOTIFY directivityChanged) + Q_PROPERTY(float directivityOrder READ directivityOrder WRITE setDirectivityOrder NOTIFY directivityOrderChanged) + Q_PROPERTY(float nearFieldGain READ nearFieldGain WRITE setNearFieldGain NOTIFY nearFieldGainChanged) + Q_PROPERTY(int loops READ loops WRITE setLoops NOTIFY loopsChanged) + Q_PROPERTY(bool autoPlay READ autoPlay WRITE setAutoPlay NOTIFY autoPlayChanged) + +public: + explicit QSpatialSound(QAudioEngine *engine); + ~QSpatialSound(); + + void setSource(const QUrl &url); + QUrl source() const; + + enum Loops + { + Infinite = -1, + Once = 1 + }; + Q_ENUM(Loops) + + int loops() const; + void setLoops(int loops); + + bool autoPlay() const; + void setAutoPlay(bool autoPlay); + + void setPosition(QVector3D pos); + QVector3D position() const; + + void setRotation(const QQuaternion &q); + QQuaternion rotation() const; + + void setVolume(float volume); + float volume() const; + + enum class DistanceModel { + Logarithmic, + Linear, + ManualAttenuation + }; + Q_ENUM(DistanceModel); + + void setDistanceModel(DistanceModel model); + DistanceModel distanceModel() const; + + void setSize(float size); + float size() const; + + void setDistanceCutoff(float cutoff); + float distanceCutoff() const; + + void setManualAttenuation(float attenuation); + float manualAttenuation() const; + + void setOcclusionIntensity(float occlusion); + float occlusionIntensity() const; + + void setDirectivity(float alpha); + float directivity() const; + + void setDirectivityOrder(float alpha); + float directivityOrder() const; + + void setNearFieldGain(float gain); + float nearFieldGain() const; + + QAudioEngine *engine() const; + +Q_SIGNALS: + void sourceChanged(); + void loopsChanged(); + void autoPlayChanged(); + void positionChanged(); + void rotationChanged(); + void volumeChanged(); + void distanceModelChanged(); + void sizeChanged(); + void distanceCutoffChanged(); + void manualAttenuationChanged(); + void occlusionIntensityChanged(); + void directivityChanged(); + void directivityOrderChanged(); + void nearFieldGainChanged(); + +public Q_SLOTS: + void play(); + void pause(); + void stop(); + +private: + void setEngine(QAudioEngine *engine); + friend class QAmbientSoundPrivate; + friend class QSpatialSoundPrivate; + QSpatialSoundPrivate *d = nullptr; +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qspatialsound_p.h b/src/spatialaudio/qspatialsound_p.h new file mode 100644 index 000000000..6e1b5d422 --- /dev/null +++ b/src/spatialaudio/qspatialsound_p.h @@ -0,0 +1,62 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only + +#ifndef QSPATIALAUDIOSOUNDSOURCE_P_H +#define QSPATIALAUDIOSOUNDSOURCE_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists for the convenience +// of other Qt classes. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <qspatialsound.h> +#include <qambientsound_p.h> +#include <qaudioengine_p.h> +#include <qurl.h> +#include <qvector3d.h> +#include <qquaternion.h> +#include <qaudiobuffer.h> +#include <qaudiodevice.h> +#include <qmutex.h> + +QT_BEGIN_NAMESPACE + +class QAudioDecoder; +class QAudioEnginePrivate; + +class QSpatialSoundPrivate : public QAmbientSoundPrivate +{ +public: + QSpatialSoundPrivate(QObject *parent) + : QAmbientSoundPrivate(parent, 1) + {} + + static QSpatialSoundPrivate *get(QSpatialSound *soundSource) + { return soundSource ? soundSource->d : nullptr; } + + QVector3D pos; + QQuaternion rotation; + QSpatialSound::DistanceModel distanceModel = QSpatialSound::DistanceModel::Logarithmic; + float size = .1f; + float distanceCutoff = 50.f; + float manualAttenuation = 0.f; + float occlusionIntensity = 0.f; + float directivity = 0.f; + float directivityOrder = 1.f; + float nearFieldGain = 0.f; + float wallDampening = 1.f; + float wallOcclusion = 0.f; + + void updateDistanceModel(); + void updateRoomEffects(); +}; + +QT_END_NAMESPACE + +#endif diff --git a/src/spatialaudio/qtspatialaudioglobal.h b/src/spatialaudio/qtspatialaudioglobal.h new file mode 100644 index 000000000..6e5b1ec0c --- /dev/null +++ b/src/spatialaudio/qtspatialaudioglobal.h @@ -0,0 +1,10 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only + +#ifndef QTSPATIALAUDIOGLOBAL_H +#define QTSPATIALAUDIOGLOBAL_H + +#include <QtMultimedia/qtmultimediaglobal.h> +#include <QtSpatialAudio/qtspatialaudioexports.h> + +#endif // QTMULTIMEDIAGLOBAL_H diff --git a/src/spatialaudio/qtspatialaudioglobal_p.h b/src/spatialaudio/qtspatialaudioglobal_p.h new file mode 100644 index 000000000..a504da6cd --- /dev/null +++ b/src/spatialaudio/qtspatialaudioglobal_p.h @@ -0,0 +1,21 @@ +// Copyright (C) 2022 The Qt Company Ltd. +// SPDX-License-Identifier: LicenseRef-Qt-Commercial OR LGPL-3.0-only OR GPL-3.0-only + +#ifndef QTSPATIALAUDIOGLOBAL_P_H +#define QTSPATIALAUDIOGLOBAL_P_H + +// +// W A R N I N G +// ------------- +// +// This file is not part of the Qt API. It exists purely as an +// implementation detail. This header file may change from version to +// version without notice, or even be removed. +// +// We mean it. +// + +#include <QtMultimedia/private/qtmultimediaglobal_p.h> +#include <QtSpatialAudio/qtspatialaudioglobal.h> + +#endif // QTQMLGLOBAL_P_H |