From a4f90d61bd2e36333401b011b59cdb94105b615b Mon Sep 17 00:00:00 2001 From: Lars Knoll Date: Tue, 10 May 2022 08:22:06 +0200 Subject: Properly handle transmission through room walls Different wall materials will cause differnt dampening and occlusions of the sound sources behind. To ensure smooth transitions, use a switchover at the corners of the room. Change-Id: If9a12d1c4fec1c08cb7aba1097ac1562c9c6fb84 Reviewed-by: Piotr Srebrny Reviewed-by: Lars Knoll --- src/multimedia/CMakeLists.txt | 1 + src/multimedia/spatial/qspatialaudioengine.cpp | 50 +++++---- src/multimedia/spatial/qspatialaudioengine_p.h | 6 +- src/multimedia/spatial/qspatialaudioroom.cpp | 55 +++++++++- src/multimedia/spatial/qspatialaudioroom_p.h | 6 ++ .../spatial/qspatialaudiosoundsource.cpp | 113 ++++++++++++++++++++- src/multimedia/spatial/qspatialaudiosoundsource.h | 1 + .../spatial/qspatialaudiosoundsource_p.h | 10 +- 8 files changed, 215 insertions(+), 27 deletions(-) diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt index a5bfa2605..4c295072c 100644 --- a/src/multimedia/CMakeLists.txt +++ b/src/multimedia/CMakeLists.txt @@ -93,6 +93,7 @@ qt_internal_add_module(Multimedia qt_internal_extend_target(Multimedia CONDITION QT_FEATURE_spatialaudio SOURCES spatial/qambisonicdecoder.cpp spatial/qambisonicdecoder_p.h spatial/qambisonicdecoderdata_p.h +# spatial/qacousticgeometry.cpp spatial/qacousticgeometry_p.h spatial/qspatialaudioengine.cpp spatial/qspatialaudioengine.h spatial/qspatialaudioengine_p.h spatial/qspatialaudiolistener.cpp spatial/qspatialaudiolistener.h spatial/qspatialaudioroom.cpp spatial/qspatialaudioroom.h spatial/qspatialaudioroom_p.h diff --git a/src/multimedia/spatial/qspatialaudioengine.cpp b/src/multimedia/spatial/qspatialaudioengine.cpp index 6b5a00c9b..6a0b831e8 100644 --- a/src/multimedia/spatial/qspatialaudioengine.cpp +++ b/src/multimedia/spatial/qspatialaudioengine.cpp @@ -222,7 +222,7 @@ void QSpatialAudioEnginePrivate::removeRoom(QSpatialAudioRoom *room) rooms.removeOne(room); } -void QSpatialAudioEnginePrivate::updateRooms() const +void QSpatialAudioEnginePrivate::updateRooms() { if (!roomEffectsEnabled) return; @@ -230,9 +230,11 @@ void QSpatialAudioEnginePrivate::updateRooms() const bool needUpdate = listenerPositionDirty; listenerPositionDirty = false; + bool roomDirty = false; for (const auto &room : rooms) { auto *rd = QSpatialAudioRoomPrivate::get(room); if (rd->dirty) { + roomDirty = true; rd->update(); needUpdate = true; } @@ -241,37 +243,51 @@ void QSpatialAudioEnginePrivate::updateRooms() const if (!needUpdate) return; - QVector3D listenerPos; - if (listener) - listenerPos = listener->position(); + QVector3D listenerPos = listenerPosition(); float roomVolume = float(qInf()); - const QSpatialAudioRoom *room = nullptr; + QSpatialAudioRoom *room = nullptr; // Find the smallest room that contains the listener and apply it's room effects - for (const auto *r : rooms) { - QVector3D dimensions = r->dimensions(); - float vol = dimensions.x()*dimensions.y()*dimensions.z(); + for (auto *r : qAsConst(rooms)) { + QVector3D dim2 = r->dimensions()/2.; + float vol = dim2.x()*dim2.y()*dim2.z(); if (vol > roomVolume) continue; QVector3D dist = r->position() - listenerPos; // transform into room coordinates dist = r->rotation().rotatedVector(dist); - if (qAbs(dist.x()) <= dimensions.x() && - qAbs(dist.y()) <= dimensions.y() && - qAbs(dist.z()) <= dimensions.z()) { + if (qAbs(dist.x()) <= dim2.x() && + qAbs(dist.y()) <= dim2.y() && + qAbs(dist.z()) <= dim2.z()) { room = r; roomVolume = vol; } } + if (room != currentRoom) + roomDirty = true; + currentRoom = room; + + if (!roomDirty) + return; // apply room to engine - if (room) { - QSpatialAudioRoomPrivate *rp = QSpatialAudioRoomPrivate::get(room); - api->EnableRoomEffects(true); - api->SetReflectionProperties(rp->reflections); - api->SetReverbProperties(rp->reverb); - } else { + if (!currentRoom) { api->EnableRoomEffects(false); + return; } + QSpatialAudioRoomPrivate *rp = QSpatialAudioRoomPrivate::get(room); + api->SetReflectionProperties(rp->reflections); + api->SetReverbProperties(rp->reverb); + + // update room effects for all sound sources + for (auto *s : qAsConst(sources)) { + auto *sp = QSpatialAudioSoundSourcePrivate::get(s); + sp->updateRoomEffects(); + } +} + +QVector3D QSpatialAudioEnginePrivate::listenerPosition() const +{ + return listener ? listener->position() : QVector3D(); } diff --git a/src/multimedia/spatial/qspatialaudioengine_p.h b/src/multimedia/spatial/qspatialaudioengine_p.h index 4bedf4a1b..b5df68451 100644 --- a/src/multimedia/spatial/qspatialaudioengine_p.h +++ b/src/multimedia/spatial/qspatialaudioengine_p.h @@ -56,6 +56,7 @@ #include #include #include +#include namespace vraudio { class ResonanceAudioApi; @@ -101,6 +102,7 @@ public: QList stereoSources; QList rooms; mutable bool listenerPositionDirty = true; + QSpatialAudioRoom *currentRoom = nullptr; void addSpatialSound(QSpatialAudioSoundSource *sound); void removeSpatialSound(QSpatialAudioSoundSource *sound); @@ -109,7 +111,9 @@ public: void addRoom(QSpatialAudioRoom *room); void removeRoom(QSpatialAudioRoom *room); - void updateRooms() const; + void updateRooms(); + + QVector3D listenerPosition() const; }; class QSpatialAudioSound : public QObject diff --git a/src/multimedia/spatial/qspatialaudioroom.cpp b/src/multimedia/spatial/qspatialaudioroom.cpp index 2b40028e8..342127466 100644 --- a/src/multimedia/spatial/qspatialaudioroom.cpp +++ b/src/multimedia/spatial/qspatialaudioroom.cpp @@ -64,6 +64,53 @@ inline void toFloats(const QQuaternion &q, float *f) f[2] = q.z(); f[3] = q.scalar(); } + +// Default values for occlusion and dampening of different wall materials. +// These values are used as defaults if a wall is only defined by a material +// and define how sound passes through the wall. +// We define both occlusion and dampening constants to be able to tune the +// sound. Dampening only reduces the level of the sound without affecting its +// tone, while occlusion will dampen higher frequencies more than lower ones +struct { + float occlusion; + float dampening; +} occlusionAndDampening[] = { + { 0.f, 1.f }, // Transparent, + { 0.f, .1f }, // AcousticCeilingTiles, + { 2.f, .4f }, // BrickBare, + { 2.f, .4f }, // BrickPainted, + { 4.f, 1.f }, // ConcreteBlockCoarse, + { 4.f, 1.f }, // ConcreteBlockPainted, + { .7f, .7f }, // CurtainHeavy, + { .5f, .5f }, // FiberGlassInsulation, + { .2f, .3f }, // GlassThin, + { .5f, .2f }, // GlassThick, + { 7.f, 1.f }, // Grass, + { 4.f, 1.f }, // LinoleumOnConcrete, + { 4.f, 1.f }, // Marble, + { 0.f, .2f }, // Metal, + { 4.f, 1.f }, // ParquetOnConcrete, + { 2.f, .4f }, // PlasterRough, + { 2.f, .4f }, // PlasterSmooth, + { 1.5f, .2f }, // PlywoodPanel, + { 4.f, 1.f }, // PolishedConcreteOrTile, + { 4.f, 1.f }, // Sheetrock, + { 4.f, 1.f }, // WaterOrIceSurface, + { 1.f, .3f }, // WoodCeiling, + { 1.f, .3f }, // WoodPanel, + { 0.f, .0f }, // UniformMaterial, +}; + +} + +float QSpatialAudioRoomPrivate::wallOcclusion(QSpatialAudioRoom::Wall wall) const +{ + return m_wallOcclusion[wall] < 0 ? occlusionAndDampening[roomProperties.material_names[wall]].occlusion : m_wallOcclusion[wall]; +} + +float QSpatialAudioRoomPrivate::wallDampening(QSpatialAudioRoom::Wall wall) const +{ + return m_wallDampening[wall] < 0 ? occlusionAndDampening[roomProperties.material_names[wall]].dampening : m_wallDampening[wall]; } void QSpatialAudioRoomPrivate::update() @@ -155,10 +202,10 @@ QSpatialAudioRoom::~QSpatialAudioRoom() \value LeftWall Left wall (negative x) \value RightWall Right wall (positive x) - \value BackWall Back wall (negative z) - \value FrontWall Front wall (positive z) - \value Floor Bottom wall (negative y) - \value Ceiling Top wall (positive y) + \value BackWall Back wall (negative y) + \value FrontWall Front wall (positive y) + \value Floor Bottom wall (negative z) + \value Ceiling Top wall (positive z) */ diff --git a/src/multimedia/spatial/qspatialaudioroom_p.h b/src/multimedia/spatial/qspatialaudioroom_p.h index d7f78b6bd..668570eef 100644 --- a/src/multimedia/spatial/qspatialaudioroom_p.h +++ b/src/multimedia/spatial/qspatialaudioroom_p.h @@ -69,6 +69,12 @@ public: vraudio::ReverbProperties reverb; vraudio::ReflectionProperties reflections; + float m_wallOcclusion[6] = { -1.f, -1.f, -1.f, -1.f, -1.f, -1.f }; + float m_wallDampening[6] = { -1.f, -1.f, -1.f, -1.f, -1.f, -1.f }; + + float wallOcclusion(QSpatialAudioRoom::Wall wall) const; + float wallDampening(QSpatialAudioRoom::Wall wall) const; + void update(); }; diff --git a/src/multimedia/spatial/qspatialaudiosoundsource.cpp b/src/multimedia/spatial/qspatialaudiosoundsource.cpp index ac7fa170d..2fd6ecef1 100644 --- a/src/multimedia/spatial/qspatialaudiosoundsource.cpp +++ b/src/multimedia/spatial/qspatialaudiosoundsource.cpp @@ -34,9 +34,11 @@ ** $QT_END_LICENSE$ ** ****************************************************************************/ +#include "qspatialaudioroom_p.h" #include "qspatialaudiosoundsource_p.h" #include "qspatialaudiolistener.h" #include "qspatialaudioengine_p.h" +#include "qspatialaudioroom.h" #include "api/resonance_audio_api.h" #include #include @@ -90,6 +92,7 @@ void QSpatialAudioSoundSource::setPosition(QVector3D pos) auto *ep = QSpatialAudioEnginePrivate::get(d->engine); if (ep) ep->api->SetSourcePosition(d->sourceId, pos.x(), pos.y(), pos.z()); + d->updateRoomEffects(); emit positionChanged(); } @@ -132,7 +135,7 @@ void QSpatialAudioSoundSource::setVolume(float volume) d->volume = volume; auto *ep = QSpatialAudioEnginePrivate::get(d->engine); if (ep) - ep->api->SetSourceVolume(d->sourceId, d->volume); + ep->api->SetSourceVolume(d->sourceId, d->volume*d->wallDampening); emit volumeChanged(); } @@ -171,9 +174,9 @@ void QSpatialAudioSoundSource::setDistanceModel(DistanceModel model) void QSpatialAudioSoundSourcePrivate::updateDistanceModel() { - auto *ep = QSpatialAudioEnginePrivate::get(engine); if (!engine || sourceId < 0) return; + auto *ep = QSpatialAudioEnginePrivate::get(engine); vraudio::DistanceRolloffModel dm = vraudio::kLogarithmic; switch (distanceModel) { @@ -190,6 +193,110 @@ void QSpatialAudioSoundSourcePrivate::updateDistanceModel() ep->api->SetSourceDistanceModel(sourceId, dm, minDistance, maxDistance); } +void QSpatialAudioSoundSourcePrivate::updateRoomEffects() +{ + if (!engine || sourceId < 0) + return; + auto *ep = QSpatialAudioEnginePrivate::get(engine); + if (!ep->currentRoom) + return; + auto *rp = QSpatialAudioRoomPrivate::get(ep->currentRoom); + + QVector3D roomDim2 = ep->currentRoom->dimensions()/2.; + QVector3D roomPos = ep->currentRoom->position(); + QQuaternion roomRot = ep->currentRoom->rotation(); + QVector3D dist = pos - roomPos; + // transform into room coordinates + dist = roomRot.rotatedVector(dist); + if (qAbs(dist.x()) <= roomDim2.x() && + qAbs(dist.y()) <= roomDim2.y() && + qAbs(dist.z()) <= roomDim2.z()) { + // Source is inside room, apply + ep->api->SetSourceRoomEffectsGain(sourceId, 1); + wallDampening = 1.; + wallOcclusion = 0.; + } else { + // ### calculate room occlusion and dampening + // This is a bit of heuristics on top of the heuristic dampening/occlusion numbers for walls + // + // We basically cast a ray from the listener through the walls. If walls have different characteristics + // and we get close to a corner, we try to use some averaging to avoid abrupt changes + auto relativeListenerPos = ep->listenerPosition() - roomPos; + relativeListenerPos = roomRot.rotatedVector(relativeListenerPos); + + auto direction = dist.normalized(); + enum { + X, Y, Z + }; + // Very rough approximation, use the size of the source plus twice the size of our head. + // One could probably improve upon this. + const float transitionDistance = minDistance + 0.4; + QSpatialAudioRoom::Wall walls[3]; + walls[X] = direction.x() > 0 ? QSpatialAudioRoom::RightWall : QSpatialAudioRoom::LeftWall; + walls[Y] = direction.y() > 0 ? QSpatialAudioRoom::FrontWall : QSpatialAudioRoom::BackWall; + walls[Z] = direction.z() > 0 ? QSpatialAudioRoom::Ceiling : QSpatialAudioRoom::Floor; + float factors[3] = { 0., 0., 0. }; + bool foundWall = false; + if (direction.x() != 0) { + float sign = direction.x() > 0 ? 1.f : -1.f; + float dx = sign * roomDim2.x() - relativeListenerPos.x(); + QVector3D intersection = relativeListenerPos + direction*dx/direction.x(); + float dy = roomDim2.y() - qAbs(intersection.y()); + float dz = roomDim2.z() - qAbs(intersection.z()); + if (dy > 0 && dz > 0) { +// qDebug() << "Hit with wall X" << walls[0] << dy << dz; + // Ray is hitting this wall + factors[Y] = qMax(0.f, 1.f/3.f - dy/transitionDistance); + factors[Z] = qMax(0.f, 1.f/3.f - dz/transitionDistance); + factors[X] = 1.f - factors[Y] - factors[Z]; + foundWall = true; + } + } + if (!foundWall && direction.y() != 0) { + float sign = direction.y() > 0 ? 1.f : -1.f; + float dy = sign * roomDim2.y() - relativeListenerPos.y(); + QVector3D intersection = relativeListenerPos + direction*dy/direction.y(); + float dx = roomDim2.x() - qAbs(intersection.x()); + float dz = roomDim2.z() - qAbs(intersection.z()); + if (dx > 0 && dz > 0) { + // Ray is hitting this wall +// qDebug() << "Hit with wall Y" << walls[1] << dx << dy; + factors[X] = qMax(0.f, 1.f/3.f - dx/transitionDistance); + factors[Z] = qMax(0.f, 1.f/3.f - dz/transitionDistance); + factors[Y] = 1.f - factors[X] - factors[Z]; + foundWall = true; + } + } + if (!foundWall) { + Q_ASSERT(direction.z() != 0); + float sign = direction.z() > 0 ? 1.f : -1.f; + float dz = sign * roomDim2.z() - relativeListenerPos.z(); + QVector3D intersection = relativeListenerPos + direction*dz/direction.z(); + float dx = roomDim2.x() - qAbs(intersection.x()); + float dy = roomDim2.y() - qAbs(intersection.y()); + if (dx > 0 && dy > 0) { + // Ray is hitting this wall +// qDebug() << "Hit with wall Z" << walls[2]; + factors[X] = qMax(0.f, 1.f/3.f - dx/transitionDistance); + factors[Y] = qMax(0.f, 1.f/3.f - dy/transitionDistance); + factors[Z] = 1.f - factors[X] - factors[Y]; + foundWall = true; + } + } + wallDampening = 0; + wallOcclusion = 0; + for (int i = 0; i < 3; ++i) { + wallDampening += factors[i]*rp->wallDampening(walls[i]); + wallOcclusion += factors[i]*rp->wallOcclusion(walls[i]); + } + +// qDebug() << "intersection with wall" << walls[0] << walls[1] << walls[2] << factors[0] << factors[1] << factors[2] << wallDampening << wallOcclusion; + ep->api->SetSourceRoomEffectsGain(sourceId, 0); + } + ep->api->SetSoundObjectOcclusionIntensity(sourceId, occlusionIntensity + wallOcclusion); + ep->api->SetSourceVolume(sourceId, volume*wallDampening); +} + QSpatialAudioSoundSource::DistanceModel QSpatialAudioSoundSource::distanceModel() const { return d->distanceModel; @@ -283,7 +390,7 @@ void QSpatialAudioSoundSource::setOcclusionIntensity(float occlusion) d->occlusionIntensity = occlusion; auto *ep = QSpatialAudioEnginePrivate::get(d->engine); if (ep) - ep->api->SetSoundObjectOcclusionIntensity(d->sourceId, d->occlusionIntensity); + ep->api->SetSoundObjectOcclusionIntensity(d->sourceId, d->occlusionIntensity + d->wallOcclusion); emit occlusionIntensityChanged(); } diff --git a/src/multimedia/spatial/qspatialaudiosoundsource.h b/src/multimedia/spatial/qspatialaudiosoundsource.h index fe1de852a..818512c68 100644 --- a/src/multimedia/spatial/qspatialaudiosoundsource.h +++ b/src/multimedia/spatial/qspatialaudiosoundsource.h @@ -152,6 +152,7 @@ public Q_SLOTS: private: void setEngine(QSpatialAudioEngine *engine); friend class QSpatialAudioSound; + friend class QSpatialAudioSoundSourcePrivate; QSpatialAudioSoundSourcePrivate *d = nullptr; }; diff --git a/src/multimedia/spatial/qspatialaudiosoundsource_p.h b/src/multimedia/spatial/qspatialaudiosoundsource_p.h index 344fd5038..3986857d0 100644 --- a/src/multimedia/spatial/qspatialaudiosoundsource_p.h +++ b/src/multimedia/spatial/qspatialaudiosoundsource_p.h @@ -70,18 +70,24 @@ public: : QSpatialAudioSound(parent, 1) {} + static QSpatialAudioSoundSourcePrivate *get(QSpatialAudioSoundSource *soundSource) + { return soundSource ? soundSource->d : nullptr; } + QVector3D pos; QQuaternion rotation; QSpatialAudioSoundSource::DistanceModel distanceModel = QSpatialAudioSoundSource::DistanceModel_Logarithmic; - float minDistance = 1.; - float maxDistance = 500.; + float minDistance = .1; + float maxDistance = 50.; float manualAttenuation = 0; float occlusionIntensity = 0.; float directivity = 0.; float directivityOrder = 1.; float nearFieldGain = 0.; + float wallDampening = 1.; + float wallOcclusion = 0.; void updateDistanceModel(); + void updateRoomEffects(); }; QT_END_NAMESPACE -- cgit v1.2.3