Properly handle transmission through room walls

Different wall materials will cause differnt dampening
and occlusions of the sound sources behind. To ensure
smooth transitions, use a switchover at the corners
of the room.

Change-Id: If9a12d1c4fec1c08cb7aba1097ac1562c9c6fb84
Reviewed-by: Piotr Srebrny <piotr.srebrny@qt.io>
Reviewed-by: Lars Knoll <lars.knoll@qt.io>

8 files changed, 215 insertions, 27 deletions

diff --git a/src/multimedia/CMakeLists.txt b/src/multimedia/CMakeLists.txt
index a5bfa2605..4c295072c 100644
--- a/src/multimedia/CMakeLists.txt
+++ b/src/multimedia/CMakeLists.txt
@@ -93,6 +93,7 @@ qt_internal_add_module(Multimedia
 qt_internal_extend_target(Multimedia CONDITION QT_FEATURE_spatialaudio
     SOURCES
         spatial/qambisonicdecoder.cpp spatial/qambisonicdecoder_p.h spatial/qambisonicdecoderdata_p.h
+#        spatial/qacousticgeometry.cpp spatial/qacousticgeometry_p.h
         spatial/qspatialaudioengine.cpp spatial/qspatialaudioengine.h spatial/qspatialaudioengine_p.h
         spatial/qspatialaudiolistener.cpp spatial/qspatialaudiolistener.h
         spatial/qspatialaudioroom.cpp spatial/qspatialaudioroom.h spatial/qspatialaudioroom_p.h
diff --git a/src/multimedia/spatial/qspatialaudioengine.cpp b/src/multimedia/spatial/qspatialaudioengine.cpp
index 6b5a00c9b..6a0b831e8 100644
--- a/src/multimedia/spatial/qspatialaudioengine.cpp
+++ b/src/multimedia/spatial/qspatialaudioengine.cpp
@@ -222,7 +222,7 @@ void QSpatialAudioEnginePrivate::removeRoom(QSpatialAudioRoom *room)
     rooms.removeOne(room);
 }
 
-void QSpatialAudioEnginePrivate::updateRooms() const
+void QSpatialAudioEnginePrivate::updateRooms()
 {
     if (!roomEffectsEnabled)
         return;
@@ -230,9 +230,11 @@ void QSpatialAudioEnginePrivate::updateRooms() const
     bool needUpdate = listenerPositionDirty;
     listenerPositionDirty = false;
 
+    bool roomDirty = false;
     for (const auto &room : rooms) {
         auto *rd = QSpatialAudioRoomPrivate::get(room);
         if (rd->dirty) {
+            roomDirty = true;
             rd->update();
             needUpdate = true;
         }
@@ -241,37 +243,51 @@ void QSpatialAudioEnginePrivate::updateRooms() const
     if (!needUpdate)
         return;
 
-    QVector3D listenerPos;
-    if (listener)
-        listenerPos = listener->position();
+    QVector3D listenerPos = listenerPosition();
     float roomVolume = float(qInf());
-    const QSpatialAudioRoom *room = nullptr;
+    QSpatialAudioRoom *room = nullptr;
     // Find the smallest room that contains the listener and apply it's room effects
-    for (const auto *r : rooms) {
-        QVector3D dimensions = r->dimensions();
-        float vol = dimensions.x()*dimensions.y()*dimensions.z();
+    for (auto *r : qAsConst(rooms)) {
+        QVector3D dim2 = r->dimensions()/2.;
+        float vol = dim2.x()*dim2.y()*dim2.z();
         if (vol > roomVolume)
             continue;
         QVector3D dist = r->position() - listenerPos;
         // transform into room coordinates
         dist = r->rotation().rotatedVector(dist);
-        if (qAbs(dist.x()) <= dimensions.x() &&
-            qAbs(dist.y()) <= dimensions.y() &&
-            qAbs(dist.z()) <= dimensions.z()) {
+        if (qAbs(dist.x()) <= dim2.x() &&
+            qAbs(dist.y()) <= dim2.y() &&
+            qAbs(dist.z()) <= dim2.z()) {
             room = r;
             roomVolume = vol;
         }
     }
+    if (room != currentRoom)
+        roomDirty = true;
+    currentRoom = room;
+
+    if (!roomDirty)
+        return;
 
     // apply room to engine
-    if (room) {
-        QSpatialAudioRoomPrivate *rp = QSpatialAudioRoomPrivate::get(room);
-        api->EnableRoomEffects(true);
-        api->SetReflectionProperties(rp->reflections);
-        api->SetReverbProperties(rp->reverb);
-    } else {
+    if (!currentRoom) {
         api->EnableRoomEffects(false);
+        return;
     }
+    QSpatialAudioRoomPrivate *rp = QSpatialAudioRoomPrivate::get(room);
+    api->SetReflectionProperties(rp->reflections);
+    api->SetReverbProperties(rp->reverb);
+
+    // update room effects for all sound sources
+    for (auto *s : qAsConst(sources)) {
+        auto *sp = QSpatialAudioSoundSourcePrivate::get(s);
+        sp->updateRoomEffects();
+    }
+}
+
+QVector3D QSpatialAudioEnginePrivate::listenerPosition() const
+{
+    return listener ? listener->position() : QVector3D();
 }
 
 
diff --git a/src/multimedia/spatial/qspatialaudioengine_p.h b/src/multimedia/spatial/qspatialaudioengine_p.h
index 4bedf4a1b..b5df68451 100644
--- a/src/multimedia/spatial/qspatialaudioengine_p.h
+++ b/src/multimedia/spatial/qspatialaudioengine_p.h
@@ -56,6 +56,7 @@
 #include <qmutex.h>
 #include <qurl.h>
 #include <qaudiobuffer.h>
+#include <qvector3d.h>
 
 namespace vraudio {
 class ResonanceAudioApi;
@@ -101,6 +102,7 @@ public:
     QList<QSpatialAudioStereoSource *> stereoSources;
     QList<QSpatialAudioRoom *> rooms;
     mutable bool listenerPositionDirty = true;
+    QSpatialAudioRoom *currentRoom = nullptr;
 
     void addSpatialSound(QSpatialAudioSoundSource *sound);
     void removeSpatialSound(QSpatialAudioSoundSource *sound);
@@ -109,7 +111,9 @@ public:
 
     void addRoom(QSpatialAudioRoom *room);
     void removeRoom(QSpatialAudioRoom *room);
-    void updateRooms() const;
+    void updateRooms();
+
+    QVector3D listenerPosition() const;
 };
 
 class QSpatialAudioSound : public QObject
diff --git a/src/multimedia/spatial/qspatialaudioroom.cpp b/src/multimedia/spatial/qspatialaudioroom.cpp
index 2b40028e8..342127466 100644
--- a/src/multimedia/spatial/qspatialaudioroom.cpp
+++ b/src/multimedia/spatial/qspatialaudioroom.cpp
@@ -64,6 +64,53 @@ inline void toFloats(const QQuaternion &q, float *f)
     f[2] = q.z();
     f[3] = q.scalar();
 }
+
+// Default values for occlusion and dampening of different wall materials.
+// These values are used as defaults if a wall is only defined by a material
+// and define how sound passes through the wall.
+// We define both occlusion and dampening constants to be able to tune the
+// sound. Dampening only reduces the level of the sound without affecting its
+// tone, while occlusion will dampen higher frequencies more than lower ones
+struct {
+    float occlusion;
+    float dampening;
+} occlusionAndDampening[] = {
+    { 0.f, 1.f }, // Transparent,
+    { 0.f, .1f }, // AcousticCeilingTiles,
+    { 2.f, .4f }, // BrickBare,
+    { 2.f, .4f }, // BrickPainted,
+    { 4.f, 1.f }, // ConcreteBlockCoarse,
+    { 4.f, 1.f }, // ConcreteBlockPainted,
+    { .7f, .7f }, // CurtainHeavy,
+    { .5f, .5f }, // FiberGlassInsulation,
+    { .2f, .3f }, // GlassThin,
+    { .5f, .2f }, // GlassThick,
+    { 7.f, 1.f }, // Grass,
+    { 4.f, 1.f }, // LinoleumOnConcrete,
+    { 4.f, 1.f }, // Marble,
+    { 0.f, .2f }, // Metal,
+    { 4.f, 1.f }, // ParquetOnConcrete,
+    { 2.f, .4f }, // PlasterRough,
+    { 2.f, .4f }, // PlasterSmooth,
+    { 1.5f, .2f }, // PlywoodPanel,
+    { 4.f, 1.f }, // PolishedConcreteOrTile,
+    { 4.f, 1.f }, // Sheetrock,
+    { 4.f, 1.f }, // WaterOrIceSurface,
+    { 1.f, .3f }, // WoodCeiling,
+    { 1.f, .3f }, // WoodPanel,
+    { 0.f, .0f }, // UniformMaterial,
+};
+
+}
+
+float QSpatialAudioRoomPrivate::wallOcclusion(QSpatialAudioRoom::Wall wall) const
+{
+    return m_wallOcclusion[wall] < 0 ? occlusionAndDampening[roomProperties.material_names[wall]].occlusion : m_wallOcclusion[wall];
+}
+
+float QSpatialAudioRoomPrivate::wallDampening(QSpatialAudioRoom::Wall wall) const
+{
+    return m_wallDampening[wall] < 0 ? occlusionAndDampening[roomProperties.material_names[wall]].dampening : m_wallDampening[wall];
 }
 
 void QSpatialAudioRoomPrivate::update()
@@ -155,10 +202,10 @@ QSpatialAudioRoom::~QSpatialAudioRoom()
 
     \value LeftWall Left wall (negative x)
     \value RightWall Right wall (positive x)
-    \value BackWall Back wall (negative z)
-    \value FrontWall Front wall (positive z)
-    \value Floor Bottom wall (negative y)
-    \value Ceiling Top wall (positive y)
+    \value BackWall Back wall (negative y)
+    \value FrontWall Front wall (positive y)
+    \value Floor Bottom wall (negative z)
+    \value Ceiling Top wall (positive z)
 */
 
 
diff --git a/src/multimedia/spatial/qspatialaudioroom_p.h b/src/multimedia/spatial/qspatialaudioroom_p.h
index d7f78b6bd..668570eef 100644
--- a/src/multimedia/spatial/qspatialaudioroom_p.h
+++ b/src/multimedia/spatial/qspatialaudioroom_p.h
@@ -69,6 +69,12 @@ public:
     vraudio::ReverbProperties reverb;
     vraudio::ReflectionProperties reflections;
 
+    float m_wallOcclusion[6] = { -1.f, -1.f, -1.f, -1.f, -1.f, -1.f };
+    float m_wallDampening[6] = { -1.f, -1.f, -1.f, -1.f, -1.f, -1.f };
+
+    float wallOcclusion(QSpatialAudioRoom::Wall wall) const;
+    float wallDampening(QSpatialAudioRoom::Wall wall) const;
+
     void update();
 };
 
diff --git a/src/multimedia/spatial/qspatialaudiosoundsource.cpp b/src/multimedia/spatial/qspatialaudiosoundsource.cpp
index ac7fa170d..2fd6ecef1 100644
--- a/src/multimedia/spatial/qspatialaudiosoundsource.cpp
+++ b/src/multimedia/spatial/qspatialaudiosoundsource.cpp
@@ -34,9 +34,11 @@
 ** $QT_END_LICENSE$
 **
 ****************************************************************************/
+#include "qspatialaudioroom_p.h"
 #include "qspatialaudiosoundsource_p.h"
 #include "qspatialaudiolistener.h"
 #include "qspatialaudioengine_p.h"
+#include "qspatialaudioroom.h"
 #include "api/resonance_audio_api.h"
 #include <qaudiosink.h>
 #include <qurl.h>
@@ -90,6 +92,7 @@ void QSpatialAudioSoundSource::setPosition(QVector3D pos)
     auto *ep = QSpatialAudioEnginePrivate::get(d->engine);
     if (ep)
         ep->api->SetSourcePosition(d->sourceId, pos.x(), pos.y(), pos.z());
+    d->updateRoomEffects();
     emit positionChanged();
 }
 
@@ -132,7 +135,7 @@ void QSpatialAudioSoundSource::setVolume(float volume)
     d->volume = volume;
     auto *ep = QSpatialAudioEnginePrivate::get(d->engine);
     if (ep)
-        ep->api->SetSourceVolume(d->sourceId, d->volume);
+        ep->api->SetSourceVolume(d->sourceId, d->volume*d->wallDampening);
     emit volumeChanged();
 }
 
@@ -171,9 +174,9 @@ void QSpatialAudioSoundSource::setDistanceModel(DistanceModel model)
 
 void QSpatialAudioSoundSourcePrivate::updateDistanceModel()
 {
-    auto *ep = QSpatialAudioEnginePrivate::get(engine);
     if (!engine || sourceId < 0)
         return;
+    auto *ep = QSpatialAudioEnginePrivate::get(engine);
 
     vraudio::DistanceRolloffModel dm = vraudio::kLogarithmic;
     switch (distanceModel) {
@@ -190,6 +193,110 @@ void QSpatialAudioSoundSourcePrivate::updateDistanceModel()
     ep->api->SetSourceDistanceModel(sourceId, dm, minDistance, maxDistance);
 }
 
+void QSpatialAudioSoundSourcePrivate::updateRoomEffects()
+{
+    if (!engine || sourceId < 0)
+        return;
+    auto *ep = QSpatialAudioEnginePrivate::get(engine);
+    if (!ep->currentRoom)
+        return;
+    auto *rp = QSpatialAudioRoomPrivate::get(ep->currentRoom);
+
+    QVector3D roomDim2 = ep->currentRoom->dimensions()/2.;
+    QVector3D roomPos = ep->currentRoom->position();
+    QQuaternion roomRot = ep->currentRoom->rotation();
+    QVector3D dist = pos - roomPos;
+    // transform into room coordinates
+    dist = roomRot.rotatedVector(dist);
+    if (qAbs(dist.x()) <= roomDim2.x() &&
+        qAbs(dist.y()) <= roomDim2.y() &&
+        qAbs(dist.z()) <= roomDim2.z()) {
+        // Source is inside room, apply
+        ep->api->SetSourceRoomEffectsGain(sourceId, 1);
+        wallDampening = 1.;
+        wallOcclusion = 0.;
+    } else {
+        // ### calculate room occlusion and dampening
+        // This is a bit of heuristics on top of the heuristic dampening/occlusion numbers for walls
+        //
+        // We basically cast a ray from the listener through the walls. If walls have different characteristics
+        // and we get close to a corner, we try to use some averaging to avoid abrupt changes
+        auto relativeListenerPos = ep->listenerPosition() - roomPos;
+        relativeListenerPos = roomRot.rotatedVector(relativeListenerPos);
+
+        auto direction = dist.normalized();
+        enum {
+            X, Y, Z
+        };
+        // Very rough approximation, use the size of the source plus twice the size of our head.
+        // One could probably improve upon this.
+        const float transitionDistance = minDistance + 0.4;
+        QSpatialAudioRoom::Wall walls[3];
+        walls[X] = direction.x() > 0 ? QSpatialAudioRoom::RightWall : QSpatialAudioRoom::LeftWall;
+        walls[Y] = direction.y() > 0 ? QSpatialAudioRoom::FrontWall : QSpatialAudioRoom::BackWall;
+        walls[Z] = direction.z() > 0 ? QSpatialAudioRoom::Ceiling : QSpatialAudioRoom::Floor;
+        float factors[3] = { 0., 0., 0. };
+        bool foundWall = false;
+        if (direction.x() != 0) {
+            float sign = direction.x() > 0 ? 1.f : -1.f;
+            float dx = sign * roomDim2.x() - relativeListenerPos.x();
+            QVector3D intersection = relativeListenerPos + direction*dx/direction.x();
+            float dy = roomDim2.y() - qAbs(intersection.y());
+            float dz = roomDim2.z() - qAbs(intersection.z());
+            if (dy > 0 && dz > 0) {
+//                qDebug() << "Hit with wall X" << walls[0] << dy << dz;
+                // Ray is hitting this wall
+                factors[Y] = qMax(0.f, 1.f/3.f - dy/transitionDistance);
+                factors[Z] = qMax(0.f, 1.f/3.f - dz/transitionDistance);
+                factors[X] = 1.f - factors[Y] - factors[Z];
+                foundWall = true;
+            }
+        }
+        if (!foundWall && direction.y() != 0) {
+            float sign = direction.y() > 0 ? 1.f : -1.f;
+            float dy = sign * roomDim2.y() - relativeListenerPos.y();
+            QVector3D intersection = relativeListenerPos + direction*dy/direction.y();
+            float dx = roomDim2.x() - qAbs(intersection.x());
+            float dz = roomDim2.z() - qAbs(intersection.z());
+            if (dx > 0 && dz > 0) {
+                // Ray is hitting this wall
+//                qDebug() << "Hit with wall Y" << walls[1] << dx << dy;
+                factors[X] = qMax(0.f, 1.f/3.f - dx/transitionDistance);
+                factors[Z] = qMax(0.f, 1.f/3.f - dz/transitionDistance);
+                factors[Y] = 1.f - factors[X] - factors[Z];
+                foundWall = true;
+            }
+        }
+        if (!foundWall) {
+            Q_ASSERT(direction.z() != 0);
+            float sign = direction.z() > 0 ? 1.f : -1.f;
+            float dz = sign * roomDim2.z() - relativeListenerPos.z();
+            QVector3D intersection = relativeListenerPos + direction*dz/direction.z();
+            float dx = roomDim2.x() - qAbs(intersection.x());
+            float dy = roomDim2.y() - qAbs(intersection.y());
+            if (dx > 0 && dy > 0) {
+                // Ray is hitting this wall
+//                qDebug() << "Hit with wall Z" << walls[2];
+                factors[X] = qMax(0.f, 1.f/3.f - dx/transitionDistance);
+                factors[Y] = qMax(0.f, 1.f/3.f - dy/transitionDistance);
+                factors[Z] = 1.f - factors[X] - factors[Y];
+                foundWall = true;
+            }
+        }
+        wallDampening = 0;
+        wallOcclusion = 0;
+        for (int i = 0; i < 3; ++i) {
+            wallDampening += factors[i]*rp->wallDampening(walls[i]);
+            wallOcclusion += factors[i]*rp->wallOcclusion(walls[i]);
+        }
+
+//        qDebug() << "intersection with wall" << walls[0] << walls[1] << walls[2] << factors[0] << factors[1] << factors[2] << wallDampening << wallOcclusion;
+        ep->api->SetSourceRoomEffectsGain(sourceId, 0);
+    }
+    ep->api->SetSoundObjectOcclusionIntensity(sourceId, occlusionIntensity + wallOcclusion);
+    ep->api->SetSourceVolume(sourceId, volume*wallDampening);
+}
+
 QSpatialAudioSoundSource::DistanceModel QSpatialAudioSoundSource::distanceModel() const
 {
     return d->distanceModel;
@@ -283,7 +390,7 @@ void QSpatialAudioSoundSource::setOcclusionIntensity(float occlusion)
     d->occlusionIntensity = occlusion;
     auto *ep = QSpatialAudioEnginePrivate::get(d->engine);
     if (ep)
-        ep->api->SetSoundObjectOcclusionIntensity(d->sourceId, d->occlusionIntensity);
+        ep->api->SetSoundObjectOcclusionIntensity(d->sourceId, d->occlusionIntensity + d->wallOcclusion);
     emit occlusionIntensityChanged();
 }
 
diff --git a/src/multimedia/spatial/qspatialaudiosoundsource.h b/src/multimedia/spatial/qspatialaudiosoundsource.h
index fe1de852a..818512c68 100644
--- a/src/multimedia/spatial/qspatialaudiosoundsource.h
+++ b/src/multimedia/spatial/qspatialaudiosoundsource.h
@@ -152,6 +152,7 @@ public Q_SLOTS:
 private:
     void setEngine(QSpatialAudioEngine *engine);
     friend class QSpatialAudioSound;
+    friend class QSpatialAudioSoundSourcePrivate;
     QSpatialAudioSoundSourcePrivate *d = nullptr;
 };
 
diff --git a/src/multimedia/spatial/qspatialaudiosoundsource_p.h b/src/multimedia/spatial/qspatialaudiosoundsource_p.h
index 344fd5038..3986857d0 100644
--- a/src/multimedia/spatial/qspatialaudiosoundsource_p.h
+++ b/src/multimedia/spatial/qspatialaudiosoundsource_p.h
@@ -70,18 +70,24 @@ public:
         : QSpatialAudioSound(parent, 1)
     {}
 
+    static QSpatialAudioSoundSourcePrivate *get(QSpatialAudioSoundSource *soundSource)
+    { return soundSource ? soundSource->d : nullptr; }
+
     QVector3D pos;
     QQuaternion rotation;
     QSpatialAudioSoundSource::DistanceModel distanceModel = QSpatialAudioSoundSource::DistanceModel_Logarithmic;
-    float minDistance = 1.;
-    float maxDistance = 500.;
+    float minDistance = .1;
+    float maxDistance = 50.;
     float manualAttenuation = 0;
     float occlusionIntensity = 0.;
     float directivity = 0.;
     float directivityOrder = 1.;
     float nearFieldGain = 0.;
+    float wallDampening = 1.;
+    float wallOcclusion = 0.;
 
     void updateDistanceModel();
+    void updateRoomEffects();
 };
 
 QT_END_NAMESPACE