wasm: fix audio controls on video player

When a QMediaPlayer is playing a video with audio output
specified, the underlying video element volume needs to be set
instead of the audio (element) volume.

Also fix audio/video selection when file as detected by Qt, is
application/octet-stream.

Fixes: QTBUG-120671
Pick-to: 6.7
Change-Id: I92b9677e5b0c962665974b0da7ca03b2348d82b0
Reviewed-by: Jøger Hansegård <joger.hansegard@qt.io>
Reviewed-by: Morten Johan Sørvig <morten.sorvig@qt.io>

5 files changed, 43 insertions, 11 deletions

diff --git a/src/plugins/multimedia/wasm/common/qwasmaudiooutput.cpp b/src/plugins/multimedia/wasm/common/qwasmaudiooutput.cpp
index c170b91b9..a9a644140 100644
--- a/src/plugins/multimedia/wasm/common/qwasmaudiooutput.cpp
+++ b/src/plugins/multimedia/wasm/common/qwasmaudiooutput.cpp
@@ -30,8 +30,23 @@ void QWasmAudioOutput::setAudioDevice(const QAudioDevice &audioDevice)
     device = audioDevice;
 }
 
+void QWasmAudioOutput::setVideoElement(emscripten::val videoElement)
+{
+    m_videoElement = videoElement;
+}
+
+emscripten::val QWasmAudioOutput::videoElement()
+{
+    return m_videoElement;
+}
+
 void QWasmAudioOutput::setMuted(bool muted)
 {
+    emscripten::val realElement = videoElement();
+    if (!realElement.isUndefined()) {
+        realElement.set("muted", muted);
+        return;
+    }
     if (m_audio.isUndefined() || m_audio.isNull()) {
         qCDebug(qWasmMediaAudioOutput) << "Error"
                                        << "Audio element could not be created";
@@ -44,14 +59,20 @@ void QWasmAudioOutput::setMuted(bool muted)
 
 void QWasmAudioOutput::setVolume(float volume)
 {
+    volume = qBound(qreal(0.0), volume, qreal(1.0));
+    emscripten::val realElement = videoElement();
+    if (!realElement.isUndefined()) {
+        realElement.set("volume", volume);
+        return;
+    }
     if (m_audio.isUndefined() || m_audio.isNull()) {
         qCDebug(qWasmMediaAudioOutput) << "Error"
-                                       << "Audio element could not be created";
+                                       << "Audio element not available";
         emit errorOccured(QMediaPlayer::ResourceError,
                           QStringLiteral("Media file could not be opened"));
         return;
     }
-    volume = qBound(qreal(0.0), volume, qreal(1.0));
+
     m_audio.set("volume", volume);
 }
 
diff --git a/src/plugins/multimedia/wasm/common/qwasmaudiooutput_p.h b/src/plugins/multimedia/wasm/common/qwasmaudiooutput_p.h
index db121783d..69fda120b 100644
--- a/src/plugins/multimedia/wasm/common/qwasmaudiooutput_p.h
+++ b/src/plugins/multimedia/wasm/common/qwasmaudiooutput_p.h
@@ -51,6 +51,7 @@ public:
 
     void setSource(const QUrl &url);
     void setSource(QIODevice *stream);
+    void setVideoElement(emscripten::val videoElement);
 
 Q_SIGNALS:
     void readyChanged(bool);
@@ -67,6 +68,8 @@ private:
     void doElementCallbacks();
     void createAudioElement(const std::string &id);
 
+    emscripten::val videoElement();
+
     QScopedPointer<QWasmAudioSink> m_sink;
     QScopedPointer<qstdweb::EventCallback> m_playEvent;
     QScopedPointer<qstdweb::EventCallback> m_endedEvent;
@@ -84,6 +87,7 @@ private:
     QString m_source;
     QIODevice *m_audioIODevice = nullptr;
     emscripten::val m_audio = emscripten::val::undefined();
+    emscripten::val m_videoElement = emscripten::val::undefined();
     QMediaPlayer::MediaStatus m_currentMediaStatus;
     qreal m_currentBufferedValue;
 };
diff --git a/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp b/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp
index 026230f10..452ea28f8 100644
--- a/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp
+++ b/src/plugins/multimedia/wasm/common/qwasmvideooutput.cpp
@@ -661,7 +661,6 @@ void QWasmVideoOutput::doElementCallbacks()
     // emptied
     auto emptiedCallback = [=](emscripten::val event) {
         Q_UNUSED(event)
-        checkNetworkState();
         qCDebug(qWasmMediaVideoOutput) << "emptied";
         emit readyChanged(false);
         m_currentMediaStatus = QMediaPlayer::EndOfMedia;
diff --git a/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h b/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h
index 5c170f29c..4864df989 100644
--- a/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h
+++ b/src/plugins/multimedia/wasm/common/qwasmvideooutput_p.h
@@ -87,6 +87,8 @@ public:
     // mediacapturesession has the videosink
     QVideoSink *m_wasmSink = nullptr;
 
+    emscripten::val currentVideoElement() { return m_video; }
+
 Q_SIGNALS:
     void readyChanged(bool);
     void bufferingChanged(qint32 percent);
diff --git a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp
index 957a6c0e3..7ace6e9a2 100644
--- a/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp
+++ b/src/plugins/multimedia/wasm/mediaplayer/qwasmmediaplayer.cpp
@@ -209,12 +209,13 @@ void QWasmMediaPlayer::setMedia(const QUrl &mediaContent, QIODevice *stream)
     if (mediaContent.isEmpty()) {
         if (stream) {
             m_mediaStream = stream;
-            if (db.mimeTypeForData(stream).name().contains("video")) {
-                setVideoAvailable(true);
-                m_videoOutput->setSource(m_mediaStream);
-            }  else {
+            qCDebug(lcMediaPlayer) << db.mimeTypeForData(stream).name();
+            if (db.mimeTypeForData(stream).name().contains("audio")) {
                 setAudioAvailable(true);
                 m_audioOutput->setSource(m_mediaStream);
+            }  else { // treat octet-stream as video
+                setVideoAvailable(true);
+                m_videoOutput->setSource(m_mediaStream);
             }
         } else {
 
@@ -222,12 +223,13 @@ void QWasmMediaPlayer::setMedia(const QUrl &mediaContent, QIODevice *stream)
         }
     } else {
         QString sourceFile = mediaContent.toLocalFile();
-        if (db.mimeTypeForFile(QFileInfo(sourceFile)).name().contains("video")) {
-                setVideoAvailable(true);
-            m_videoOutput->setSource(mediaContent);
-        } else {
+        qCDebug(lcMediaPlayer) << db.mimeTypeForFile(QFileInfo(sourceFile)).name();
+        if (db.mimeTypeForFile(QFileInfo(sourceFile)).name().contains("audio")) {
             setAudioAvailable(true);
             m_audioOutput->setSource(mediaContent);
+        } else { // treat octet-stream as video
+            setVideoAvailable(true);
+            m_videoOutput->setSource(mediaContent);
         }
     }
 
@@ -246,6 +248,9 @@ void QWasmMediaPlayer::setVideoSink(QVideoSink *sink)
 
     initVideo();
     m_videoOutput->setSurface(sink);
+    setVideoAvailable(true);
+    if (isAudioAvailable() && m_audioOutput)
+        m_audioOutput->setVideoElement(m_videoOutput->currentVideoElement());
 }
 
 void QWasmMediaPlayer::setAudioOutput(QPlatformAudioOutput *output)
@@ -256,6 +261,7 @@ void QWasmMediaPlayer::setAudioOutput(QPlatformAudioOutput *output)
     if (m_audioOutput)
         m_audioOutput->q->disconnect(this);
     m_audioOutput = static_cast<QWasmAudioOutput *>(output);
+    setAudioAvailable(true);
 }
 
 void QWasmMediaPlayer::updateAudioDevice()