Add audio probe handling in the player example

Adds histogram widgets to visualize the data received from the audio
probes, similar to what's done for the video probe already.

Change-Id: Ie49a7766dc7ddcab1d9ccaf31372fb23f9ff5b68
Reviewed-by: Yoann Lopes <yoann.lopes@qt.io>

4 files changed, 219 insertions, 8 deletions

diff --git a/examples/multimediawidgets/player/histogramwidget.cpp b/examples/multimediawidgets/player/histogramwidget.cpp
index ff8d84e9b..71c243e23 100644
--- a/examples/multimediawidgets/player/histogramwidget.cpp
+++ b/examples/multimediawidgets/player/histogramwidget.cpp
@@ -40,6 +40,54 @@
 
 #include "histogramwidget.h"
 #include <QPainter>
+#include <QHBoxLayout>
+
+template <class T>
+static QVector<qreal> getBufferLevels(const T *buffer, int frames, int channels);
+
+class QAudioLevel : public QWidget
+{
+    Q_OBJECT
+public:
+    explicit QAudioLevel(QWidget *parent = 0);
+
+    // Using [0; 1.0] range
+    void setLevel(qreal level);
+
+protected:
+    void paintEvent(QPaintEvent *event);
+
+private:
+    qreal m_level;
+};
+
+QAudioLevel::QAudioLevel(QWidget *parent)
+  : QWidget(parent)
+  , m_level(0.0)
+{
+    setMinimumHeight(15);
+    setMaximumHeight(50);
+}
+
+void QAudioLevel::setLevel(qreal level)
+{
+    if (m_level != level) {
+        m_level = level;
+        update();
+    }
+}
+
+void QAudioLevel::paintEvent(QPaintEvent *event)
+{
+    Q_UNUSED(event);
+
+    QPainter painter(this);
+    // draw level
+    qreal widthLevel = m_level * width();
+    painter.fillRect(0, 0, widthLevel, height(), Qt::red);
+    // clear the rest of the control
+    painter.fillRect(widthLevel, 0, width(), height(), Qt::black);
+}
 
 HistogramWidget::HistogramWidget(QWidget *parent)
     : QWidget(parent)
@@ -50,6 +98,7 @@ HistogramWidget::HistogramWidget(QWidget *parent)
     qRegisterMetaType<QVector<qreal> >("QVector<qreal>");
     connect(&m_processor, SIGNAL(histogramReady(QVector<qreal>)), SLOT(setHistogram(QVector<qreal>)));
     m_processorThread.start(QThread::LowestPriority);
+    setLayout(new QHBoxLayout);
 }
 
 HistogramWidget::~HistogramWidget()
@@ -60,7 +109,7 @@ HistogramWidget::~HistogramWidget()
 
 void HistogramWidget::processFrame(QVideoFrame frame)
 {
-    if (m_isBusy)
+    if (m_isBusy && frame.isValid())
         return; //drop frame
 
     m_isBusy = true;
@@ -68,6 +117,132 @@ void HistogramWidget::processFrame(QVideoFrame frame)
                               Qt::QueuedConnection, Q_ARG(QVideoFrame, frame), Q_ARG(int, m_levels));
 }
 
+// This function returns the maximum possible sample value for a given audio format
+qreal getPeakValue(const QAudioFormat& format)
+{
+    // Note: Only the most common sample formats are supported
+    if (!format.isValid())
+        return qreal(0);
+
+    if (format.codec() != "audio/pcm")
+        return qreal(0);
+
+    switch (format.sampleType()) {
+    case QAudioFormat::Unknown:
+        break;
+    case QAudioFormat::Float:
+        if (format.sampleSize() != 32) // other sample formats are not supported
+            return qreal(0);
+        return qreal(1.00003);
+    case QAudioFormat::SignedInt:
+        if (format.sampleSize() == 32)
+            return qreal(INT_MAX);
+        if (format.sampleSize() == 16)
+            return qreal(SHRT_MAX);
+        if (format.sampleSize() == 8)
+            return qreal(CHAR_MAX);
+        break;
+    case QAudioFormat::UnSignedInt:
+        if (format.sampleSize() == 32)
+            return qreal(UINT_MAX);
+        if (format.sampleSize() == 16)
+            return qreal(USHRT_MAX);
+        if (format.sampleSize() == 8)
+            return qreal(UCHAR_MAX);
+        break;
+    }
+
+    return qreal(0);
+}
+
+// returns the audio level for each channel
+QVector<qreal> getBufferLevels(const QAudioBuffer& buffer)
+{
+    QVector<qreal> values;
+
+    if (!buffer.isValid())
+        return values;
+
+    if (!buffer.format().isValid() || buffer.format().byteOrder() != QAudioFormat::LittleEndian)
+        return values;
+
+    if (buffer.format().codec() != "audio/pcm")
+        return values;
+
+    int channelCount = buffer.format().channelCount();
+    values.fill(0, channelCount);
+    qreal peak_value = getPeakValue(buffer.format());
+    if (qFuzzyCompare(peak_value, qreal(0)))
+        return values;
+
+    switch (buffer.format().sampleType()) {
+    case QAudioFormat::Unknown:
+    case QAudioFormat::UnSignedInt:
+        if (buffer.format().sampleSize() == 32)
+            values = getBufferLevels(buffer.constData<quint32>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 16)
+            values = getBufferLevels(buffer.constData<quint16>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 8)
+            values = getBufferLevels(buffer.constData<quint8>(), buffer.frameCount(), channelCount);
+        for (int i = 0; i < values.size(); ++i)
+            values[i] = qAbs(values.at(i) - peak_value / 2) / (peak_value / 2);
+        break;
+    case QAudioFormat::Float:
+        if (buffer.format().sampleSize() == 32) {
+            values = getBufferLevels(buffer.constData<float>(), buffer.frameCount(), channelCount);
+            for (int i = 0; i < values.size(); ++i)
+                values[i] /= peak_value;
+        }
+        break;
+    case QAudioFormat::SignedInt:
+        if (buffer.format().sampleSize() == 32)
+            values = getBufferLevels(buffer.constData<qint32>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 16)
+            values = getBufferLevels(buffer.constData<qint16>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 8)
+            values = getBufferLevels(buffer.constData<qint8>(), buffer.frameCount(), channelCount);
+        for (int i = 0; i < values.size(); ++i)
+            values[i] /= peak_value;
+        break;
+    }
+
+    return values;
+}
+
+template <class T>
+QVector<qreal> getBufferLevels(const T *buffer, int frames, int channels)
+{
+    QVector<qreal> max_values;
+    max_values.fill(0, channels);
+
+    for (int i = 0; i < frames; ++i) {
+        for (int j = 0; j < channels; ++j) {
+            qreal value = qAbs(qreal(buffer[i * channels + j]));
+            if (value > max_values.at(j))
+                max_values.replace(j, value);
+        }
+    }
+
+    return max_values;
+}
+
+void HistogramWidget::processBuffer(QAudioBuffer buffer)
+{
+    if (audioLevels.count() != buffer.format().channelCount()) {
+        qDeleteAll(audioLevels);
+        audioLevels.clear();
+        for (int i = 0; i < buffer.format().channelCount(); ++i) {
+            QAudioLevel *level = new QAudioLevel(this);
+            audioLevels.append(level);
+            layout()->addWidget(level);
+        }
+    }
+
+    QVector<qreal> levels = getBufferLevels(buffer);
+    for (int i = 0; i < levels.count(); ++i)
+        audioLevels.at(i)->setLevel(levels.at(i));
+}
+
 void HistogramWidget::setHistogram(QVector<qreal> histogram)
 {
     m_isBusy = false;
@@ -79,6 +254,9 @@ void HistogramWidget::paintEvent(QPaintEvent *event)
 {
     Q_UNUSED(event);
 
+    if (!audioLevels.isEmpty())
+        return;
+
     QPainter painter(this);
 
     if (m_histogram.isEmpty()) {
@@ -152,3 +330,5 @@ void FrameProcessor::processFrame(QVideoFrame frame, int levels)
 
     emit histogramReady(histogram);
 }
+
+#include "histogramwidget.moc"
diff --git a/examples/multimediawidgets/player/histogramwidget.h b/examples/multimediawidgets/player/histogramwidget.h
index 9462b1c84..a85dd27e1 100644
--- a/examples/multimediawidgets/player/histogramwidget.h
+++ b/examples/multimediawidgets/player/histogramwidget.h
@@ -43,8 +43,11 @@
 
 #include <QThread>
 #include <QVideoFrame>
+#include <QAudioBuffer>
 #include <QWidget>
 
+class QAudioLevel;
+
 class FrameProcessor: public QObject
 {
     Q_OBJECT
@@ -67,6 +70,7 @@ public:
 
 public slots:
     void processFrame(QVideoFrame frame);
+    void processBuffer(QAudioBuffer buffer);
     void setHistogram(QVector<qreal> histogram);
 
 protected:
@@ -78,6 +82,7 @@ private:
     FrameProcessor m_processor;
     QThread m_processorThread;
     bool m_isBusy;
+    QVector<QAudioLevel *> audioLevels;
 };
 
 #endif // HISTOGRAMWIDGET_H
diff --git a/examples/multimediawidgets/player/player.cpp b/examples/multimediawidgets/player/player.cpp
index ab048838a..8f291c501 100644
--- a/examples/multimediawidgets/player/player.cpp
+++ b/examples/multimediawidgets/player/player.cpp
@@ -47,6 +47,7 @@
 #include <QMediaService>
 #include <QMediaPlaylist>
 #include <QVideoProbe>
+#include <QAudioProbe>
 #include <QMediaMetaData>
 #include <QtWidgets>
 
@@ -73,6 +74,7 @@ Player::Player(QWidget *parent)
     connect(player, SIGNAL(bufferStatusChanged(int)), this, SLOT(bufferingProgress(int)));
     connect(player, SIGNAL(videoAvailableChanged(bool)), this, SLOT(videoAvailableChanged(bool)));
     connect(player, SIGNAL(error(QMediaPlayer::Error)), this, SLOT(displayErrorMessage()));
+    connect(player, &QMediaPlayer::stateChanged, this, &Player::stateChanged);
 
 //! [2]
     videoWidget = new VideoWidget(this);
@@ -96,14 +98,20 @@ Player::Player(QWidget *parent)
 
     labelHistogram = new QLabel(this);
     labelHistogram->setText("Histogram:");
-    histogram = new HistogramWidget(this);
+    videoHistogram = new HistogramWidget(this);
+    audioHistogram = new HistogramWidget(this);
     QHBoxLayout *histogramLayout = new QHBoxLayout;
     histogramLayout->addWidget(labelHistogram);
-    histogramLayout->addWidget(histogram, 1);
+    histogramLayout->addWidget(videoHistogram, 1);
+    histogramLayout->addWidget(audioHistogram, 2);
 
-    probe = new QVideoProbe(this);
-    connect(probe, SIGNAL(videoFrameProbed(QVideoFrame)), histogram, SLOT(processFrame(QVideoFrame)));
-    probe->setSource(player);
+    videoProbe = new QVideoProbe(this);
+    connect(videoProbe, SIGNAL(videoFrameProbed(QVideoFrame)), videoHistogram, SLOT(processFrame(QVideoFrame)));
+    videoProbe->setSource(player);
+
+    audioProbe = new QAudioProbe(this);
+    connect(audioProbe, SIGNAL(audioBufferProbed(QAudioBuffer)), audioHistogram, SLOT(processBuffer(QAudioBuffer)));
+    audioProbe->setSource(player);
 
     QPushButton *openButton = new QPushButton(tr("Open"), this);
 
@@ -269,6 +277,7 @@ void Player::jump(const QModelIndex &index)
 
 void Player::playlistPositionChanged(int currentItem)
 {
+    clearHistogram();
     playlistView->setCurrentIndex(playlistModel->index(currentItem, 0));
 }
 
@@ -305,6 +314,12 @@ void Player::statusChanged(QMediaPlayer::MediaStatus status)
     }
 }
 
+void Player::stateChanged(QMediaPlayer::State state)
+{
+    if (state == QMediaPlayer::StoppedState)
+        clearHistogram();
+}
+
 void Player::handleCursor(QMediaPlayer::MediaStatus status)
 {
 #ifndef QT_NO_CURSOR
@@ -423,3 +438,9 @@ void Player::showColorDialog()
     }
     colorDialog->show();
 }
+
+void Player::clearHistogram()
+{
+    QMetaObject::invokeMethod(videoHistogram, "processFrame", Qt::QueuedConnection, Q_ARG(QVideoFrame, QVideoFrame()));
+    QMetaObject::invokeMethod(audioHistogram, "processBuffer", Qt::QueuedConnection, Q_ARG(QAudioBuffer, QAudioBuffer()));
+}
diff --git a/examples/multimediawidgets/player/player.h b/examples/multimediawidgets/player/player.h
index ca643bd7d..ff60f8c63 100644
--- a/examples/multimediawidgets/player/player.h
+++ b/examples/multimediawidgets/player/player.h
@@ -56,6 +56,7 @@ class QPushButton;
 class QSlider;
 class QVideoProbe;
 class QVideoWidget;
+class QAudioProbe;
 QT_END_NAMESPACE
 
 class PlaylistModel;
@@ -89,6 +90,7 @@ private slots:
     void playlistPositionChanged(int);
 
     void statusChanged(QMediaPlayer::MediaStatus status);
+    void stateChanged(QMediaPlayer::State state);
     void bufferingProgress(int progress);
     void videoAvailableChanged(bool available);
 
@@ -97,6 +99,7 @@ private slots:
     void showColorDialog();
 
 private:
+    void clearHistogram();
     void setTrackInfo(const QString &info);
     void setStatusInfo(const QString &info);
     void handleCursor(QMediaPlayer::MediaStatus status);
@@ -113,8 +116,10 @@ private:
     QDialog *colorDialog;
 
     QLabel *labelHistogram;
-    HistogramWidget *histogram;
-    QVideoProbe *probe;
+    HistogramWidget *videoHistogram;
+    HistogramWidget *audioHistogram;
+    QVideoProbe *videoProbe;
+    QAudioProbe *audioProbe;
 
     PlaylistModel *playlistModel;
     QAbstractItemView *playlistView;