Add audio probe handling in the player example

Adds histogram widgets to visualize the data received from the audio
probes, similar to what's done for the video probe already.

Change-Id: Ie49a7766dc7ddcab1d9ccaf31372fb23f9ff5b68
Reviewed-by: Yoann Lopes <yoann.lopes@qt.io>

1 files changed, 181 insertions, 1 deletions

diff --git a/examples/multimediawidgets/player/histogramwidget.cpp b/examples/multimediawidgets/player/histogramwidget.cpp
index ff8d84e9b..71c243e23 100644
--- a/examples/multimediawidgets/player/histogramwidget.cpp
+++ b/examples/multimediawidgets/player/histogramwidget.cpp
@@ -40,6 +40,54 @@
 
 #include "histogramwidget.h"
 #include <QPainter>
+#include <QHBoxLayout>
+
+template <class T>
+static QVector<qreal> getBufferLevels(const T *buffer, int frames, int channels);
+
+class QAudioLevel : public QWidget
+{
+    Q_OBJECT
+public:
+    explicit QAudioLevel(QWidget *parent = 0);
+
+    // Using [0; 1.0] range
+    void setLevel(qreal level);
+
+protected:
+    void paintEvent(QPaintEvent *event);
+
+private:
+    qreal m_level;
+};
+
+QAudioLevel::QAudioLevel(QWidget *parent)
+  : QWidget(parent)
+  , m_level(0.0)
+{
+    setMinimumHeight(15);
+    setMaximumHeight(50);
+}
+
+void QAudioLevel::setLevel(qreal level)
+{
+    if (m_level != level) {
+        m_level = level;
+        update();
+    }
+}
+
+void QAudioLevel::paintEvent(QPaintEvent *event)
+{
+    Q_UNUSED(event);
+
+    QPainter painter(this);
+    // draw level
+    qreal widthLevel = m_level * width();
+    painter.fillRect(0, 0, widthLevel, height(), Qt::red);
+    // clear the rest of the control
+    painter.fillRect(widthLevel, 0, width(), height(), Qt::black);
+}
 
 HistogramWidget::HistogramWidget(QWidget *parent)
     : QWidget(parent)
@@ -50,6 +98,7 @@ HistogramWidget::HistogramWidget(QWidget *parent)
     qRegisterMetaType<QVector<qreal> >("QVector<qreal>");
     connect(&m_processor, SIGNAL(histogramReady(QVector<qreal>)), SLOT(setHistogram(QVector<qreal>)));
     m_processorThread.start(QThread::LowestPriority);
+    setLayout(new QHBoxLayout);
 }
 
 HistogramWidget::~HistogramWidget()
@@ -60,7 +109,7 @@ HistogramWidget::~HistogramWidget()
 
 void HistogramWidget::processFrame(QVideoFrame frame)
 {
-    if (m_isBusy)
+    if (m_isBusy && frame.isValid())
         return; //drop frame
 
     m_isBusy = true;
@@ -68,6 +117,132 @@ void HistogramWidget::processFrame(QVideoFrame frame)
                               Qt::QueuedConnection, Q_ARG(QVideoFrame, frame), Q_ARG(int, m_levels));
 }
 
+// This function returns the maximum possible sample value for a given audio format
+qreal getPeakValue(const QAudioFormat& format)
+{
+    // Note: Only the most common sample formats are supported
+    if (!format.isValid())
+        return qreal(0);
+
+    if (format.codec() != "audio/pcm")
+        return qreal(0);
+
+    switch (format.sampleType()) {
+    case QAudioFormat::Unknown:
+        break;
+    case QAudioFormat::Float:
+        if (format.sampleSize() != 32) // other sample formats are not supported
+            return qreal(0);
+        return qreal(1.00003);
+    case QAudioFormat::SignedInt:
+        if (format.sampleSize() == 32)
+            return qreal(INT_MAX);
+        if (format.sampleSize() == 16)
+            return qreal(SHRT_MAX);
+        if (format.sampleSize() == 8)
+            return qreal(CHAR_MAX);
+        break;
+    case QAudioFormat::UnSignedInt:
+        if (format.sampleSize() == 32)
+            return qreal(UINT_MAX);
+        if (format.sampleSize() == 16)
+            return qreal(USHRT_MAX);
+        if (format.sampleSize() == 8)
+            return qreal(UCHAR_MAX);
+        break;
+    }
+
+    return qreal(0);
+}
+
+// returns the audio level for each channel
+QVector<qreal> getBufferLevels(const QAudioBuffer& buffer)
+{
+    QVector<qreal> values;
+
+    if (!buffer.isValid())
+        return values;
+
+    if (!buffer.format().isValid() || buffer.format().byteOrder() != QAudioFormat::LittleEndian)
+        return values;
+
+    if (buffer.format().codec() != "audio/pcm")
+        return values;
+
+    int channelCount = buffer.format().channelCount();
+    values.fill(0, channelCount);
+    qreal peak_value = getPeakValue(buffer.format());
+    if (qFuzzyCompare(peak_value, qreal(0)))
+        return values;
+
+    switch (buffer.format().sampleType()) {
+    case QAudioFormat::Unknown:
+    case QAudioFormat::UnSignedInt:
+        if (buffer.format().sampleSize() == 32)
+            values = getBufferLevels(buffer.constData<quint32>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 16)
+            values = getBufferLevels(buffer.constData<quint16>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 8)
+            values = getBufferLevels(buffer.constData<quint8>(), buffer.frameCount(), channelCount);
+        for (int i = 0; i < values.size(); ++i)
+            values[i] = qAbs(values.at(i) - peak_value / 2) / (peak_value / 2);
+        break;
+    case QAudioFormat::Float:
+        if (buffer.format().sampleSize() == 32) {
+            values = getBufferLevels(buffer.constData<float>(), buffer.frameCount(), channelCount);
+            for (int i = 0; i < values.size(); ++i)
+                values[i] /= peak_value;
+        }
+        break;
+    case QAudioFormat::SignedInt:
+        if (buffer.format().sampleSize() == 32)
+            values = getBufferLevels(buffer.constData<qint32>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 16)
+            values = getBufferLevels(buffer.constData<qint16>(), buffer.frameCount(), channelCount);
+        if (buffer.format().sampleSize() == 8)
+            values = getBufferLevels(buffer.constData<qint8>(), buffer.frameCount(), channelCount);
+        for (int i = 0; i < values.size(); ++i)
+            values[i] /= peak_value;
+        break;
+    }
+
+    return values;
+}
+
+template <class T>
+QVector<qreal> getBufferLevels(const T *buffer, int frames, int channels)
+{
+    QVector<qreal> max_values;
+    max_values.fill(0, channels);
+
+    for (int i = 0; i < frames; ++i) {
+        for (int j = 0; j < channels; ++j) {
+            qreal value = qAbs(qreal(buffer[i * channels + j]));
+            if (value > max_values.at(j))
+                max_values.replace(j, value);
+        }
+    }
+
+    return max_values;
+}
+
+void HistogramWidget::processBuffer(QAudioBuffer buffer)
+{
+    if (audioLevels.count() != buffer.format().channelCount()) {
+        qDeleteAll(audioLevels);
+        audioLevels.clear();
+        for (int i = 0; i < buffer.format().channelCount(); ++i) {
+            QAudioLevel *level = new QAudioLevel(this);
+            audioLevels.append(level);
+            layout()->addWidget(level);
+        }
+    }
+
+    QVector<qreal> levels = getBufferLevels(buffer);
+    for (int i = 0; i < levels.count(); ++i)
+        audioLevels.at(i)->setLevel(levels.at(i));
+}
+
 void HistogramWidget::setHistogram(QVector<qreal> histogram)
 {
     m_isBusy = false;
@@ -79,6 +254,9 @@ void HistogramWidget::paintEvent(QPaintEvent *event)
 {
     Q_UNUSED(event);
 
+    if (!audioLevels.isEmpty())
+        return;
+
     QPainter painter(this);
 
     if (m_histogram.isEmpty()) {
@@ -152,3 +330,5 @@ void FrameProcessor::processFrame(QVideoFrame frame, int levels)
 
     emit histogramReady(histogram);
 }
+
+#include "histogramwidget.moc"