From 47c672cdd67853658a2f86688ec72eb9b4d8c1ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Str=C3=B8mme?= Date: Thu, 10 Nov 2016 17:55:06 +0100 Subject: Add audio probe handling in the player example Adds histogram widgets to visualize the data received from the audio probes, similar to what's done for the video probe already. Change-Id: Ie49a7766dc7ddcab1d9ccaf31372fb23f9ff5b68 Reviewed-by: Yoann Lopes --- .../multimediawidgets/player/histogramwidget.cpp | 182 ++++++++++++++++++++- .../multimediawidgets/player/histogramwidget.h | 5 + examples/multimediawidgets/player/player.cpp | 31 +++- examples/multimediawidgets/player/player.h | 9 +- 4 files changed, 219 insertions(+), 8 deletions(-) (limited to 'examples') diff --git a/examples/multimediawidgets/player/histogramwidget.cpp b/examples/multimediawidgets/player/histogramwidget.cpp index ff8d84e9b..71c243e23 100644 --- a/examples/multimediawidgets/player/histogramwidget.cpp +++ b/examples/multimediawidgets/player/histogramwidget.cpp @@ -40,6 +40,54 @@ #include "histogramwidget.h" #include +#include + +template +static QVector getBufferLevels(const T *buffer, int frames, int channels); + +class QAudioLevel : public QWidget +{ + Q_OBJECT +public: + explicit QAudioLevel(QWidget *parent = 0); + + // Using [0; 1.0] range + void setLevel(qreal level); + +protected: + void paintEvent(QPaintEvent *event); + +private: + qreal m_level; +}; + +QAudioLevel::QAudioLevel(QWidget *parent) + : QWidget(parent) + , m_level(0.0) +{ + setMinimumHeight(15); + setMaximumHeight(50); +} + +void QAudioLevel::setLevel(qreal level) +{ + if (m_level != level) { + m_level = level; + update(); + } +} + +void QAudioLevel::paintEvent(QPaintEvent *event) +{ + Q_UNUSED(event); + + QPainter painter(this); + // draw level + qreal widthLevel = m_level * width(); + painter.fillRect(0, 0, widthLevel, height(), Qt::red); + // clear the rest of the control + painter.fillRect(widthLevel, 0, width(), height(), Qt::black); +} HistogramWidget::HistogramWidget(QWidget *parent) : QWidget(parent) @@ -50,6 +98,7 @@ HistogramWidget::HistogramWidget(QWidget *parent) qRegisterMetaType >("QVector"); connect(&m_processor, SIGNAL(histogramReady(QVector)), SLOT(setHistogram(QVector))); m_processorThread.start(QThread::LowestPriority); + setLayout(new QHBoxLayout); } HistogramWidget::~HistogramWidget() @@ -60,7 +109,7 @@ HistogramWidget::~HistogramWidget() void HistogramWidget::processFrame(QVideoFrame frame) { - if (m_isBusy) + if (m_isBusy && frame.isValid()) return; //drop frame m_isBusy = true; @@ -68,6 +117,132 @@ void HistogramWidget::processFrame(QVideoFrame frame) Qt::QueuedConnection, Q_ARG(QVideoFrame, frame), Q_ARG(int, m_levels)); } +// This function returns the maximum possible sample value for a given audio format +qreal getPeakValue(const QAudioFormat& format) +{ + // Note: Only the most common sample formats are supported + if (!format.isValid()) + return qreal(0); + + if (format.codec() != "audio/pcm") + return qreal(0); + + switch (format.sampleType()) { + case QAudioFormat::Unknown: + break; + case QAudioFormat::Float: + if (format.sampleSize() != 32) // other sample formats are not supported + return qreal(0); + return qreal(1.00003); + case QAudioFormat::SignedInt: + if (format.sampleSize() == 32) + return qreal(INT_MAX); + if (format.sampleSize() == 16) + return qreal(SHRT_MAX); + if (format.sampleSize() == 8) + return qreal(CHAR_MAX); + break; + case QAudioFormat::UnSignedInt: + if (format.sampleSize() == 32) + return qreal(UINT_MAX); + if (format.sampleSize() == 16) + return qreal(USHRT_MAX); + if (format.sampleSize() == 8) + return qreal(UCHAR_MAX); + break; + } + + return qreal(0); +} + +// returns the audio level for each channel +QVector getBufferLevels(const QAudioBuffer& buffer) +{ + QVector values; + + if (!buffer.isValid()) + return values; + + if (!buffer.format().isValid() || buffer.format().byteOrder() != QAudioFormat::LittleEndian) + return values; + + if (buffer.format().codec() != "audio/pcm") + return values; + + int channelCount = buffer.format().channelCount(); + values.fill(0, channelCount); + qreal peak_value = getPeakValue(buffer.format()); + if (qFuzzyCompare(peak_value, qreal(0))) + return values; + + switch (buffer.format().sampleType()) { + case QAudioFormat::Unknown: + case QAudioFormat::UnSignedInt: + if (buffer.format().sampleSize() == 32) + values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); + if (buffer.format().sampleSize() == 16) + values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); + if (buffer.format().sampleSize() == 8) + values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); + for (int i = 0; i < values.size(); ++i) + values[i] = qAbs(values.at(i) - peak_value / 2) / (peak_value / 2); + break; + case QAudioFormat::Float: + if (buffer.format().sampleSize() == 32) { + values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); + for (int i = 0; i < values.size(); ++i) + values[i] /= peak_value; + } + break; + case QAudioFormat::SignedInt: + if (buffer.format().sampleSize() == 32) + values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); + if (buffer.format().sampleSize() == 16) + values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); + if (buffer.format().sampleSize() == 8) + values = getBufferLevels(buffer.constData(), buffer.frameCount(), channelCount); + for (int i = 0; i < values.size(); ++i) + values[i] /= peak_value; + break; + } + + return values; +} + +template +QVector getBufferLevels(const T *buffer, int frames, int channels) +{ + QVector max_values; + max_values.fill(0, channels); + + for (int i = 0; i < frames; ++i) { + for (int j = 0; j < channels; ++j) { + qreal value = qAbs(qreal(buffer[i * channels + j])); + if (value > max_values.at(j)) + max_values.replace(j, value); + } + } + + return max_values; +} + +void HistogramWidget::processBuffer(QAudioBuffer buffer) +{ + if (audioLevels.count() != buffer.format().channelCount()) { + qDeleteAll(audioLevels); + audioLevels.clear(); + for (int i = 0; i < buffer.format().channelCount(); ++i) { + QAudioLevel *level = new QAudioLevel(this); + audioLevels.append(level); + layout()->addWidget(level); + } + } + + QVector levels = getBufferLevels(buffer); + for (int i = 0; i < levels.count(); ++i) + audioLevels.at(i)->setLevel(levels.at(i)); +} + void HistogramWidget::setHistogram(QVector histogram) { m_isBusy = false; @@ -79,6 +254,9 @@ void HistogramWidget::paintEvent(QPaintEvent *event) { Q_UNUSED(event); + if (!audioLevels.isEmpty()) + return; + QPainter painter(this); if (m_histogram.isEmpty()) { @@ -152,3 +330,5 @@ void FrameProcessor::processFrame(QVideoFrame frame, int levels) emit histogramReady(histogram); } + +#include "histogramwidget.moc" diff --git a/examples/multimediawidgets/player/histogramwidget.h b/examples/multimediawidgets/player/histogramwidget.h index 9462b1c84..a85dd27e1 100644 --- a/examples/multimediawidgets/player/histogramwidget.h +++ b/examples/multimediawidgets/player/histogramwidget.h @@ -43,8 +43,11 @@ #include #include +#include #include +class QAudioLevel; + class FrameProcessor: public QObject { Q_OBJECT @@ -67,6 +70,7 @@ public: public slots: void processFrame(QVideoFrame frame); + void processBuffer(QAudioBuffer buffer); void setHistogram(QVector histogram); protected: @@ -78,6 +82,7 @@ private: FrameProcessor m_processor; QThread m_processorThread; bool m_isBusy; + QVector audioLevels; }; #endif // HISTOGRAMWIDGET_H diff --git a/examples/multimediawidgets/player/player.cpp b/examples/multimediawidgets/player/player.cpp index ab048838a..8f291c501 100644 --- a/examples/multimediawidgets/player/player.cpp +++ b/examples/multimediawidgets/player/player.cpp @@ -47,6 +47,7 @@ #include #include #include +#include #include #include @@ -73,6 +74,7 @@ Player::Player(QWidget *parent) connect(player, SIGNAL(bufferStatusChanged(int)), this, SLOT(bufferingProgress(int))); connect(player, SIGNAL(videoAvailableChanged(bool)), this, SLOT(videoAvailableChanged(bool))); connect(player, SIGNAL(error(QMediaPlayer::Error)), this, SLOT(displayErrorMessage())); + connect(player, &QMediaPlayer::stateChanged, this, &Player::stateChanged); //! [2] videoWidget = new VideoWidget(this); @@ -96,14 +98,20 @@ Player::Player(QWidget *parent) labelHistogram = new QLabel(this); labelHistogram->setText("Histogram:"); - histogram = new HistogramWidget(this); + videoHistogram = new HistogramWidget(this); + audioHistogram = new HistogramWidget(this); QHBoxLayout *histogramLayout = new QHBoxLayout; histogramLayout->addWidget(labelHistogram); - histogramLayout->addWidget(histogram, 1); + histogramLayout->addWidget(videoHistogram, 1); + histogramLayout->addWidget(audioHistogram, 2); - probe = new QVideoProbe(this); - connect(probe, SIGNAL(videoFrameProbed(QVideoFrame)), histogram, SLOT(processFrame(QVideoFrame))); - probe->setSource(player); + videoProbe = new QVideoProbe(this); + connect(videoProbe, SIGNAL(videoFrameProbed(QVideoFrame)), videoHistogram, SLOT(processFrame(QVideoFrame))); + videoProbe->setSource(player); + + audioProbe = new QAudioProbe(this); + connect(audioProbe, SIGNAL(audioBufferProbed(QAudioBuffer)), audioHistogram, SLOT(processBuffer(QAudioBuffer))); + audioProbe->setSource(player); QPushButton *openButton = new QPushButton(tr("Open"), this); @@ -269,6 +277,7 @@ void Player::jump(const QModelIndex &index) void Player::playlistPositionChanged(int currentItem) { + clearHistogram(); playlistView->setCurrentIndex(playlistModel->index(currentItem, 0)); } @@ -305,6 +314,12 @@ void Player::statusChanged(QMediaPlayer::MediaStatus status) } } +void Player::stateChanged(QMediaPlayer::State state) +{ + if (state == QMediaPlayer::StoppedState) + clearHistogram(); +} + void Player::handleCursor(QMediaPlayer::MediaStatus status) { #ifndef QT_NO_CURSOR @@ -423,3 +438,9 @@ void Player::showColorDialog() } colorDialog->show(); } + +void Player::clearHistogram() +{ + QMetaObject::invokeMethod(videoHistogram, "processFrame", Qt::QueuedConnection, Q_ARG(QVideoFrame, QVideoFrame())); + QMetaObject::invokeMethod(audioHistogram, "processBuffer", Qt::QueuedConnection, Q_ARG(QAudioBuffer, QAudioBuffer())); +} diff --git a/examples/multimediawidgets/player/player.h b/examples/multimediawidgets/player/player.h index ca643bd7d..ff60f8c63 100644 --- a/examples/multimediawidgets/player/player.h +++ b/examples/multimediawidgets/player/player.h @@ -56,6 +56,7 @@ class QPushButton; class QSlider; class QVideoProbe; class QVideoWidget; +class QAudioProbe; QT_END_NAMESPACE class PlaylistModel; @@ -89,6 +90,7 @@ private slots: void playlistPositionChanged(int); void statusChanged(QMediaPlayer::MediaStatus status); + void stateChanged(QMediaPlayer::State state); void bufferingProgress(int progress); void videoAvailableChanged(bool available); @@ -97,6 +99,7 @@ private slots: void showColorDialog(); private: + void clearHistogram(); void setTrackInfo(const QString &info); void setStatusInfo(const QString &info); void handleCursor(QMediaPlayer::MediaStatus status); @@ -113,8 +116,10 @@ private: QDialog *colorDialog; QLabel *labelHistogram; - HistogramWidget *histogram; - QVideoProbe *probe; + HistogramWidget *videoHistogram; + HistogramWidget *audioHistogram; + QVideoProbe *videoProbe; + QAudioProbe *audioProbe; PlaylistModel *playlistModel; QAbstractItemView *playlistView; -- cgit v1.2.3