diff options
| -rw-r--r-- | plugins/alexainterface/AlexaInterface.cpp | 8 | ||||
| -rw-r--r-- | plugins/alexainterface/AlexaInterface.h | 3 | ||||
| -rw-r--r-- | plugins/alexainterface/QtMicrophoneWrapper.cpp | 220 | ||||
| -rw-r--r-- | plugins/alexainterface/QtMicrophoneWrapper.h | 53 |
4 files changed, 275 insertions, 9 deletions
diff --git a/plugins/alexainterface/AlexaInterface.cpp b/plugins/alexainterface/AlexaInterface.cpp index 6577177..ab4c95a 100644 --- a/plugins/alexainterface/AlexaInterface.cpp +++ b/plugins/alexainterface/AlexaInterface.cpp @@ -910,13 +910,19 @@ bool AlexaInterface::initialize( QSettings settings(QStringLiteral("Luxoft Sweden AB"), QStringLiteral("AlexaApp")); QString captureDeviceName = settings.value(QStringLiteral("capture/device_name"), QStringLiteral("default")).toString(); - std::shared_ptr<QtMicrophoneWrapper> m_micWrapper = QtMicrophoneWrapper::create(sharedDataStream, + m_micWrapper = QtMicrophoneWrapper::create(sharedDataStream, captureDeviceName); if (!m_micWrapper) { ACSDK_CRITICAL(LX("Failed to create QtMicrophoneWrapper!")); return false; } + // Turn on/off audio level processing, might be CPU consuming + bool processAudioLevel = settings.value(QStringLiteral("capture/process_input"), true).toBool(); + m_micWrapper->setLevelProcess(processAudioLevel); + QObject::connect(m_micWrapper.get(), &QtMicrophoneWrapper::audioLevelChanged, this, &AlexaInterface::audioLevelChanged); + + // Creating wake word audio provider, if necessary #ifdef KWD bool wakeAlwaysReadable = true; diff --git a/plugins/alexainterface/AlexaInterface.h b/plugins/alexainterface/AlexaInterface.h index bcfa56e..492d8df 100644 --- a/plugins/alexainterface/AlexaInterface.h +++ b/plugins/alexainterface/AlexaInterface.h @@ -87,6 +87,7 @@ class AlexaInterface: public QObject { Q_PROPERTY(QString authCode READ authCode NOTIFY authCodeChanged) Q_PROPERTY(ConnectionManager::ConnectionStatus connectionStatus READ connectionStatus NOTIFY connectionStatusChanged) Q_PROPERTY(LogLevel logLevel READ logLevel WRITE setLogLevel NOTIFY logLevelChanged) + Q_PROPERTY(qreal audioLevel READ audioLevel NOTIFY audioLevelChanged) public: @@ -117,6 +118,7 @@ public: QString authCode() const { return m_authCode; } ConnectionManager::ConnectionStatus connectionStatus() const { return m_connectionStatus; } LogLevel logLevel() const { return m_logLevel; } + qreal audioLevel() const { return m_micWrapper ? m_micWrapper->audioLevel() : 0.0; } explicit AlexaInterface(QObject* parent = nullptr); /// Destructor which manages the @c AlexaInterface shutdown sequence. @@ -213,6 +215,7 @@ Q_SIGNALS: void connectionStatusChanged(); void logLevelChanged(); void cardReady(BaseCard *card); + void audioLevelChanged(); private: static std::unique_ptr<AlexaInterface> instance; diff --git a/plugins/alexainterface/QtMicrophoneWrapper.cpp b/plugins/alexainterface/QtMicrophoneWrapper.cpp index c3fe0ef..86bfa6f 100644 --- a/plugins/alexainterface/QtMicrophoneWrapper.cpp +++ b/plugins/alexainterface/QtMicrophoneWrapper.cpp @@ -30,7 +30,7 @@ ****************************************************************************/ #include "QtMicrophoneWrapper.h" - +#include <QtEndian> #include <QDebug> using alexaClientSDK::avsCommon::avs::AudioInputStream; @@ -121,7 +121,7 @@ void QtMicrophoneWrapper::setAudioDevice(const QString &deviceName) { format.setByteOrder(QAudioFormat::LittleEndian); format.setSampleType(QAudioFormat::SignedInt); - m_audioInfo = QAudioDeviceInfo::defaultInputDevice(); + QAudioDeviceInfo audioInfo = QAudioDeviceInfo::defaultInputDevice(); QList<QAudioDeviceInfo> devices = QAudioDeviceInfo::availableDevices(QAudio::AudioInput); @@ -129,20 +129,23 @@ void QtMicrophoneWrapper::setAudioDevice(const QString &deviceName) { for (QAudioDeviceInfo &device : devices) { qDebug() << " device name: " << device.deviceName(); if (device.deviceName() == deviceName) { - m_audioInfo = device; + audioInfo = device; } } - qDebug() << "Selected capture device:" << m_audioInfo.deviceName(); + qDebug() << "Selected capture device:" << audioInfo.deviceName(); qDebug() << "Requested format" << format; - if (!m_audioInfo.isFormatSupported(format)) { + if (!audioInfo.isFormatSupported(format)) { qWarning() << "QtMicrophoneWrapper: Default format not supported, trying to use the nearest."; - format = m_audioInfo.nearestFormat(format); + format = audioInfo.nearestFormat(format); qWarning() << "QtMicrophoneWrapper: Nearest format" << format; } - m_audioInput = new QAudioInput(m_audioInfo, format, this); + m_audioInput = new QAudioInput(audioInfo, format, this); + + m_audioLevelInfo.init(m_audioInput->format()); + QObject::connect(m_audioInput, &QAudioInput::notify, this, [this](){ QByteArray readBytes = m_audioInputIODevice->readAll(); @@ -153,6 +156,12 @@ void QtMicrophoneWrapper::setAudioDevice(const QString &deviceName) { static_cast<size_t>(m_readAudioDataBytes)/m_writer->getWordSize() : static_cast<size_t>(m_readAudioDataBytes); m_writer->write(m_readAudioData.data(), nWords); + + if (m_levelProcess) { + m_audioLevel = m_audioLevelInfo.processBuffer(m_readAudioData); + Q_EMIT audioLevelChanged(); + } + m_readAudioData.clear(); m_readAudioDataBytes = 0; }); @@ -161,3 +170,200 @@ void QtMicrophoneWrapper::setAudioDevice(const QString &deviceName) { m_audioInput->setNotifyInterval(latency); qDebug("QtMicrophoneWrapper: Latency is configured to: %d ms", m_audioInput->notifyInterval()); } + +AudioLevelInfo::AudioLevelInfo(const QAudioFormat &format) +{ + init(format); +} + +bool AudioLevelInfo::init(const QAudioFormat &format) +{ + m_valid = true; + m_maxAmplitude = 0; + + switch (format.sampleSize()) { + case 8: + switch (format.sampleType()) { + case QAudioFormat::UnSignedInt: + m_maxAmplitude = 255; + break; + case QAudioFormat::SignedInt: + m_maxAmplitude = 127; + break; + default: + break; + } + break; + case 16: + switch (format.sampleType()) { + case QAudioFormat::UnSignedInt: + m_maxAmplitude = 65535; + break; + case QAudioFormat::SignedInt: + m_maxAmplitude = 32767; + break; + default: + break; + } + break; + case 32: + switch (format.sampleType()) { + case QAudioFormat::UnSignedInt: + m_maxAmplitude = 0xffffffff; + break; + case QAudioFormat::SignedInt: + m_maxAmplitude = 0x7fffffff; + break; + case QAudioFormat::Float: + m_maxAmplitude = 0x7fffffff; + break; + default: + break; + } + break; + default: + m_valid = false; + return false; + } + + if (format.sampleSize() == 8 && format.sampleType() == QAudioFormat::UnSignedInt) { + m_getAudioLevelValue = &AudioLevelInfo::processUnSignedInt8; + } else if (format.sampleSize() == 8 && format.sampleType() == QAudioFormat::SignedInt) { + m_getAudioLevelValue = &AudioLevelInfo::processSignedInt8; + } else if (format.sampleSize() == 16 && format.sampleType() == QAudioFormat::UnSignedInt) { + if (format.byteOrder() == QAudioFormat::LittleEndian) { + m_getAudioLevelValue = &AudioLevelInfo::processUnSignedInt16LE; + } + else { + m_getAudioLevelValue = &AudioLevelInfo::processUnSignedInt16BE; + } + } else if (format.sampleSize() == 16 && format.sampleType() == QAudioFormat::SignedInt) { + if (format.byteOrder() == QAudioFormat::LittleEndian) { + m_getAudioLevelValue = &AudioLevelInfo::processSignedInt16LE; + } + else{ + m_getAudioLevelValue = &AudioLevelInfo::processSignedInt16BE; + } + } else if (format.sampleSize() == 32 && format.sampleType() == QAudioFormat::UnSignedInt) { + if (format.byteOrder() == QAudioFormat::LittleEndian) { + m_getAudioLevelValue = &AudioLevelInfo::processUnSignedInt32LE; + } + else { + m_getAudioLevelValue = &AudioLevelInfo::processUnSignedInt32BE; + } + } else if (format.sampleSize() == 32 && format.sampleType() == QAudioFormat::SignedInt) { + if (format.byteOrder() == QAudioFormat::LittleEndian) { + m_getAudioLevelValue = &AudioLevelInfo::processSignedInt32LE; + } + else { + m_getAudioLevelValue = &AudioLevelInfo::processSignedInt32BE; + } + } else if (format.sampleSize() == 32 && format.sampleType() == QAudioFormat::Float) { + m_getAudioLevelValue = &AudioLevelInfo::processFloat; + } else { + m_getAudioLevelValue = &AudioLevelInfo::processDefault; + m_valid = false; + } + + if (format.sampleSize() % 8 != 0) { + m_valid = false; + } + + m_channelBytes = format.sampleSize() / 8; + m_sampleBytes = m_channelBytes * format.channelCount(); + m_channelCount = format.channelCount(); + + return m_valid; +} + +qreal AudioLevelInfo::processBuffer(const QByteArray &ba) const +{ + if (ba.size() == 0) { + return 0.0; + } + + if (m_valid) { + if (ba.size() % m_sampleBytes != 0) + return 0.0; + + const int numSamples = ba.size() / m_sampleBytes; + const int step = numSamples / 50 + 1; //check 50 samples from buffer, skip all other info + + quint32 maxValue = 0; + const char *ptr = ba.constData(); + + for (int i = 0; i < numSamples; i += step) { + for (int j = 0; j < m_channelCount; ++j) { + quint32 value = (*m_getAudioLevelValue)(ptr); + maxValue = qMax(value, maxValue); + ptr += m_channelBytes; + } + ptr += m_channelBytes * (m_channelCount) * (step - 1); + } + + maxValue = qMin(maxValue, m_maxAmplitude); + return qreal(maxValue) / m_maxAmplitude; + } + + return 0.0; +} + +quint32 AudioLevelInfo::processUnSignedInt8(const char *ptr) +{ + return *reinterpret_cast<const quint8*>(ptr); +} + +quint32 AudioLevelInfo::processSignedInt8(const char *ptr) +{ + return static_cast<quint32>(qAbs(*reinterpret_cast<const qint8*>(ptr))); +} + +quint32 AudioLevelInfo::processUnSignedInt16LE(const char *ptr) +{ + return qFromLittleEndian<quint16>(ptr); +} + +quint32 AudioLevelInfo::processUnSignedInt16BE(const char *ptr) +{ + return qFromBigEndian<quint16>(ptr); +} + +quint32 AudioLevelInfo::processSignedInt16LE(const char *ptr) +{ + return static_cast<quint32>(qAbs(qFromLittleEndian<qint16>(ptr))); +} + +quint32 AudioLevelInfo::processSignedInt16BE(const char *ptr) +{ + return static_cast<quint32>(qAbs(qFromBigEndian<qint16>(ptr))); +} + +quint32 AudioLevelInfo::processUnSignedInt32LE(const char *ptr) +{ + return qFromLittleEndian<quint32>(ptr); +} + +quint32 AudioLevelInfo::processUnSignedInt32BE(const char *ptr) +{ + return qFromBigEndian<quint32>(ptr); +} + +quint32 AudioLevelInfo::processSignedInt32LE(const char *ptr) +{ + return static_cast<quint32>(qAbs(qFromLittleEndian<qint32>(ptr))); +} + +quint32 AudioLevelInfo::processSignedInt32BE(const char *ptr) +{ + return static_cast<quint32>(qAbs(qFromBigEndian<qint32>(ptr))); +} + +quint32 AudioLevelInfo::processFloat(const char *ptr) +{ + return static_cast<quint32>(qAbs(*reinterpret_cast<const float*>(ptr) * 0x7fffffff)); // assumes 0-1.0 +} +quint32 AudioLevelInfo::processDefault(const char *ptr) +{ + Q_UNUSED(ptr) + return 0; +} diff --git a/plugins/alexainterface/QtMicrophoneWrapper.h b/plugins/alexainterface/QtMicrophoneWrapper.h index cf81596..bfeb5ae 100644 --- a/plugins/alexainterface/QtMicrophoneWrapper.h +++ b/plugins/alexainterface/QtMicrophoneWrapper.h @@ -40,6 +40,48 @@ using namespace alexaClientSDK; +/** + * @brief The AudioLevelInfo class + * + * provides processBuffer() function to get avg audio level from QByteArray data + * should be initialized with QAudioFormat in constructor or by init() function + * + */ +class AudioLevelInfo +{ +public: + explicit AudioLevelInfo(const QAudioFormat &format); + AudioLevelInfo() {} + bool init(const QAudioFormat &format); + + qreal processBuffer(const QByteArray &ba) const; +private: + bool m_valid = false; + + /// format-specific for processing speed-up + quint32 m_maxAmplitude = 0; + int m_sampleBytes = 0; + int m_channelBytes = 0; + int m_channelCount = 0; + + /// pointer to static member function chosen for format + quint32 (*m_getAudioLevelValue)(const char *) = &AudioLevelInfo::processDefault; + + /// buffer process functions + static quint32 processUnSignedInt8(const char *ptr); + static quint32 processSignedInt8(const char *ptr); + static quint32 processUnSignedInt16LE(const char *ptr); + static quint32 processUnSignedInt16BE(const char *ptr); + static quint32 processSignedInt16LE(const char *ptr); + static quint32 processSignedInt16BE(const char *ptr); + static quint32 processUnSignedInt32LE(const char *ptr); + static quint32 processUnSignedInt32BE(const char *ptr); + static quint32 processSignedInt32LE(const char *ptr); + static quint32 processSignedInt32BE(const char *ptr); + static quint32 processFloat(const char *ptr); + static quint32 processDefault(const char *ptr); +}; + class QtMicrophoneWrapper : public QObject , public applicationUtilities::resources::audio::MicrophoneInterface { @@ -70,6 +112,12 @@ public: virtual ~QtMicrophoneWrapper() override; + qreal audioLevel() const { return m_audioLevel; } + void setLevelProcess(bool enable) { m_levelProcess = enable; } + +Q_SIGNALS: + void audioLevelChanged(); + private: /** * Constructor. @@ -77,11 +125,14 @@ private: */ QtMicrophoneWrapper(std::shared_ptr<avsCommon::avs::AudioInputStream> stream); - QAudioDeviceInfo m_audioInfo; QAudioInput *m_audioInput = nullptr; QIODevice *m_audioInputIODevice = nullptr; int m_readAudioDataBytes = 0; QByteArray m_readAudioData; + qreal m_audioLevel = 0.0; // 0.0 <= m_audioLevel <= 1.0 + bool m_levelProcess = false; + AudioLevelInfo m_audioLevelInfo; + /// Initializes Audio bool initialize(const QString &deviceName); |