diff options
Diffstat (limited to 'src/multimedia/platform/android/audio')
6 files changed, 153 insertions, 80 deletions
diff --git a/src/multimedia/platform/android/audio/qandroidaudiodecoder.cpp b/src/multimedia/platform/android/audio/qandroidaudiodecoder.cpp index 7e3fe9a01..2a35a06eb 100644 --- a/src/multimedia/platform/android/audio/qandroidaudiodecoder.cpp +++ b/src/multimedia/platform/android/audio/qandroidaudiodecoder.cpp @@ -53,8 +53,7 @@ QT_BEGIN_NAMESPACE -static const char tempFile[] = "encoded.tmp"; -static const char tempPath[] = "/storage/emulated/0/data/local/tmp/audiodecoder/"; +static const char tempFile[] = "encoded.wav"; constexpr int dequeueTimeout = 5000; Q_LOGGING_CATEGORY(adLogger, "QAndroidAudioDecoder") @@ -92,6 +91,20 @@ void Decoder::stop() void Decoder::setSource(const QUrl &source) { + const QJniObject path = QJniObject::callStaticObjectMethod( + "org/qtproject/qt/android/multimedia/QtMultimediaUtils", + "getMimeType", + "(Landroid/content/Context;Ljava/lang/String;)Ljava/lang/String;", + QNativeInterface::QAndroidApplication::context(), + QJniObject::fromString(source.path()).object()); + + const QString mime = path.isValid() ? path.toString() : ""; + + if (!mime.isEmpty() && !mime.contains("audio", Qt::CaseInsensitive)) { + m_formatError = tr("Cannot set source, invalid mime type for the provided source."); + return; + } + if (!m_extractor) m_extractor = AMediaExtractor_new(); @@ -108,9 +121,9 @@ void Decoder::setSource(const QUrl &source) } if (fd < 0) { - emit error(QAudioDecoder::ResourceError, tr("Invalid fileDescriptor for source.")); - return; - } + emit error(QAudioDecoder::ResourceError, tr("Invalid fileDescriptor for source.")); + return; + } const int size = QFile(source.toString()).size(); media_status_t status = AMediaExtractor_setDataSourceFd(m_extractor, fd, 0, size > 0 ? size : LONG_MAX); @@ -121,7 +134,7 @@ void Decoder::setSource(const QUrl &source) AMediaExtractor_delete(m_extractor); m_extractor = nullptr; } - emit error(QAudioDecoder::ResourceError, tr("Setting source for Audio Decoder failed.")); + m_formatError = tr("Setting source for Audio Decoder failed."); } } @@ -162,6 +175,11 @@ void Decoder::createDecoder() void Decoder::doDecode() { + if (!m_formatError.isEmpty()) { + emit error(QAudioDecoder::FormatError, m_formatError); + return; + } + if (!m_extractor) { emit error(QAudioDecoder::ResourceError, tr("Cannot decode, source not set.")); return; @@ -190,6 +208,7 @@ void Decoder::doDecode() AMediaExtractor_selectTrack(m_extractor, 0); + emit decodingChanged(true); m_inputEOS = false; while (!m_inputEOS) { // handle input buffer @@ -212,6 +231,15 @@ void Decoder::doDecode() // handle output buffer AMediaCodecBufferInfo info; ssize_t idx = AMediaCodec_dequeueOutputBuffer(m_codec, &info, dequeueTimeout); + + while (idx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED + || idx == AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED) { + if (idx == AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED) + qCWarning(adLogger) << "dequeueOutputBuffer() status: outputFormat changed"; + + idx = AMediaCodec_dequeueOutputBuffer(m_codec, &info, dequeueTimeout); + } + if (idx >= 0) { if (info.flags & AMEDIACODEC_BUFFER_FLAG_END_OF_STREAM) break; @@ -222,30 +250,22 @@ void Decoder::doDecode() &bufferSize); const QByteArray data((const char*)(bufferData + info.offset), info.size); auto audioBuffer = QAudioBuffer(data, m_outputFormat, presentationTimeUs); - if (presentationTimeUs > 0) + if (presentationTimeUs >= 0) emit positionChanged(std::move(audioBuffer), presentationTimeUs / 1000); + AMediaCodec_releaseOutputBuffer(m_codec, idx, false); } + } else if (idx == AMEDIACODEC_INFO_TRY_AGAIN_LATER) { + qCWarning(adLogger) << "dequeueOutputBuffer() status: try again later"; + break; } else { - // The outputIndex doubles as a status return if its value is < 0 - switch (idx) { - case AMEDIACODEC_INFO_TRY_AGAIN_LATER: - qCWarning(adLogger) << "dequeueOutputBuffer() status: try again later"; - break; - case AMEDIACODEC_INFO_OUTPUT_BUFFERS_CHANGED: - qCWarning(adLogger) << "dequeueOutputBuffer() status: output buffers changed"; - break; - case AMEDIACODEC_INFO_OUTPUT_FORMAT_CHANGED: - m_format = AMediaCodec_getOutputFormat(m_codec); - qCWarning(adLogger) << "dequeueOutputBuffer() status: outputFormat changed"; - break; - } + qCWarning(adLogger) << + "AMediaCodec_dequeueOutputBuffer() status: invalid buffer idx " << idx; } } else { qCWarning(adLogger) << "dequeueInputBuffer() status: invalid buffer idx " << bufferIdx; } } - emit finished(); } @@ -257,12 +277,16 @@ QAndroidAudioDecoder::QAndroidAudioDecoder(QAudioDecoder *parent) connect(m_decoder, &Decoder::durationChanged, this, &QAndroidAudioDecoder::durationChanged); connect(m_decoder, &Decoder::error, this, &QAndroidAudioDecoder::error); connect(m_decoder, &Decoder::finished, this, &QAndroidAudioDecoder::finished); + connect(m_decoder, &Decoder::decodingChanged, this, &QPlatformAudioDecoder::setIsDecoding); + connect(this, &QAndroidAudioDecoder::setSourceUrl, m_decoder, & Decoder::setSource); } QAndroidAudioDecoder::~QAndroidAudioDecoder() { - m_decoder->thread()->exit(); - m_decoder->deleteLater(); + m_decoder->thread()->quit(); + m_decoder->thread()->wait(); + delete m_threadDecoder; + delete m_decoder; } void QAndroidAudioDecoder::setSource(const QUrl &fileName) @@ -278,7 +302,7 @@ void QAndroidAudioDecoder::setSource(const QUrl &fileName) if (m_source != fileName) { m_source = fileName; - m_decoder->setSource(m_source); + emit setSourceUrl(m_source); sourceChanged(); } } @@ -304,38 +328,45 @@ void QAndroidAudioDecoder::start() if (isDecoding()) return; - setIsDecoding(true); m_position = -1; - m_threadDecoder = new QThread(this); - m_decoder->moveToThread(m_threadDecoder); - m_threadDecoder->start(); + if (m_device && (!m_device->isOpen() || !m_device->isReadable())) { + emit error(QAudioDecoder::ResourceError, + QString::fromUtf8("Unable to read from the specified device")); + return; + } + + if (!m_threadDecoder) { + m_threadDecoder = new QThread(this); + m_decoder->moveToThread(m_threadDecoder); + m_threadDecoder->start(); + } + decode(); } void QAndroidAudioDecoder::stop() { - if (!isDecoding()) + if (!isDecoding() && m_position < 0 && m_duration < 0) return; m_decoder->stop(); - - if (m_threadDecoder && m_threadDecoder->isRunning()) - m_threadDecoder->exit(); - - QMutexLocker locker(&m_buffersMutex); - m_position = -1; m_audioBuffer.clear(); - locker.unlock(); + m_position = -1; + m_duration = -1; setIsDecoding(false); + + emit bufferAvailableChanged(false); + emit QPlatformAudioDecoder::positionChanged(m_position); } QAudioBuffer QAndroidAudioDecoder::read() { - QMutexLocker locker(&m_buffersMutex); - if (m_buffersAvailable && !m_audioBuffer.isEmpty()) { - --m_buffersAvailable; - return m_audioBuffer.takeFirst(); + if (!m_audioBuffer.isEmpty()) { + QPair<QAudioBuffer, int> buffer = m_audioBuffer.takeFirst(); + m_position = buffer.second; + emit QPlatformAudioDecoder::positionChanged(buffer.second); + return buffer.first; } // no buffers available @@ -344,38 +375,29 @@ QAudioBuffer QAndroidAudioDecoder::read() bool QAndroidAudioDecoder::bufferAvailable() const { - QMutexLocker locker(&m_buffersMutex); - return m_buffersAvailable; + return m_audioBuffer.size() > 0; } qint64 QAndroidAudioDecoder::position() const { - QMutexLocker locker(&m_buffersMutex); return m_position; } qint64 QAndroidAudioDecoder::duration() const { - QMutexLocker locker(&m_buffersMutex); return m_duration; } void QAndroidAudioDecoder::positionChanged(QAudioBuffer audioBuffer, qint64 position) { - QMutexLocker locker(&m_buffersMutex); - m_audioBuffer.append(audioBuffer); + m_audioBuffer.append(QPair<QAudioBuffer, int>(audioBuffer, position)); m_position = position; - m_buffersAvailable++; - locker.unlock(); emit bufferReady(); - emit QPlatformAudioDecoder::positionChanged(position); } void QAndroidAudioDecoder::durationChanged(qint64 duration) { - QMutexLocker locker(&m_buffersMutex); m_duration = duration; - locker.unlock(); emit QPlatformAudioDecoder::durationChanged(duration); } @@ -387,9 +409,13 @@ void QAndroidAudioDecoder::error(const QAudioDecoder::Error err, const QString & void QAndroidAudioDecoder::finished() { - stop(); + emit bufferAvailableChanged(m_audioBuffer.size() > 0); + + if (m_duration != -1) + emit durationChanged(m_duration); + // remove temp file when decoding is finished - QFile(QString::fromUtf8(tempPath).append(QString::fromUtf8(tempFile))).remove(); + QFile(QString(QDir::tempPath()).append(QString::fromUtf8(tempFile))).remove(); emit QPlatformAudioDecoder::finished(); } @@ -415,22 +441,22 @@ void QAndroidAudioDecoder::decode() bool QAndroidAudioDecoder::createTempFile() { - QFile file = QFile(QString::fromUtf8(tempPath).append(QString::fromUtf8(tempFile))); - if (!QDir().mkpath(QString::fromUtf8(tempPath)) || !file.open(QIODevice::WriteOnly)) { - emit error(QAudioDecoder::ResourceError, - QString::fromUtf8("Error while creating or opening tmp file")); - return false; - } + QFile file = QFile(QDir::tempPath().append(QString::fromUtf8(tempFile)), this); - QDataStream out; - out.setDevice(&file); - out << m_deviceBuffer; - file.close(); + bool success = file.open(QIODevice::QIODevice::ReadWrite); + if (!success) + emit error(QAudioDecoder::ResourceError, tr("Error while opening tmp file")); + success &= (file.write(m_deviceBuffer) == m_deviceBuffer.size()); + if (!success) + emit error(QAudioDecoder::ResourceError, tr("Error while writing data to tmp file")); + + file.close(); m_deviceBuffer.clear(); - m_decoder->setSource(file.fileName()); + if (success) + m_decoder->setSource(file.fileName()); - return true; + return success; } void QAndroidAudioDecoder::readDevice() { diff --git a/src/multimedia/platform/android/audio/qandroidaudiodecoder_p.h b/src/multimedia/platform/android/audio/qandroidaudiodecoder_p.h index efb7cdc24..3707f4c50 100644 --- a/src/multimedia/platform/android/audio/qandroidaudiodecoder_p.h +++ b/src/multimedia/platform/android/audio/qandroidaudiodecoder_p.h @@ -53,7 +53,6 @@ #include "private/qplatformaudiodecoder_p.h" #include <QtCore/qurl.h> -#include <QtCore/qmutex.h> #include <QThread> #include "media/NdkMediaCodec.h" @@ -81,6 +80,7 @@ signals: void durationChanged(const qint64 duration); void error(const QAudioDecoder::Error error, const QString &errorString); void finished(); + void decodingChanged(bool decoding); private: void createDecoder(); @@ -90,6 +90,7 @@ private: AMediaFormat *m_format = nullptr; QAudioFormat m_outputFormat; + QString m_formatError; bool m_inputEOS; }; @@ -119,6 +120,9 @@ public: qint64 position() const override; qint64 duration() const override; +signals: + void setSourceUrl(const QUrl &source); + private slots: void positionChanged(QAudioBuffer audioBuffer, qint64 position); void durationChanged(qint64 duration); @@ -134,18 +138,16 @@ private: QIODevice *m_device = nullptr; Decoder *m_decoder; - QList<QAudioBuffer> m_audioBuffer; + QList<QPair<QAudioBuffer, int>> m_audioBuffer; QUrl m_source; - mutable QMutex m_buffersMutex; qint64 m_position = -1; qint64 m_duration = -1; long long m_presentationTimeUs = 0; - int m_buffersAvailable = 0; QByteArray m_deviceBuffer; - QThread *m_threadDecoder; + QThread *m_threadDecoder = nullptr; }; QT_END_NAMESPACE diff --git a/src/multimedia/platform/android/audio/qandroidaudiosink.cpp b/src/multimedia/platform/android/audio/qandroidaudiosink.cpp index 1a0b622a3..d7a67f207 100644 --- a/src/multimedia/platform/android/audio/qandroidaudiosink.cpp +++ b/src/multimedia/platform/android/audio/qandroidaudiosink.cpp @@ -328,6 +328,9 @@ void QAndroidAudioSink::bufferAvailable(quint32 count, quint32 playIndex) m_nextBuffer = (m_nextBuffer + 1) % BUFFER_COUNT; QMetaObject::invokeMethod(this, "onBytesProcessed", Qt::QueuedConnection, Q_ARG(qint64, readSize)); + + if (m_audioSource->atEnd()) + setState(QAudio::IdleState); } void QAndroidAudioSink::playCallback(SLPlayItf player, void *ctx, SLuint32 event) @@ -353,6 +356,9 @@ bool QAndroidAudioSink::preparePlayer() else return true; + if (!QOpenSLESEngine::setAudioOutput(m_deviceName)) + qWarning() << "Unable to setup Audio Output Device"; + SLEngineItf engine = QOpenSLESEngine::instance()->slEngine(); if (!engine) { qWarning() << "No engine"; @@ -361,7 +367,7 @@ bool QAndroidAudioSink::preparePlayer() } SLDataLocator_BufferQueue bufferQueueLocator = { SL_DATALOCATOR_BUFFERQUEUE, BUFFER_COUNT }; - SLDataFormat_PCM pcmFormat = QOpenSLESEngine::audioFormatToSLFormatPCM(m_format); + SLAndroidDataFormat_PCM_EX pcmFormat = QOpenSLESEngine::audioFormatToSLFormatPCM(m_format); SLDataSource audioSrc = { &bufferQueueLocator, &pcmFormat }; diff --git a/src/multimedia/platform/android/audio/qandroidaudiosource.cpp b/src/multimedia/platform/android/audio/qandroidaudiosource.cpp index c7eaf57ad..2f8d52830 100644 --- a/src/multimedia/platform/android/audio/qandroidaudiosource.cpp +++ b/src/multimedia/platform/android/audio/qandroidaudiosource.cpp @@ -220,7 +220,7 @@ bool QAndroidAudioSource::startRecording() SLDataLocator_BufferQueue loc_bq = { SL_DATALOCATOR_BUFFERQUEUE, NUM_BUFFERS }; #endif - SLDataFormat_PCM format_pcm = QOpenSLESEngine::audioFormatToSLFormatPCM(m_format); + SLAndroidDataFormat_PCM_EX format_pcm = QOpenSLESEngine::audioFormatToSLFormatPCM(m_format); SLDataSink audioSnk = { &loc_bq, &format_pcm }; // create audio recorder diff --git a/src/multimedia/platform/android/audio/qopenslesengine.cpp b/src/multimedia/platform/android/audio/qopenslesengine.cpp index 7d207a369..6032c06cc 100644 --- a/src/multimedia/platform/android/audio/qopenslesengine.cpp +++ b/src/multimedia/platform/android/audio/qopenslesengine.cpp @@ -51,6 +51,8 @@ #define CheckError(message) if (result != SL_RESULT_SUCCESS) { qWarning(message); return; } +#define SL_ANDROID_PCM_REPRESENTATION_INVALID 0 + Q_GLOBAL_STATIC(QOpenSLESEngine, openslesEngine); QOpenSLESEngine::QOpenSLESEngine() @@ -81,12 +83,12 @@ QOpenSLESEngine *QOpenSLESEngine::instance() return openslesEngine(); } -SLDataFormat_PCM QOpenSLESEngine::audioFormatToSLFormatPCM(const QAudioFormat &format) +SLAndroidDataFormat_PCM_EX QOpenSLESEngine::audioFormatToSLFormatPCM(const QAudioFormat &format) { - SLDataFormat_PCM format_pcm; - format_pcm.formatType = SL_DATAFORMAT_PCM; + SLAndroidDataFormat_PCM_EX format_pcm; + format_pcm.formatType = SL_ANDROID_DATAFORMAT_PCM_EX; format_pcm.numChannels = format.channelCount(); - format_pcm.samplesPerSec = format.sampleRate() * 1000; + format_pcm.sampleRate = format.sampleRate() * 1000; format_pcm.bitsPerSample = format.bytesPerSample() * 8; format_pcm.containerSize = format.bytesPerSample() * 8; format_pcm.channelMask = (format.channelCount() == 1 ? @@ -95,8 +97,25 @@ SLDataFormat_PCM QOpenSLESEngine::audioFormatToSLFormatPCM(const QAudioFormat &f format_pcm.endianness = (QSysInfo::ByteOrder == QSysInfo::LittleEndian ? SL_BYTEORDER_LITTLEENDIAN : SL_BYTEORDER_BIGENDIAN); - return format_pcm; + switch (format.sampleFormat()) { + case QAudioFormat::SampleFormat::UInt8: + format_pcm.representation = SL_ANDROID_PCM_REPRESENTATION_UNSIGNED_INT; + break; + case QAudioFormat::SampleFormat::Int16: + case QAudioFormat::SampleFormat::Int32: + format_pcm.representation = SL_ANDROID_PCM_REPRESENTATION_SIGNED_INT; + break; + case QAudioFormat::SampleFormat::Float: + format_pcm.representation = SL_ANDROID_PCM_REPRESENTATION_FLOAT; + break; + case QAudioFormat::SampleFormat::NSampleFormats: + case QAudioFormat::SampleFormat::Unknown: + format_pcm.representation = SL_ANDROID_PCM_REPRESENTATION_INVALID; + break; + } + + return format_pcm; } QList<QAudioDevice> QOpenSLESEngine::availableDevices(QAudioDevice::Mode mode) @@ -128,15 +147,34 @@ QList<QAudioDevice> QOpenSLESEngine::availableDevices(QAudioDevice::Mode mode) return devices; } +bool QOpenSLESEngine::setAudioOutput(const QByteArray &deviceId) +{ + return QJniObject::callStaticMethod<jboolean>( + "org/qtproject/qt/android/multimedia/QtAudioDeviceManager", + "setAudioOutput", + "(I)Z", + deviceId.toInt()); +} + static bool hasRecordPermission() { const auto recordPerm = QtAndroidPrivate::checkPermission(QtAndroidPrivate::Microphone); return recordPerm.result() == QtAndroidPrivate::Authorized; } +static bool requestPermissions() +{ + const auto recordPerm = QtAndroidPrivate::requestPermission(QtAndroidPrivate::Microphone); + return recordPerm.result() == QtAndroidPrivate::Authorized; +} + QList<int> QOpenSLESEngine::supportedChannelCounts(QAudioDevice::Mode mode) const { - if (mode == QAudioDevice::Input && hasRecordPermission()) { + bool hasRecordPermissions = hasRecordPermission(); + if (!hasRecordPermissions) + hasRecordPermissions = requestPermissions(); + + if (mode == QAudioDevice::Input && hasRecordPermissions) { if (!m_checkedInputFormats) const_cast<QOpenSLESEngine *>(this)->checkSupportedInputFormats(); return m_supportedInputChannelCounts; @@ -302,9 +340,9 @@ void QOpenSLESEngine::checkSupportedInputFormats() defaultFormat.sampleRate = SL_SAMPLINGRATE_44_1; defaultFormat.bitsPerSample = SL_PCMSAMPLEFORMAT_FIXED_32; defaultFormat.containerSize = SL_PCMSAMPLEFORMAT_FIXED_32; - defaultFormat.representation = SL_ANDROID_PCM_REPRESENTATION_FLOAT; defaultFormat.channelMask = SL_ANDROID_MAKE_INDEXED_CHANNEL_MASK(SL_SPEAKER_FRONT_CENTER); defaultFormat.endianness = SL_BYTEORDER_LITTLEENDIAN; + defaultFormat.representation = SL_ANDROID_PCM_REPRESENTATION_SIGNED_INT; const SLuint32 rates[13] = { SL_SAMPLINGRATE_8, SL_SAMPLINGRATE_11_025, diff --git a/src/multimedia/platform/android/audio/qopenslesengine_p.h b/src/multimedia/platform/android/audio/qopenslesengine_p.h index 36e994fb2..0f9781bd5 100644 --- a/src/multimedia/platform/android/audio/qopenslesengine_p.h +++ b/src/multimedia/platform/android/audio/qopenslesengine_p.h @@ -72,9 +72,10 @@ public: SLEngineItf slEngine() const { return m_engine; } - static SLDataFormat_PCM audioFormatToSLFormatPCM(const QAudioFormat &format); + static SLAndroidDataFormat_PCM_EX audioFormatToSLFormatPCM(const QAudioFormat &format); static QList<QAudioDevice> availableDevices(QAudioDevice::Mode mode); + static bool setAudioOutput(const QByteArray &deviceId); QList<int> supportedChannelCounts(QAudioDevice::Mode mode) const; QList<int> supportedSampleRates(QAudioDevice::Mode mode) const; |