path: root/src/multimedia/platform/android/mediacapture/qandroidcapturesession.cpp

/****************************************************************************
**
** Copyright (C) 2016 The Qt Company Ltd.
** Contact: https://www.qt.io/licensing/
**
** This file is part of the Qt Toolkit.
**
** $QT_BEGIN_LICENSE:LGPL$
** Commercial License Usage
** Licensees holding valid commercial Qt licenses may use this file in
** accordance with the commercial license agreement provided with the
** Software or, alternatively, in accordance with the terms contained in
** a written agreement between you and The Qt Company. For licensing terms
** and conditions see https://www.qt.io/terms-conditions. For further
** information use the contact form at https://www.qt.io/contact-us.
**
** GNU Lesser General Public License Usage
** Alternatively, this file may be used under the terms of the GNU Lesser
** General Public License version 3 as published by the Free Software
** Foundation and appearing in the file LICENSE.LGPL3 included in the
** packaging of this file. Please review the following information to
** ensure the GNU Lesser General Public License version 3 requirements
** will be met: https://www.gnu.org/licenses/lgpl-3.0.html.
**
** GNU General Public License Usage
** Alternatively, this file may be used under the terms of the GNU
** General Public License version 2.0 or (at your option) the GNU General
** Public license version 3 or any later version approved by the KDE Free
** Qt Foundation. The licenses are as published by the Free Software
** Foundation and appearing in the file LICENSE.GPL2 and LICENSE.GPL3
** included in the packaging of this file. Please review the following
** information to ensure the GNU General Public License requirements will
** be met: https://www.gnu.org/licenses/gpl-2.0.html and
** https://www.gnu.org/licenses/gpl-3.0.html.
**
** $QT_END_LICENSE$
**
****************************************************************************/

#include "qandroidcapturesession_p.h"

#include "androidcamera_p.h"
#include "qandroidcamerasession_p.h"
#include "androidmediaplayer_p.h"
#include "androidmultimediautils_p.h"
#include "qandroidmultimediautils_p.h"
#include "qandroidvideooutput_p.h"
#include "qandroidglobal_p.h"
#include <private/qplatformaudioinput_p.h>
#include <private/qplatformaudiooutput_p.h>
#include <private/qmediarecorder_p.h>
#include <private/qmediastoragelocation_p.h>
#include <QtCore/qmimetype.h>

#include <algorithm>

QT_BEGIN_NAMESPACE

QAndroidCaptureSession::QAndroidCaptureSession()
    : QObject()
    , m_mediaRecorder(0)
    , m_cameraSession(0)
    , m_duration(0)
    , m_state(QMediaRecorder::StoppedState)
    , m_outputFormat(AndroidMediaRecorder::DefaultOutputFormat)
    , m_audioEncoder(AndroidMediaRecorder::DefaultAudioEncoder)
    , m_videoEncoder(AndroidMediaRecorder::DefaultVideoEncoder)
{
    m_notifyTimer.setInterval(1000);
    connect(&m_notifyTimer, &QTimer::timeout, this, &QAndroidCaptureSession::updateDuration);
}

QAndroidCaptureSession::~QAndroidCaptureSession()
{
    stop();
    m_mediaRecorder = nullptr;
}

void QAndroidCaptureSession::setCameraSession(QAndroidCameraSession *cameraSession)
{
    if (m_cameraSession) {
        disconnect(m_connOpenCamera);
        disconnect(m_connActiveChangedCamera);
    }

    m_cameraSession = cameraSession;
    if (m_cameraSession) {
        m_connOpenCamera = connect(cameraSession, &QAndroidCameraSession::opened,
                                   this, &QAndroidCaptureSession::onCameraOpened);
        m_connActiveChangedCamera = connect(cameraSession, &QAndroidCameraSession::activeChanged,
                                            this, [this](bool isActive) {
            if (!isActive)
                stop();
        });
    }
}

void QAndroidCaptureSession::setAudioInput(QPlatformAudioInput *input)
{
    m_audioInput = input;
}

void QAndroidCaptureSession::setAudioOutput(QPlatformAudioOutput *output)
{
    if (m_audioOutput == output)
        return;

    m_audioOutput = output;

    if (m_audioOutput)
        AndroidMediaPlayer::setAudioOutput(m_audioOutput->device.id());
}

QMediaRecorder::RecorderState QAndroidCaptureSession::state() const
{
    return m_state;
}

void QAndroidCaptureSession::setKeepAlive(bool keepAlive)
{
    if (m_cameraSession)
        m_cameraSession->setKeepAlive(keepAlive);
}


void QAndroidCaptureSession::start(QMediaEncoderSettings &settings, const QUrl &outputLocation)
{
    if (m_state == QMediaRecorder::RecordingState)
        return;

    if (!m_cameraSession && !m_audioInput) {
        emit error(QMediaRecorder::ResourceError, QLatin1String("No devices are set"));
        return;
    }

    setKeepAlive(true);

    if (m_cameraSession && !qt_androidRequestCameraPermission()) {
        emit error(QMediaRecorder::ResourceError, QLatin1String("Camera permission denied."));
        setKeepAlive(false);
        return;
    }

    if (m_audioInput && !qt_androidRequestRecordingPermission()) {
        emit error(QMediaRecorder::ResourceError, QLatin1String("Microphone permission denied."));
        setKeepAlive(false);
        return;
    }

    m_mediaRecorder = std::make_shared<AndroidMediaRecorder>();
    connect(m_mediaRecorder.get(), &AndroidMediaRecorder::error, this,
            &QAndroidCaptureSession::onError);
    connect(m_mediaRecorder.get(), &AndroidMediaRecorder::info, this,
            &QAndroidCaptureSession::onInfo);

    applySettings(settings);

    // Set audio/video sources
    if (m_cameraSession) {
        m_cameraSession->camera()->stopPreviewSynchronous();
        m_cameraSession->applyResolution(settings.videoResolution(), false);
        m_cameraSession->camera()->unlock();

        m_mediaRecorder->setCamera(m_cameraSession->camera());
        m_mediaRecorder->setVideoSource(AndroidMediaRecorder::Camera);
    }

    if (m_audioInput) {
        m_mediaRecorder->setAudioSource(AndroidMediaRecorder::Camcorder);
        m_mediaRecorder->setAudioInput(m_audioInput->device.id());
        if (!m_mediaRecorder->isAudioSourceSet())
            m_mediaRecorder->setAudioSource(AndroidMediaRecorder::DefaultAudioSource);
    }

    // Set output format
    m_mediaRecorder->setOutputFormat(m_outputFormat);

    // Set video encoder settings
    if (m_cameraSession) {
        m_mediaRecorder->setVideoSize(settings.videoResolution());
        m_mediaRecorder->setVideoFrameRate(qRound(settings.videoFrameRate()));
        m_mediaRecorder->setVideoEncodingBitRate(settings.videoBitRate());
        m_mediaRecorder->setVideoEncoder(m_videoEncoder);

        // media recorder is also compensanting the mirror on front camera
        auto rotation = m_cameraSession->currentCameraRotation();
        if (m_cameraSession->camera()->getFacing() == AndroidCamera::CameraFacingFront)
            rotation = (360 - rotation) % 360; // remove mirror compensation

        m_mediaRecorder->setOrientationHint(rotation);
    }

    // Set audio encoder settings
    if (m_audioInput) {
        m_mediaRecorder->setAudioChannels(settings.audioChannelCount());
        m_mediaRecorder->setAudioEncodingBitRate(settings.audioBitRate());
        m_mediaRecorder->setAudioSamplingRate(settings.audioSampleRate());
        m_mediaRecorder->setAudioEncoder(m_audioEncoder);
    }

    QString extension = settings.mimeType().preferredSuffix();

    // Set output file
    auto location = outputLocation.toString(QUrl::PreferLocalFile);
    auto filePath = QMediaStorageLocation::generateFileName(
                location, m_cameraSession ? QStandardPaths::MoviesLocation : QStandardPaths::MusicLocation, extension);

    m_usedOutputLocation = QUrl::fromLocalFile(filePath);
    m_outputLocationIsStandard = location.isEmpty() || QFileInfo(location).isRelative();
    m_mediaRecorder->setOutputFile(filePath);

    // Even though the Android doc explicitly says that calling MediaRecorder.setPreviewDisplay()
    // is not necessary when the Camera already has a Surface, it doesn't actually work on some
    // devices. For example on the Samsung Galaxy Tab 2, the camera server dies after prepare()
    // and start() if MediaRecorder.setPreviewDisplay() is not called.
    if (m_cameraSession) {
        // When using a SurfaceTexture, we need to pass a new one to the MediaRecorder, not the same
        // one that is set on the Camera or it will crash, hence the reset().
        m_cameraSession->videoOutput()->reset();
        if (m_cameraSession->videoOutput()->surfaceTexture())
            m_mediaRecorder->setSurfaceTexture(m_cameraSession->videoOutput()->surfaceTexture());
        else if (m_cameraSession->videoOutput()->surfaceHolder())
            m_mediaRecorder->setSurfaceHolder(m_cameraSession->videoOutput()->surfaceHolder());

        m_cameraSession->disableRotation();
    }

    if (!m_mediaRecorder->prepare()) {
        emit error(QMediaRecorder::FormatError, QLatin1String("Unable to prepare the media recorder."));
        restartViewfinder();

        return;
    }

    if (!m_mediaRecorder->start()) {
        emit error(QMediaRecorder::FormatError,
                   QMediaRecorderPrivate::msgFailedStartRecording());
        restartViewfinder();

        return;
    }

    m_elapsedTime.start();
    m_notifyTimer.start();
    updateDuration();

    if (m_cameraSession) {
        m_cameraSession->setReadyForCapture(false);

        // Preview frame callback is cleared when setting up the camera with the media recorder.
        // We need to reset it.
        m_cameraSession->camera()->setupPreviewFrameCallback();
    }

    m_state = QMediaRecorder::RecordingState;
    emit stateChanged(m_state);
}

void QAndroidCaptureSession::stop(bool error)
{
    if (m_state == QMediaRecorder::StoppedState || m_mediaRecorder == nullptr)
        return;

    m_mediaRecorder->stop();
    m_notifyTimer.stop();
    updateDuration();
    m_elapsedTime.invalidate();

    m_mediaRecorder = nullptr;

    if (m_cameraSession && m_cameraSession->isActive()) {
        // Viewport needs to be restarted after recording
        restartViewfinder();
    }

    if (!error) {
        // if the media is saved into the standard media location, register it
        // with the Android media scanner so it appears immediately in apps
        // such as the gallery.
        if (m_outputLocationIsStandard)
            AndroidMultimediaUtils::registerMediaFile(m_usedOutputLocation.toLocalFile());

        emit actualLocationChanged(m_usedOutputLocation);
    }

    m_state = QMediaRecorder::StoppedState;
    emit stateChanged(m_state);
}

qint64 QAndroidCaptureSession::duration() const
{
    return m_duration;
}

void QAndroidCaptureSession::applySettings(QMediaEncoderSettings &settings)
{
    // container settings
    auto fileFormat = settings.mediaFormat().fileFormat();
    if (!m_cameraSession && fileFormat == QMediaFormat::AAC) {
        m_outputFormat = AndroidMediaRecorder::AAC_ADTS;
    } else if (fileFormat == QMediaFormat::Ogg) {
        m_outputFormat = AndroidMediaRecorder::OGG;
    } else if (fileFormat == QMediaFormat::WebM) {
        m_outputFormat = AndroidMediaRecorder::WEBM;
//    } else if (fileFormat == QLatin1String("3gp")) {
//        m_outputFormat = AndroidMediaRecorder::THREE_GPP;
    } else {
        // fallback to MP4
        m_outputFormat = AndroidMediaRecorder::MPEG_4;
    }

    // audio settings
    if (settings.audioChannelCount() <= 0)
        settings.setAudioChannelCount(m_defaultSettings.audioChannels);
    if (settings.audioBitRate() <= 0)
        settings.setAudioBitRate(m_defaultSettings.audioBitRate);
    if (settings.audioSampleRate() <= 0)
        settings.setAudioSampleRate(m_defaultSettings.audioSampleRate);

    if (settings.audioCodec() == QMediaFormat::AudioCodec::AAC)
        m_audioEncoder = AndroidMediaRecorder::AAC;
    else if (settings.audioCodec() == QMediaFormat::AudioCodec::Opus)
        m_audioEncoder = AndroidMediaRecorder::OPUS;
    else if (settings.audioCodec() == QMediaFormat::AudioCodec::Vorbis)
        m_audioEncoder = AndroidMediaRecorder::VORBIS;
    else
        m_audioEncoder = m_defaultSettings.audioEncoder;


    // video settings
    if (m_cameraSession && m_cameraSession->camera()) {
        if (settings.videoResolution().isEmpty()) {
            settings.setVideoResolution(m_defaultSettings.videoResolution);
        } else if (!m_supportedResolutions.contains(settings.videoResolution())) {
            // if the requested resolution is not supported, find the closest one
            QSize reqSize = settings.videoResolution();
            int reqPixelCount = reqSize.width() * reqSize.height();
            QList<int> supportedPixelCounts;
            for (int i = 0; i < m_supportedResolutions.size(); ++i) {
                const QSize &s = m_supportedResolutions.at(i);
                supportedPixelCounts.append(s.width() * s.height());
            }
            int closestIndex = qt_findClosestValue(supportedPixelCounts, reqPixelCount);
            settings.setVideoResolution(m_supportedResolutions.at(closestIndex));
        }

        if (settings.videoFrameRate() <= 0)
            settings.setVideoFrameRate(m_defaultSettings.videoFrameRate);
        if (settings.videoBitRate() <= 0)
            settings.setVideoBitRate(m_defaultSettings.videoBitRate);

        if (settings.videoCodec() == QMediaFormat::VideoCodec::H264)
            m_videoEncoder = AndroidMediaRecorder::H264;
        else if (settings.videoCodec() == QMediaFormat::VideoCodec::H265)
            m_videoEncoder = AndroidMediaRecorder::HEVC;
        else if (settings.videoCodec() == QMediaFormat::VideoCodec::MPEG4)
            m_videoEncoder = AndroidMediaRecorder::MPEG_4_SP;
        else
            m_videoEncoder = m_defaultSettings.videoEncoder;

    }
}

void QAndroidCaptureSession::restartViewfinder()
{
    if (!m_cameraSession)
        return;
    m_cameraSession->camera()->reconnect();

    // This is not necessary on most devices, but it crashes on some if we don't stop the
    // preview and reset the preview display on the camera when recording is over.
    m_cameraSession->camera()->stopPreviewSynchronous();
    m_cameraSession->videoOutput()->reset();
    if (m_cameraSession->videoOutput()->surfaceTexture())
        m_cameraSession->camera()->setPreviewTexture(m_cameraSession->videoOutput()->surfaceTexture());
    else if (m_cameraSession->videoOutput()->surfaceHolder())
        m_cameraSession->camera()->setPreviewDisplay(m_cameraSession->videoOutput()->surfaceHolder());

    m_cameraSession->camera()->startPreview();
    m_cameraSession->setReadyForCapture(true);
    m_cameraSession->enableRotation();
    setKeepAlive(false);
    m_mediaRecorder = nullptr;
}

void QAndroidCaptureSession::updateDuration()
{
    if (m_elapsedTime.isValid())
        m_duration = m_elapsedTime.elapsed();

    emit durationChanged(m_duration);
}

void QAndroidCaptureSession::onCameraOpened()
{
    m_supportedResolutions.clear();
    m_supportedFramerates.clear();

    // get supported resolutions from predefined profiles
    for (int i = 0; i < 8; ++i) {
        CaptureProfile profile = getProfile(i);
        if (!profile.isNull) {
            if (i == AndroidCamcorderProfile::QUALITY_HIGH)
                m_defaultSettings = profile;

            if (!m_supportedResolutions.contains(profile.videoResolution))
                m_supportedResolutions.append(profile.videoResolution);
            if (!m_supportedFramerates.contains(profile.videoFrameRate))
                m_supportedFramerates.append(profile.videoFrameRate);
        }
    }

    std::sort(m_supportedResolutions.begin(), m_supportedResolutions.end(), qt_sizeLessThan);
    std::sort(m_supportedFramerates.begin(), m_supportedFramerates.end());
}

QAndroidCaptureSession::CaptureProfile QAndroidCaptureSession::getProfile(int id)
{
    CaptureProfile profile;
    const bool hasProfile = AndroidCamcorderProfile::hasProfile(m_cameraSession->camera()->cameraId(),
                                                                AndroidCamcorderProfile::Quality(id));

    if (hasProfile) {
        AndroidCamcorderProfile camProfile = AndroidCamcorderProfile::get(m_cameraSession->camera()->cameraId(),
                                                                          AndroidCamcorderProfile::Quality(id));

        profile.outputFormat = AndroidMediaRecorder::OutputFormat(camProfile.getValue(AndroidCamcorderProfile::fileFormat));
        profile.audioEncoder = AndroidMediaRecorder::AudioEncoder(camProfile.getValue(AndroidCamcorderProfile::audioCodec));
        profile.audioBitRate = camProfile.getValue(AndroidCamcorderProfile::audioBitRate);
        profile.audioChannels = camProfile.getValue(AndroidCamcorderProfile::audioChannels);
        profile.audioSampleRate = camProfile.getValue(AndroidCamcorderProfile::audioSampleRate);
        profile.videoEncoder = AndroidMediaRecorder::VideoEncoder(camProfile.getValue(AndroidCamcorderProfile::videoCodec));
        profile.videoBitRate = camProfile.getValue(AndroidCamcorderProfile::videoBitRate);
        profile.videoFrameRate = camProfile.getValue(AndroidCamcorderProfile::videoFrameRate);
        profile.videoResolution = QSize(camProfile.getValue(AndroidCamcorderProfile::videoFrameWidth),
                                        camProfile.getValue(AndroidCamcorderProfile::videoFrameHeight));

        if (profile.outputFormat == AndroidMediaRecorder::MPEG_4)
            profile.outputFileExtension = QStringLiteral("mp4");
        else if (profile.outputFormat == AndroidMediaRecorder::THREE_GPP)
            profile.outputFileExtension = QStringLiteral("3gp");
        else if (profile.outputFormat == AndroidMediaRecorder::AMR_NB_Format)
            profile.outputFileExtension = QStringLiteral("amr");
        else if (profile.outputFormat == AndroidMediaRecorder::AMR_WB_Format)
            profile.outputFileExtension = QStringLiteral("awb");

        profile.isNull = false;
    }

    return profile;
}

void QAndroidCaptureSession::onError(int what, int extra)
{
    Q_UNUSED(what);
    Q_UNUSED(extra);
    stop(true);
    emit error(QMediaRecorder::ResourceError, QLatin1String("Unknown error."));
}

void QAndroidCaptureSession::onInfo(int what, int extra)
{
    Q_UNUSED(extra);
    if (what == 800) {
        // MEDIA_RECORDER_INFO_MAX_DURATION_REACHED
        stop();
        emit error(QMediaRecorder::OutOfSpaceError, QLatin1String("Maximum duration reached."));
    } else if (what == 801) {
        // MEDIA_RECORDER_INFO_MAX_FILESIZE_REACHED
        stop();
        emit error(QMediaRecorder::OutOfSpaceError, QLatin1String("Maximum file size reached."));
    }
}

QT_END_NAMESPACE