From c76a0eb45eaabd1463bcd7291a407ff34c7407c1 Mon Sep 17 00:00:00 2001 From: Doris Verria Date: Wed, 7 Apr 2021 07:13:29 +0200 Subject: Support Audio-Only recordings on macOS Add functionality to AVFAssetWriter to capture audio-only recordings. Change AVFCameraSession and and AVFMediaEncoder accordingly. Change-Id: I65afc50b524eaed82c2baa0dd99fba101a7b111b Reviewed-by: Lars Knoll --- examples/multimedia/audiorecorder/CMakeLists.txt | 1 + examples/multimedia/audiorecorder/Info.plist.in | 47 ++++++ src/multimedia/platform/darwin/camera/avfcamera.mm | 7 - .../platform/darwin/camera/avfcamera_p.h | 1 - .../platform/darwin/camera/avfcameraservice.mm | 1 - .../platform/darwin/camera/avfcameraservice_p.h | 1 + .../platform/darwin/camera/avfcamerasession.mm | 150 ++++++++++++----- .../platform/darwin/camera/avfcamerasession_p.h | 28 ++-- .../platform/darwin/camera/avfmediaassetwriter.mm | 176 +++++++------------ .../platform/darwin/camera/avfmediaencoder.mm | 186 ++++++++++----------- .../platform/darwin/camera/avfmediaencoder_p.h | 1 + .../platform/darwin/camera/avfstoragelocation.mm | 4 +- .../platform/darwin/camera/avfstoragelocation_p.h | 3 +- .../platform/darwin/qdarwinformatsinfo.mm | 33 ++-- 14 files changed, 353 insertions(+), 286 deletions(-) create mode 100644 examples/multimedia/audiorecorder/Info.plist.in diff --git a/examples/multimedia/audiorecorder/CMakeLists.txt b/examples/multimedia/audiorecorder/CMakeLists.txt index 0b8d149a4..bd7305971 100644 --- a/examples/multimedia/audiorecorder/CMakeLists.txt +++ b/examples/multimedia/audiorecorder/CMakeLists.txt @@ -28,6 +28,7 @@ qt_add_executable(audiorecorder set_target_properties(audiorecorder PROPERTIES WIN32_EXECUTABLE TRUE MACOSX_BUNDLE TRUE + MACOSX_BUNDLE_INFO_PLIST ${CMAKE_CURRENT_SOURCE_DIR}/Info.plist.in ) # special case begin target_include_directories(audiorecorder PUBLIC diff --git a/examples/multimedia/audiorecorder/Info.plist.in b/examples/multimedia/audiorecorder/Info.plist.in new file mode 100644 index 000000000..ae2c6dfd9 --- /dev/null +++ b/examples/multimedia/audiorecorder/Info.plist.in @@ -0,0 +1,47 @@ + + + + + CFBundleInfoDictionaryVersion + 6.0 + CFBundlePackageType + APPL + + CFBundleName + ${MACOSX_BUNDLE_BUNDLE_NAME} + CFBundleIdentifier + ${MACOSX_BUNDLE_GUI_IDENTIFIER} + CFBundleExecutable + ${MACOSX_BUNDLE_EXECUTABLE_NAME} + + CFBundleVersion + ${MACOSX_BUNDLE_BUNDLE_VERSION} + CFBundleShortVersionString + ${MACOSX_BUNDLE_SHORT_VERSION_STRING} + CFBundleLongVersionString + ${MACOSX_BUNDLE_LONG_VERSION_STRING} + + LSMinimumSystemVersion + ${CMAKE_OSX_DEPLOYMENT_TARGET} + + CFBundleGetInfoString + ${MACOSX_BUNDLE_INFO_STRING} + NSHumanReadableCopyright + ${MACOSX_BUNDLE_COPYRIGHT} + + CFBundleIconFile + ${MACOSX_BUNDLE_ICON_FILE} + + CFBundleDevelopmentRegion + English + + NSPrincipalClass + NSApplication + + NSMicrophoneUsageDescription + Qt Multimedia Example + + NSSupportsAutomaticGraphicsSwitching + + + diff --git a/src/multimedia/platform/darwin/camera/avfcamera.mm b/src/multimedia/platform/darwin/camera/avfcamera.mm index e6fc4bf3b..b93987fe7 100644 --- a/src/multimedia/platform/darwin/camera/avfcamera.mm +++ b/src/multimedia/platform/darwin/camera/avfcamera.mm @@ -130,8 +130,6 @@ void AVFCamera::setCaptureSession(QPlatformMediaCaptureSession *session) m_session->setActiveCamera(QCameraInfo()); m_session->setActive(m_active); m_session->setActiveCamera(m_cameraInfo); - - captureSessionChanged(); } void AVFCamera::updateStatus() @@ -180,9 +178,4 @@ QPlatformCameraImageProcessing *AVFCamera::imageProcessingControl() return m_cameraImageProcessingControl; } -void AVFCamera::captureSessionChanged() -{ - -} - #include "moc_avfcamera_p.cpp" diff --git a/src/multimedia/platform/darwin/camera/avfcamera_p.h b/src/multimedia/platform/darwin/camera/avfcamera_p.h index dc83421fb..4660c6576 100644 --- a/src/multimedia/platform/darwin/camera/avfcamera_p.h +++ b/src/multimedia/platform/darwin/camera/avfcamera_p.h @@ -95,7 +95,6 @@ public: private Q_SLOTS: void updateStatus(); - void captureSessionChanged(); private: friend class AVFCameraSession; diff --git a/src/multimedia/platform/darwin/camera/avfcameraservice.mm b/src/multimedia/platform/darwin/camera/avfcameraservice.mm index 1f71b793a..a12a476ac 100644 --- a/src/multimedia/platform/darwin/camera/avfcameraservice.mm +++ b/src/multimedia/platform/darwin/camera/avfcameraservice.mm @@ -57,7 +57,6 @@ QT_USE_NAMESPACE AVFCameraService::AVFCameraService() { m_session = new AVFCameraSession(this); - m_audioCaptureDevice = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio]; } diff --git a/src/multimedia/platform/darwin/camera/avfcameraservice_p.h b/src/multimedia/platform/darwin/camera/avfcameraservice_p.h index 53d208809..869ba113f 100644 --- a/src/multimedia/platform/darwin/camera/avfcameraservice_p.h +++ b/src/multimedia/platform/darwin/camera/avfcameraservice_p.h @@ -95,6 +95,7 @@ public: AVFCamera *avfCameraControl() const { return m_cameraControl; } AVFMediaEncoder *recorderControl() const { return m_encoder; } AVFCameraImageCapture *avfImageCaptureControl() const { return m_imageCaptureControl; } + AVCaptureDevice *audioCaptureDevice() const { return m_audioCaptureDevice; } private: bool m_muted = false; diff --git a/src/multimedia/platform/darwin/camera/avfcamerasession.mm b/src/multimedia/platform/darwin/camera/avfcamerasession.mm index d0110cc6c..2f6dcff40 100644 --- a/src/multimedia/platform/darwin/camera/avfcamerasession.mm +++ b/src/multimedia/platform/darwin/camera/avfcamerasession.mm @@ -43,6 +43,7 @@ #include "avfcamera_p.h" #include "avfcamerarenderer_p.h" #include "avfcameraimagecapture_p.h" +#include "avfmediaencoder_p.h" #include "avfcamerautility_p.h" #include @@ -57,8 +58,6 @@ QT_USE_NAMESPACE -int AVFCameraSession::m_defaultCameraIndex; - @interface AVFCameraSessionObserver : NSObject - (AVFCameraSessionObserver *) initWithCameraSession:(AVFCameraSession*)session; @@ -142,16 +141,10 @@ int AVFCameraSession::m_defaultCameraIndex; AVFCameraSession::AVFCameraSession(AVFCameraService *service, QObject *parent) : QObject(parent) , m_service(service) - , m_videoSink(nullptr) - , m_videoInput(nil) , m_defaultCodec(0) { m_captureSession = [[AVCaptureSession alloc] init]; m_observer = [[AVFCameraSessionObserver alloc] initWithCameraSession:this]; - - //configuration is commited during transition to Active state - [m_captureSession beginConfiguration]; - setVideoOutput(new AVFCameraRenderer(this)); } AVFCameraSession::~AVFCameraSession() @@ -165,6 +158,19 @@ AVFCameraSession::~AVFCameraSession() [m_videoInput release]; } + if (m_audioInput) { + [m_captureSession removeInput:m_audioInput]; + [m_audioInput release]; + } + + if (m_audioOutput) { + [m_captureSession removeOutput:m_audioOutput]; + [m_audioOutput release]; + } + + if (m_videoOutput) + delete m_videoOutput; + [m_observer release]; [m_captureSession release]; } @@ -173,20 +179,41 @@ void AVFCameraSession::setActiveCamera(const QCameraInfo &info) { if (m_activeCameraInfo != info) { m_activeCameraInfo = info; - if (info.isNull()) - removeVideoInputDevice(); - else - attachVideoInputDevice(); + [m_captureSession beginConfiguration]; + attachVideoInputDevice(); + if (!m_videoOutput) + setVideoOutput(new AVFCameraRenderer(this)); + [m_captureSession commitConfiguration]; } } void AVFCameraSession::setVideoOutput(AVFCameraRenderer *output) { + if (m_videoOutput == output) + return; + + delete m_videoOutput; m_videoOutput = output; if (output) output->configureAVCaptureSession(this); } +void AVFCameraSession::setAudioOutput() +{ + if (m_audioOutput) { + [m_captureSession removeOutput:m_audioOutput]; + [m_audioOutput release]; + m_audioOutput = nullptr; + } + + m_audioOutput = [[AVCaptureAudioDataOutput alloc] init]; + if (m_audioOutput && [m_captureSession canAddOutput:m_audioOutput]) { + [m_captureSession addOutput:m_audioOutput]; + } else { + qWarning() << Q_FUNC_INFO << "failed to add audio output"; + } +} + AVCaptureDevice *AVFCameraSession::videoCaptureDevice() const { if (m_videoInput) @@ -195,6 +222,14 @@ AVCaptureDevice *AVFCameraSession::videoCaptureDevice() const return nullptr; } +AVCaptureDevice *AVFCameraSession::audioCaptureDevice() const +{ + if (m_audioInput) + return m_audioInput.device; + + return nullptr; +} + bool AVFCameraSession::isActive() const { return m_active; @@ -208,29 +243,35 @@ void AVFCameraSession::setActive(bool active) qDebugCamera() << Q_FUNC_INFO << m_active << " -> " << active; if (active) { - attachVideoInputDevice(); - Q_EMIT readyToConfigureConnections(); - m_defaultCodec = 0; - defaultCodec(); + [m_captureSession beginConfiguration]; + + if (m_service->audioCaptureDevice()) { + attachAudioInputDevice(); + setAudioOutput(); + } + if (!m_activeCameraInfo.isNull()) { + attachVideoInputDevice(); + Q_EMIT readyToConfigureConnections(); + m_defaultCodec = 0; + defaultCodec(); + } bool activeFormatSet = applyImageEncoderSettings(); [m_captureSession commitConfiguration]; if (activeFormatSet) { - // According to the doc, the capture device must be locked before - // startRunning to prevent the format we set to be overriden by the - // session preset. + // According to the doc, the capture device must be locked before + // startRunning to prevent the format we set to be overridden by the + // session preset. [videoCaptureDevice() lockForConfiguration:nil]; } - [m_captureSession startRunning]; if (activeFormatSet) [videoCaptureDevice() unlockForConfiguration]; } else { [m_captureSession stopRunning]; - [m_captureSession beginConfiguration]; } } @@ -259,16 +300,7 @@ void AVFCameraSession::processSessionStopped() } } -//void AVFCameraSession::onCaptureModeChanged(QCamera::CaptureModes mode) -//{ -// Q_UNUSED(mode); - -// const QCamera::State s = state(); -// if (s == QCamera::LoadedState || s == QCamera::ActiveState) -// applyImageEncoderSettings(); -//} - -AVCaptureDevice *AVFCameraSession::createCaptureDevice() +AVCaptureDevice *AVFCameraSession::createVideoCaptureDevice() { AVCaptureDevice *device = nullptr; @@ -279,26 +311,23 @@ AVCaptureDevice *AVFCameraSession::createCaptureDevice() deviceId.constData()]]; } - if (!device) - device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]; - return device; } -void AVFCameraSession::removeVideoInputDevice() +void AVFCameraSession::attachVideoInputDevice() { + //Attach video input device: if (m_videoInput) { [m_captureSession removeInput:m_videoInput]; [m_videoInput release]; m_videoInput = nullptr; } -} -void AVFCameraSession::attachVideoInputDevice() -{ - //Attach video input device: - removeVideoInputDevice(); - AVCaptureDevice *videoDevice = createCaptureDevice(); + AVCaptureDevice *videoDevice = createVideoCaptureDevice(); + if (!videoDevice) { + m_activeCameraInfo = QCameraInfo(); + return; + } NSError *error = nil; m_videoInput = [AVCaptureDeviceInput @@ -318,6 +347,34 @@ void AVFCameraSession::attachVideoInputDevice() } } +void AVFCameraSession::attachAudioInputDevice() +{ + //Attach audio input device: + if (m_audioInput) { + [m_captureSession removeInput:m_audioInput]; + [m_audioInput release]; + m_audioInput = nullptr; + } + + AVCaptureDevice *audioDevice = m_service->audioCaptureDevice(); + + NSError *error = nil; + m_audioInput = [AVCaptureDeviceInput + deviceInputWithDevice:audioDevice + error:&error]; + + if (!m_audioInput) { + qWarning() << "Failed to create audio device input"; + } else { + if ([m_captureSession canAddInput:m_audioInput]) { + [m_audioInput retain]; + [m_captureSession addInput:m_audioInput]; + } else { + qWarning() << "Failed to connect audio device input"; + } + } +} + bool AVFCameraSession::applyImageEncoderSettings() { if (AVFCameraImageCapture *control = m_service->avfImageCaptureControl()) @@ -326,6 +383,15 @@ bool AVFCameraSession::applyImageEncoderSettings() return false; } +bool AVFCameraSession::applyEncoderSettings() +{ + if (AVFMediaEncoder *encoder = m_service->recorderControl()) { + encoder->applySettings(); + return true; + } + return false; +} + FourCharCode AVFCameraSession::defaultCodec() { if (!m_defaultCodec) { @@ -351,7 +417,7 @@ void AVFCameraSession::setVideoSink(QVideoSink *sink) m_videoSink = videoSink; - if (m_videoSink) { + if (m_videoSink && m_videoOutput) { m_videoOutput->setVideoSink(videoSink); AVCaptureVideoPreviewLayer *previewLayer = [AVCaptureVideoPreviewLayer layerWithSession:m_captureSession]; m_videoOutput->setLayer(previewLayer); diff --git a/src/multimedia/platform/darwin/camera/avfcamerasession_p.h b/src/multimedia/platform/darwin/camera/avfcamerasession_p.h index 128a20c5a..f440c3d3f 100644 --- a/src/multimedia/platform/darwin/camera/avfcamerasession_p.h +++ b/src/multimedia/platform/darwin/camera/avfcamerasession_p.h @@ -78,16 +78,19 @@ public: QCameraInfo activeCameraInfo() const { return m_activeCameraInfo; } void setActiveCamera(const QCameraInfo &info); - void setVideoOutput(AVFCameraRenderer *output); AVFCameraRenderer *videoOutput() const { return m_videoOutput; } + AVCaptureAudioDataOutput *audioOutput() const { return m_audioOutput; } + AVCaptureSession *captureSession() const { return m_captureSession; } AVCaptureDevice *videoCaptureDevice() const; + AVCaptureDevice *audioCaptureDevice() const; bool isActive() const; FourCharCode defaultCodec(); AVCaptureDeviceInput *videoInput() const {return m_videoInput;} + AVCaptureDeviceInput *audioInput() const {return m_audioInput;} void setVideoSink(QVideoSink *sink); @@ -104,24 +107,29 @@ Q_SIGNALS: void error(int error, const QString &errorString); private: - AVCaptureDevice *createCaptureDevice(); + void setVideoOutput(AVFCameraRenderer *output); + void setAudioOutput(); + AVCaptureDevice *createVideoCaptureDevice(); void attachVideoInputDevice(); - void removeVideoInputDevice(); + void attachAudioInputDevice(); bool applyImageEncoderSettings(); + bool applyEncoderSettings(); - static int m_defaultCameraIndex; QCameraInfo m_activeCameraInfo; AVFCameraService *m_service; - AVFCameraRenderer *m_videoOutput; - AVFVideoSink *m_videoSink; - - bool m_active = false; - AVCaptureSession *m_captureSession; - AVCaptureDeviceInput *m_videoInput; AVFCameraSessionObserver *m_observer; + AVFCameraRenderer *m_videoOutput = nullptr; + AVFVideoSink *m_videoSink = nullptr; + + AVCaptureDeviceInput *m_videoInput = nullptr; + AVCaptureDeviceInput *m_audioInput = nullptr; + AVCaptureAudioDataOutput *m_audioOutput = nullptr; + + bool m_active = false; + FourCharCode m_defaultCodec; }; diff --git a/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm b/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm index 32465bef9..d65519404 100644 --- a/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm +++ b/src/multimedia/platform/darwin/camera/avfmediaassetwriter.mm @@ -52,7 +52,7 @@ QT_USE_NAMESPACE namespace { -bool qt_camera_service_isValid(AVFCameraService *service) +bool qt_capture_session_isValid(AVFCameraService *service) { if (!service || !service->session()) return false; @@ -61,14 +61,6 @@ bool qt_camera_service_isValid(AVFCameraService *service) if (!session->captureSession()) return false; - if (!session->videoInput()) - return false; - - if (!session->videoOutput() - || !session->videoOutput()->videoDataOutput()) { - return false; - } - return true; } @@ -84,7 +76,6 @@ using AVFAtomicInt64 = QAtomicInteger; } // unnamed namespace @interface QT_MANGLE_NAMESPACE(AVFMediaAssetWriter) (PrivateAPI) -- (bool)addAudioCapture; - (bool)addWriterInputs; - (void)setQueues; - (void)updateDuration:(CMTime)newTimeStamp; @@ -96,12 +87,8 @@ using AVFAtomicInt64 = QAtomicInteger; AVFCameraService *m_service; AVFScopedPointer m_cameraWriterInput; - AVFScopedPointer m_audioInput; - AVFScopedPointer m_audioOutput; AVFScopedPointer m_audioWriterInput; - AVCaptureDevice *m_audioCaptureDevice; - // Queue to write sample buffers: AVFScopedPointer m_writerQueue; // High priority serial queue for video output: @@ -152,8 +139,8 @@ using AVFAtomicInt64 = QAtomicInteger; { Q_ASSERT(fileURL); - if (!qt_camera_service_isValid(service)) { - qDebugCamera() << Q_FUNC_INFO << "invalid camera service"; + if (!qt_capture_session_isValid(service)) { + qDebugCamera() << Q_FUNC_INFO << "invalid capture session"; return false; } @@ -161,21 +148,28 @@ using AVFAtomicInt64 = QAtomicInteger; m_audioSettings = audioSettings; m_videoSettings = videoSettings; + AVFCameraSession *session = m_service->session(); + m_writerQueue.reset(dispatch_queue_create("asset-writer-queue", DISPATCH_QUEUE_SERIAL)); if (!m_writerQueue) { qDebugCamera() << Q_FUNC_INFO << "failed to create an asset writer's queue"; return false; } - m_videoQueue.reset(dispatch_queue_create("video-output-queue", DISPATCH_QUEUE_SERIAL)); - if (!m_videoQueue) { - qDebugCamera() << Q_FUNC_INFO << "failed to create video queue"; - return false; + if (session->videoOutput() && session->videoOutput()->videoDataOutput()) { + m_videoQueue.reset(dispatch_queue_create("video-output-queue", DISPATCH_QUEUE_SERIAL)); + if (!m_videoQueue) { + qDebugCamera() << Q_FUNC_INFO << "failed to create video queue"; + return false; + } + dispatch_set_target_queue(m_videoQueue, dispatch_get_global_queue(QOS_CLASS_USER_INITIATED, 0)); } - dispatch_set_target_queue(m_videoQueue, dispatch_get_global_queue(DISPATCH_QUEUE_PRIORITY_HIGH, 0)); + m_audioQueue.reset(dispatch_queue_create("audio-output-queue", DISPATCH_QUEUE_SERIAL)); if (!m_audioQueue) { qDebugCamera() << Q_FUNC_INFO << "failed to create audio queue"; + if (!m_videoQueue) + return false; // But we still can write video! } @@ -190,24 +184,16 @@ using AVFAtomicInt64 = QAtomicInteger; } bool audioCaptureOn = false; - if (m_audioQueue) - audioCaptureOn = [self addAudioCapture]; + audioCaptureOn = session->audioOutput() != nil; if (![self addWriterInputs]) { - if (audioCaptureOn) { - AVCaptureSession *session = m_service->session()->captureSession(); - [session removeOutput:m_audioOutput]; - [session removeInput:m_audioInput]; - m_audioOutput.reset(); - m_audioInput.reset(); - m_audioCaptureDevice = 0; - } m_assetWriter.reset(); return false; } - m_cameraWriterInput.data().transform = transform; + if (m_cameraWriterInput) + m_cameraWriterInput.data().transform = transform; // Ready to start ... return true; @@ -247,7 +233,8 @@ using AVFAtomicInt64 = QAtomicInteger; dispatch_sync(m_writerQueue, ^{}); // Done, but now we also want to prevent video queue // from updating our viewfinder: - dispatch_sync(m_videoQueue, ^{}); + if (m_videoQueue) + dispatch_sync(m_videoQueue, ^{}); // Now we're safe to stop: [m_assetWriter finishWritingWithCompletionHandler:^{ @@ -259,8 +246,6 @@ using AVFAtomicInt64 = QAtomicInteger; AVCaptureSession *session = m_service->session()->captureSession(); if (session.running) [session stopRunning]; - [session removeOutput:m_audioOutput]; - [session removeInput:m_audioInput]; QMetaObject::invokeMethod(m_delegate, "assetWriterFinished", Qt::QueuedConnection); }]; } @@ -284,7 +269,8 @@ using AVFAtomicInt64 = QAtomicInteger; dispatch_sync(m_writerQueue, ^{}); // At this point next block (if any) on the writer's queue // will see m_state preventing it from any further processing. - dispatch_sync(m_videoQueue, ^{}); + if (m_videoQueue) + dispatch_sync(m_videoQueue, ^{}); // After this point video queue will not try to modify our // viewfider, so we're safe to delete now. @@ -351,25 +337,26 @@ using AVFAtomicInt64 = QAtomicInteger; fromConnection:(AVCaptureConnection *)connection { Q_UNUSED(connection); + Q_ASSERT(m_service && m_service->session()); if (m_state.loadAcquire() != WriterStateActive) return; if (!CMSampleBufferDataIsReady(sampleBuffer)) { - qDebugCamera() << Q_FUNC_INFO << "sample buffer is not ready, skipping."; + qWarning() << Q_FUNC_INFO << "sample buffer is not ready, skipping."; return; } CFRetain(sampleBuffer); - if (captureOutput != m_audioOutput.data()) { + if (captureOutput != m_service->session()->audioOutput()) { if (m_state.loadRelaxed() != WriterStateActive) { CFRelease(sampleBuffer); return; } // Find renderercontrol's delegate and invoke its method to // show updated viewfinder's frame. - if (m_service && m_service->session() && m_service->session()->videoOutput()) { + if (m_service->session()->videoOutput()) { NSObject *vfDelegate = (NSObject *)m_service->session()->videoOutput()->captureDelegate(); if (vfDelegate) @@ -386,93 +373,55 @@ using AVFAtomicInt64 = QAtomicInteger; } } -- (bool)addAudioCapture +- (bool)addWriterInputs { - Q_ASSERT(m_service && m_service->session() && m_service->session()->captureSession()); - - AVCaptureSession *captureSession = m_service->session()->captureSession(); + Q_ASSERT(m_service && m_service->session()); + Q_ASSERT(m_assetWriter.data()); - // ##### - const auto audioDeviceInfo = m_delegate->cameraService()->audioInput(); - m_audioCaptureDevice = [AVCaptureDevice deviceWithUniqueID: - [NSString stringWithUTF8String:audioDeviceInfo.id().constData()]]; + AVFCameraSession *session = m_service->session(); - if (!m_audioCaptureDevice) { - qWarning() << Q_FUNC_INFO << "no audio input device available"; - return false; - } else { - NSError *error = nil; - m_audioInput.reset([[AVCaptureDeviceInput deviceInputWithDevice:m_audioCaptureDevice error:&error] retain]); + if (m_videoQueue) + { + Q_ASSERT(session->videoOutput() && session->videoOutput()->videoDataOutput()); + m_cameraWriterInput.reset([[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo + outputSettings:m_videoSettings + sourceFormatHint:session->videoCaptureDevice().activeFormat.formatDescription]); - if (!m_audioInput || error) { - qWarning() << Q_FUNC_INFO << "failed to create audio device input"; - m_audioCaptureDevice = 0; - m_audioInput.reset(); - return false; - } else if (![captureSession canAddInput:m_audioInput]) { - qWarning() << Q_FUNC_INFO << "could not connect the audio input"; - m_audioCaptureDevice = 0; - m_audioInput.reset(); + if (!m_cameraWriterInput) { + qDebugCamera() << Q_FUNC_INFO << "failed to create camera writer input"; return false; + } + if ([m_assetWriter canAddInput:m_cameraWriterInput]) { + [m_assetWriter addInput:m_cameraWriterInput]; } else { - [captureSession addInput:m_audioInput]; + qDebugCamera() << Q_FUNC_INFO << "failed to add camera writer input"; + m_cameraWriterInput.reset(); + return false; } - } - - - m_audioOutput.reset([[AVCaptureAudioDataOutput alloc] init]); - if (m_audioOutput.data() && [captureSession canAddOutput:m_audioOutput]) { - [captureSession addOutput:m_audioOutput]; - } else { - qDebugCamera() << Q_FUNC_INFO << "failed to add audio output"; - [captureSession removeInput:m_audioInput]; - m_audioCaptureDevice = 0; - m_audioInput.reset(); - m_audioOutput.reset(); - return false; - } - - return true; -} - -- (bool)addWriterInputs -{ - Q_ASSERT(m_service && m_service->session() && m_service->session()->videoOutput() - && m_service->session()->videoOutput()->videoDataOutput()); - Q_ASSERT(m_assetWriter.data()); - m_cameraWriterInput.reset([[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeVideo - outputSettings:m_videoSettings - sourceFormatHint:m_service->session()->videoCaptureDevice().activeFormat.formatDescription]); - if (!m_cameraWriterInput) { - qDebugCamera() << Q_FUNC_INFO << "failed to create camera writer input"; - return false; + m_cameraWriterInput.data().expectsMediaDataInRealTime = YES; } - if ([m_assetWriter canAddInput:m_cameraWriterInput]) { - [m_assetWriter addInput:m_cameraWriterInput]; - } else { - qDebugCamera() << Q_FUNC_INFO << "failed to add camera writer input"; - m_cameraWriterInput.reset(); - return false; - } - - m_cameraWriterInput.data().expectsMediaDataInRealTime = YES; - - if (m_audioOutput.data()) { - CMFormatDescriptionRef sourceFormat = m_audioCaptureDevice ? m_audioCaptureDevice.activeFormat.formatDescription : 0; + if (session->audioOutput()) { + CMFormatDescriptionRef sourceFormat = session->audioCaptureDevice() + ? session->audioCaptureDevice().activeFormat.formatDescription + : 0; m_audioWriterInput.reset([[AVAssetWriterInput alloc] initWithMediaType:AVMediaTypeAudio outputSettings:m_audioSettings sourceFormatHint:sourceFormat]); if (!m_audioWriterInput) { - qDebugCamera() << Q_FUNC_INFO << "failed to create audio writer input"; + qWarning() << Q_FUNC_INFO << "failed to create audio writer input"; // But we still can record video. + if (!m_cameraWriterInput) + return false; } else if ([m_assetWriter canAddInput:m_audioWriterInput]) { [m_assetWriter addInput:m_audioWriterInput]; m_audioWriterInput.data().expectsMediaDataInRealTime = YES; } else { - qDebugCamera() << Q_FUNC_INFO << "failed to add audio writer input"; + qWarning() << Q_FUNC_INFO << "failed to add audio writer input"; m_audioWriterInput.reset(); + if (!m_cameraWriterInput) + return false; // We can (still) write video though ... } } @@ -482,15 +431,16 @@ using AVFAtomicInt64 = QAtomicInteger; - (void)setQueues { - Q_ASSERT(m_service && m_service->session() && m_service->session()->videoOutput() - && m_service->session()->videoOutput()->videoDataOutput()); - Q_ASSERT(m_videoQueue); - - [m_service->session()->videoOutput()->videoDataOutput() setSampleBufferDelegate:self queue:m_videoQueue]; + Q_ASSERT(m_service && m_service->session()); + if (m_videoQueue) { + Q_ASSERT(m_service->session()->videoOutput() + && m_service->session()->videoOutput()->videoDataOutput()); + [m_service->session()->videoOutput()->videoDataOutput() setSampleBufferDelegate:self queue:m_videoQueue]; + } - if (m_audioOutput.data()) { + if (m_service->session()->audioOutput()) { Q_ASSERT(m_audioQueue); - [m_audioOutput setSampleBufferDelegate:self queue:m_audioQueue]; + [m_service->session()->audioOutput() setSampleBufferDelegate:self queue:m_audioQueue]; } } diff --git a/src/multimedia/platform/darwin/camera/avfmediaencoder.mm b/src/multimedia/platform/darwin/camera/avfmediaencoder.mm index be347b6a0..89b12f76a 100644 --- a/src/multimedia/platform/darwin/camera/avfmediaencoder.mm +++ b/src/multimedia/platform/darwin/camera/avfmediaencoder.mm @@ -389,11 +389,6 @@ void AVFMediaEncoder::applySettings() void AVFMediaEncoder::unapplySettings() { -// m_service->audioEncoderSettingsControl()->unapplySettings(); - -// AVCaptureConnection *conn = [m_service->videoOutput()->videoDataOutput() connectionWithMediaType:AVMediaTypeVideo]; -// m_service->videoEncoderSettingsControl()->unapplySettings(conn); - if (m_audioSettings) { [m_audioSettings release]; m_audioSettings = nil; @@ -442,101 +437,105 @@ void AVFMediaEncoder::setState(QMediaEncoder::State state) return; switch (state) { - case QMediaEncoder::RecordingState: - { - AVFCamera *cameraControl = m_service->avfCameraControl(); + case QMediaEncoder::RecordingState: + m_service->session()->setActive(true); + record(); + break; + case QMediaEncoder::PausedState: + Q_EMIT error(QMediaEncoder::FormatError, tr("Recording pause not supported")); + return; + case QMediaEncoder::StoppedState: + // Do not check the camera status, we can stop if we started. + stopWriter(); + } +} + +void AVFMediaEncoder::record() +{ + const bool audioOnly = m_settings.mode() == QMediaFormat::AudioOnly; + AVCaptureSession *session = m_service->session()->captureSession(); + float rotation = 0; + if (!audioOnly) { + AVFCamera *cameraControl = m_service->avfCameraControl(); if (!cameraControl || cameraControl->status() != QCamera::ActiveStatus) { qDebugCamera() << Q_FUNC_INFO << "can not start record while camera is not active"; Q_EMIT error(QMediaEncoder::ResourceError, tr("Failed to start recording")); return; } - const QString path(m_outputLocation.scheme() == QLatin1String("file") ? - m_outputLocation.path() : m_outputLocation.toString()); - auto encoderSettings = m_settings; - encoderSettings.resolveFormat(); - const QUrl fileURL(QUrl::fromLocalFile(m_storageLocation.generateFileName(path, AVFStorageLocation::Video, - QLatin1String("clip_"), - encoderSettings.mimeType().preferredSuffix()))); - - NSURL *nsFileURL = fileURL.toNSURL(); - if (!nsFileURL) { - qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL; - Q_EMIT error(QMediaEncoder::ResourceError, tr("Invalid output file URL")); - return; - } - if (!qt_is_writable_file_URL(nsFileURL)) { - qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL - << "(the location is not writable)"; - Q_EMIT error(QMediaEncoder::ResourceError, tr("Non-writeable file location")); - return; - } - if (qt_file_exists(nsFileURL)) { - // We test for/handle this error here since AWAssetWriter will raise an - // Objective-C exception, which is not good at all. - qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL - << "(file already exists)"; - Q_EMIT error(QMediaEncoder::ResourceError, tr("File already exists")); - return; - } - - AVCaptureSession *session = m_service->session()->captureSession(); - // We stop session now so that no more frames for renderer's queue - // generated, will restart in assetWriterStarted. - [session stopRunning]; - - applySettings(); - // Make sure the video is recorded in device orientation. // The top of the video will match the side of the device which is on top // when recording starts (regardless of the UI orientation). - QCameraInfo cameraInfo = m_service->session()->activeCameraInfo(); - int screenOrientation = 360 - m_orientationHandler.currentOrientation(); - float rotation = 0; - // ### -// if (cameraInfo.position() == QCameraInfo::FrontFace) -// rotation = (screenOrientation + cameraInfo.orientation()) % 360; -// else -// rotation = (screenOrientation + (360 - cameraInfo.orientation())) % 360; - - if ([m_writer setupWithFileURL:nsFileURL - cameraService:m_service - audioSettings:m_audioSettings - videoSettings:m_videoSettings - transform:CGAffineTransformMakeRotation(qDegreesToRadians(rotation))]) { - - m_state = QMediaEncoder::RecordingState; - m_lastStatus = QMediaEncoder::StartingStatus; - - Q_EMIT actualLocationChanged(fileURL); - Q_EMIT stateChanged(m_state); - Q_EMIT statusChanged(m_lastStatus); - - // Apple recommends to call startRunning and do all - // setup on a special queue, and that's what we had - // initially (dispatch_async to writerQueue). Unfortunately, - // writer's queue is not the only queue/thread that can - // access/modify the session, and as a result we have - // all possible data/race-conditions with Obj-C exceptions - // at best and something worse in general. - // Now we try to only modify session on the same thread. - [m_writer start]; - } else { - [session startRunning]; - Q_EMIT error(QMediaEncoder::FormatError, tr("Failed to start recording")); - } - } break; - case QMediaEncoder::PausedState: - { - Q_EMIT error(QMediaEncoder::FormatError, tr("Recording pause not supported")); + // QCameraInfo cameraInfo = m_service->session()->activeCameraInfo(); + // int screenOrientation = 360 - m_orientationHandler.currentOrientation(); + + // ### + // if (cameraInfo.position() == QCameraInfo::FrontFace) + // rotation = (screenOrientation + cameraInfo.orientation()) % 360; + // else + // rotation = (screenOrientation + (360 - cameraInfo.orientation())) % 360; + } + + const QString path(m_outputLocation.scheme() == QLatin1String("file") ? + m_outputLocation.path() : m_outputLocation.toString()); + const QUrl fileURL(QUrl::fromLocalFile(m_storageLocation.generateFileName(path, + audioOnly ? AVFStorageLocation::Audio : AVFStorageLocation::Video, + QLatin1String("clip_"), + encoderSettings().mimeType().preferredSuffix()))); + + NSURL *nsFileURL = fileURL.toNSURL(); + if (!nsFileURL) { + qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL; + Q_EMIT error(QMediaEncoder::ResourceError, tr("Invalid output file URL")); return; - } break; - case QMediaEncoder::StoppedState: - { - // Do not check the camera status, we can stop if we started. - stopWriter(); } + if (!qt_is_writable_file_URL(nsFileURL)) { + qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL + << "(the location is not writable)"; + Q_EMIT error(QMediaEncoder::ResourceError, tr("Non-writeable file location")); + return; + } + if (qt_file_exists(nsFileURL)) { + // We test for/handle this error here since AWAssetWriter will raise an + // Objective-C exception, which is not good at all. + qWarning() << Q_FUNC_INFO << "invalid output URL:" << fileURL + << "(file already exists)"; + Q_EMIT error(QMediaEncoder::ResourceError, tr("File already exists")); + return; + } + + applySettings(); + + // We stop session now so that no more frames for renderer's queue + // generated, will restart in assetWriterStarted. + [session stopRunning]; + + if ([m_writer setupWithFileURL:nsFileURL + cameraService:m_service + audioSettings:m_audioSettings + videoSettings:m_videoSettings + transform:CGAffineTransformMakeRotation(qDegreesToRadians(rotation))]) { + + m_state = QMediaEncoder::RecordingState; + m_lastStatus = QMediaEncoder::StartingStatus; + + Q_EMIT actualLocationChanged(fileURL); + Q_EMIT stateChanged(m_state); + Q_EMIT statusChanged(m_lastStatus); + + // Apple recommends to call startRunning and do all + // setup on a special queue, and that's what we had + // initially (dispatch_async to writerQueue). Unfortunately, + // writer's queue is not the only queue/thread that can + // access/modify the session, and as a result we have + // all possible data/race-conditions with Obj-C exceptions + // at best and something worse in general. + // Now we try to only modify session on the same thread. + [m_writer start]; + } else { + [session startRunning]; + Q_EMIT error(QMediaEncoder::FormatError, tr("Failed to start recording")); } } @@ -548,17 +547,18 @@ void AVFMediaEncoder::assetWriterStarted() void AVFMediaEncoder::assetWriterFinished() { - Q_ASSERT(m_service); - AVFCamera *cameraControl = m_service->avfCameraControl(); - Q_ASSERT(cameraControl); + Q_ASSERT(m_service && m_service->session()); + AVFCameraSession *session = m_service->session(); const QMediaEncoder::Status lastStatus = m_lastStatus; const QMediaEncoder::State lastState = m_state; unapplySettings(); - m_service->session()->videoOutput()->resetCaptureDelegate(); - [m_service->session()->captureSession() startRunning]; + if (session->videoOutput()) { + session->videoOutput()->resetCaptureDelegate(); + } + [session->captureSession() startRunning]; m_state = QMediaEncoder::StoppedState; if (m_lastStatus != lastStatus) Q_EMIT statusChanged(m_lastStatus); diff --git a/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h b/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h index dfb21c482..880a561c5 100644 --- a/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h +++ b/src/multimedia/platform/darwin/camera/avfmediaencoder_p.h @@ -112,6 +112,7 @@ private Q_SLOTS: void cameraStatusChanged(QCamera::Status newStatus); private: + void record(); void stopWriter(); AVFCameraService *m_service = nullptr; diff --git a/src/multimedia/platform/darwin/camera/avfstoragelocation.mm b/src/multimedia/platform/darwin/camera/avfstoragelocation.mm index f2c223c82..0d37ffad5 100644 --- a/src/multimedia/platform/darwin/camera/avfstoragelocation.mm +++ b/src/multimedia/platform/darwin/camera/avfstoragelocation.mm @@ -78,8 +78,10 @@ QDir AVFStorageLocation::defaultDir(Mode mode) const if (mode == Video) { dirCandidates << QStandardPaths::writableLocation(QStandardPaths::MoviesLocation); - } else { + } else if (mode == Image) { dirCandidates << QStandardPaths::writableLocation(QStandardPaths::PicturesLocation); + } else { + dirCandidates << QStandardPaths::writableLocation(QStandardPaths::MusicLocation); } dirCandidates << QStandardPaths::writableLocation(QStandardPaths::DocumentsLocation); diff --git a/src/multimedia/platform/darwin/camera/avfstoragelocation_p.h b/src/multimedia/platform/darwin/camera/avfstoragelocation_p.h index 25221f68c..ec0b82426 100644 --- a/src/multimedia/platform/darwin/camera/avfstoragelocation_p.h +++ b/src/multimedia/platform/darwin/camera/avfstoragelocation_p.h @@ -66,7 +66,8 @@ public: enum Mode { Image, - Video + Video, + Audio }; QString generateFileName(const QString &requestedName, diff --git a/src/multimedia/platform/darwin/qdarwinformatsinfo.mm b/src/multimedia/platform/darwin/qdarwinformatsinfo.mm index 167b3d297..82a8b66ae 100644 --- a/src/multimedia/platform/darwin/qdarwinformatsinfo.mm +++ b/src/multimedia/platform/darwin/qdarwinformatsinfo.mm @@ -103,13 +103,11 @@ QDarwinFormatInfo::QDarwinFormatInfo() QList video; QList audio; -// qDebug() << "Media container" << m->name << "supported"; auto *v = videoCodecMap; while (v->name) { QByteArray extendedMimetype = m->name; extendedMimetype += v->name; -// qDebug() << "video" << extendedMimetype << [AVURLAsset isPlayableExtendedMIMEType:[NSString stringWithUTF8String:extendedMimetype.constData()]]; if ([AVURLAsset isPlayableExtendedMIMEType:[NSString stringWithUTF8String:extendedMimetype.constData()]]) video << v->value; ++v; @@ -135,10 +133,10 @@ QDarwinFormatInfo::QDarwinFormatInfo() } } -#if 1 - // ### Verify that this is correct - encoders = decoders; -#else +// #if 1 +// // ### Verify that this is correct +// encoders = decoders; +// #else // ### Haven't seen a good way to figure this out. // seems AVFoundation only supports those for encoding encoders = { @@ -148,20 +146,21 @@ QDarwinFormatInfo::QDarwinFormatInfo() { QMediaFormat::QuickTime, { QMediaFormat::AudioCodec::AAC, QMediaFormat::AudioCodec::MP3, QMediaFormat::AudioCodec::ALAC, QMediaFormat::AudioCodec::AC3, QMediaFormat::AudioCodec::EAC3, }, { QMediaFormat::VideoCodec::H264, QMediaFormat::VideoCodec::H265, QMediaFormat::VideoCodec::MotionJPEG } }, - { QMediaFormat::AAC, - { QMediaFormat::AudioCodec::AAC }, - {} }, - { QMediaFormat::MP3, - { QMediaFormat::AudioCodec::MP3 }, - {} }, - { QMediaFormat::FLAC, - { QMediaFormat::AudioCodec::FLAC }, - {} }, + // seems AVFoundation does not support directly encoding to an AAC and MP3 file + // { QMediaFormat::AAC, + // { QMediaFormat::AudioCodec::AAC }, + // {} }, + // { QMediaFormat::MP3, + // { QMediaFormat::AudioCodec::MP3 }, + // {} }, + // { QMediaFormat::FLAC, + // { QMediaFormat::AudioCodec::FLAC }, + // {} }, { QMediaFormat::Mpeg4Audio, { QMediaFormat::AudioCodec::AAC }, - {} } + {} }, }; -#endif +// #endif // ### imageFormats << QImageEncoderSettings::JPEG; -- cgit v1.2.3