From 8e4d966f4e5546787257de046d2c3af8a19214b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christian=20Str=C3=B8mme?= Date: Thu, 20 Oct 2016 18:32:48 +0200 Subject: DirectShow: Enable audio and video probes in DirectShow This change enables the video and audio probe functionality, so it can be used with the media player. [ChangeLog][DirectShow] Added support for audio and video probes in the mediaplayer. Task-number: QTBUG-56415 Change-Id: If6f36a693b1d22372eab130df07d435c9df5a796 Reviewed-by: Yoann Lopes --- .../directshow/player/directshowplayerservice.cpp | 340 ++++++++++++++++++++- .../directshow/player/directshowplayerservice.h | 56 +++- src/plugins/directshow/player/player.pri | 2 + 3 files changed, 381 insertions(+), 17 deletions(-) diff --git a/src/plugins/directshow/player/directshowplayerservice.cpp b/src/plugins/directshow/player/directshowplayerservice.cpp index 4a75c9551..de427f781 100644 --- a/src/plugins/directshow/player/directshowplayerservice.cpp +++ b/src/plugins/directshow/player/directshowplayerservice.cpp @@ -53,7 +53,11 @@ #include "directshowiosource.h" #include "directshowplayercontrol.h" #include "directshowvideorenderercontrol.h" - +#include "directshowutils.h" +#include "directshowglobal.h" +#include "directshowaudioprobecontrol.h" +#include "directshowvideoprobecontrol.h" +#include "directshowsamplegrabber.h" #if QT_CONFIG(evr) #include "directshowevrvideowindowcontrol.h" @@ -68,6 +72,11 @@ #include #include #include +#include + +#include +#include +#include #if QT_CONFIG(wmsdk) # include @@ -117,6 +126,10 @@ DirectShowPlayerService::DirectShowPlayerService(QObject *parent) , m_videoRendererControl(0) , m_videoWindowControl(0) , m_audioEndpointControl(0) + , m_audioProbeControl(nullptr) + , m_videoProbeControl(nullptr) + , m_audioSampleGrabber(nullptr) + , m_videoSampleGrabber(nullptr) , m_taskThread(0) , m_loop(qt_directShowEventLoop()) , m_pendingTasks(0) @@ -176,6 +189,8 @@ DirectShowPlayerService::~DirectShowPlayerService() delete m_metaDataControl; delete m_videoRendererControl; delete m_videoWindowControl; + delete m_audioProbeControl; + delete m_videoProbeControl; ::CloseHandle(m_taskHandle); } @@ -219,6 +234,18 @@ QMediaControl *DirectShowPlayerService::requestControl(const char *name) return m_videoWindowControl; } + } else if (qstrcmp(name, QMediaAudioProbeControl_iid) == 0) { + if (!m_audioProbeControl) + m_audioProbeControl = new DirectShowAudioProbeControl(); + m_audioProbeControl->ref(); + updateAudioProbe(); + return m_audioProbeControl; + } else if (qstrcmp(name, QMediaVideoProbeControl_iid) == 0) { + if (!m_videoProbeControl) + m_videoProbeControl = new DirectShowVideoProbeControl(); + m_videoProbeControl->ref(); + updateVideoProbe(); + return m_videoProbeControl; } return 0; } @@ -240,6 +267,20 @@ void DirectShowPlayerService::releaseControl(QMediaControl *control) delete m_videoWindowControl; m_videoWindowControl = 0; + } else if (control == m_audioProbeControl) { + if (!m_audioProbeControl->deref()) { + DirectShowAudioProbeControl *old = m_audioProbeControl; + m_audioProbeControl = nullptr; + updateAudioProbe(); + delete old; + } + } else if (control == m_videoProbeControl) { + if (!m_videoProbeControl->deref()) { + DirectShowVideoProbeControl *old = m_videoProbeControl; + m_videoProbeControl = nullptr; + updateVideoProbe(); + delete old; + } } } @@ -349,6 +390,10 @@ void DirectShowPlayerService::doSetUrlSource(QMutexLocker *locker) m_pendingTasks |= SetAudioOutput; if (m_videoOutput) m_pendingTasks |= SetVideoOutput; + if (m_audioProbeControl) + m_pendingTasks |= SetAudioProbe; + if (m_videoProbeControl) + m_pendingTasks |= SetVideoProbe; if (m_rate != 1.0) m_pendingTasks |= SetRate; @@ -434,6 +479,18 @@ void DirectShowPlayerService::doRender(QMutexLocker *locker) m_executedTasks |= SetVideoOutput; } + if (m_pendingTasks & SetAudioProbe) { + doSetAudioProbe(locker); + m_pendingTasks ^= SetAudioProbe; + m_executedTasks |= SetAudioProbe; + } + + if (m_pendingTasks & SetVideoProbe) { + doSetVideoProbe(locker); + m_pendingTasks ^= SetVideoProbe; + m_executedTasks |= SetVideoProbe; + } + IFilterGraph2 *graph = m_graph; graph->AddRef(); @@ -601,6 +658,9 @@ void DirectShowPlayerService::doReleaseGraph(QMutexLocker *locker) control->Release(); } + doReleaseAudioProbe(locker); + doReleaseVideoProbe(locker); + if (m_source) { m_source->Release(); m_source = 0; @@ -614,6 +674,139 @@ void DirectShowPlayerService::doReleaseGraph(QMutexLocker *locker) m_loop->wake(); } +void DirectShowPlayerService::doSetVideoProbe(QMutexLocker *locker) +{ + Q_UNUSED(locker); + + if (!m_graph) { + qCWarning(qtDirectShowPlugin, "Attempting to set a video probe without a valid graph!"); + return; + } + + // Create the sample grabber, if necessary. + if (!m_videoSampleGrabber) { + m_videoSampleGrabber = new DirectShowSampleGrabber; + connect(m_videoSampleGrabber, &DirectShowSampleGrabber::bufferAvailable, this, &DirectShowPlayerService::onVideoBufferAvailable); + } + + if (FAILED(m_graph->AddFilter(m_videoSampleGrabber->filter(), L"Video Sample Grabber"))) { + qCWarning(qtDirectShowPlugin, "Failed to add the video sample grabber into the graph!"); + return; + } + + // TODO: Make util function for getting this, so it's easy to keep it in sync. + static const GUID subtypes[] = { MEDIASUBTYPE_ARGB32, + MEDIASUBTYPE_RGB32, + MEDIASUBTYPE_RGB24, + MEDIASUBTYPE_RGB565, + MEDIASUBTYPE_RGB555, + MEDIASUBTYPE_AYUV, + MEDIASUBTYPE_I420, + MEDIASUBTYPE_IYUV, + MEDIASUBTYPE_YV12, + MEDIASUBTYPE_UYVY, + MEDIASUBTYPE_YUYV, + MEDIASUBTYPE_YUY2, + MEDIASUBTYPE_NV12, + MEDIASUBTYPE_MJPG, + MEDIASUBTYPE_IMC1, + MEDIASUBTYPE_IMC2, + MEDIASUBTYPE_IMC3, + MEDIASUBTYPE_IMC4 }; + + // Negotiate the subtype + DirectShowMediaType mediaType(AM_MEDIA_TYPE { MEDIATYPE_Video }); + const int items = (sizeof subtypes / sizeof(GUID)); + bool connected = false; + for (int i = 0; i != items; ++i) { + mediaType->subtype = subtypes[i]; + m_videoSampleGrabber->setMediaType(&mediaType); + if (SUCCEEDED(DirectShowUtils::connectFilters(m_graph, m_source, m_videoSampleGrabber->filter(), true))) + connected = true; + break; + } + + if (!connected) { + qCWarning(qtDirectShowPlugin, "Unable to connect the video probe!"); + return; + } + + m_videoSampleGrabber->start(DirectShowSampleGrabber::CallbackMethod::BufferCB); +} + +void DirectShowPlayerService::doSetAudioProbe(QMutexLocker *locker) +{ + Q_UNUSED(locker); + + if (!m_graph) { + qCWarning(qtDirectShowPlugin, "Attempting to set an audio probe without a valid graph!"); + return; + } + + // Create the sample grabber, if necessary. + if (!m_audioSampleGrabber) { + m_audioSampleGrabber = new DirectShowSampleGrabber; + connect(m_audioSampleGrabber, &DirectShowSampleGrabber::bufferAvailable, this, &DirectShowPlayerService::onAudioBufferAvailable); + } + + static const AM_MEDIA_TYPE mediaType { MEDIATYPE_Audio, MEDIASUBTYPE_PCM }; + m_audioSampleGrabber->setMediaType(&mediaType); + + if (FAILED(m_graph->AddFilter(m_audioSampleGrabber->filter(), L"Audio Sample Grabber"))) { + qCWarning(qtDirectShowPlugin, "Failed to add the audio sample grabber into the graph!"); + return; + } + + if (FAILED(DirectShowUtils::connectFilters(m_graph, m_source, m_audioSampleGrabber->filter(), true))) { + qCWarning(qtDirectShowPlugin, "Failed to connect the audio sample grabber"); + return; + } + + m_audioSampleGrabber->start(DirectShowSampleGrabber::CallbackMethod::BufferCB); +} + +void DirectShowPlayerService::doReleaseVideoProbe(QMutexLocker *locker) +{ + Q_UNUSED(locker); + + if (!m_graph) + return; + + if (!m_videoSampleGrabber) + return; + + m_videoSampleGrabber->stop(); + HRESULT hr = m_graph->RemoveFilter(m_videoSampleGrabber->filter()); + if (FAILED(hr)) { + qCWarning(qtDirectShowPlugin, "Failed to remove the video sample grabber!"); + return; + } + + m_videoSampleGrabber->deleteLater(); + m_videoSampleGrabber = nullptr; +} + +void DirectShowPlayerService::doReleaseAudioProbe(QMutexLocker *locker) +{ + Q_UNUSED(locker); + + if (!m_graph) + return; + + if (!m_audioSampleGrabber) + return; + + m_audioSampleGrabber->stop(); + HRESULT hr = m_graph->RemoveFilter(m_audioSampleGrabber->filter()); + if (FAILED(hr)) { + qCWarning(qtDirectShowPlugin, "Failed to remove the audio sample grabber!"); + return; + } + + m_audioSampleGrabber->deleteLater(); + m_audioSampleGrabber = nullptr; +} + int DirectShowPlayerService::findStreamTypes(IBaseFilter *source) const { QVarLengthArray filters; @@ -1121,6 +1314,40 @@ void DirectShowPlayerService::setVideoOutput(IBaseFilter *filter) } } +void DirectShowPlayerService::updateAudioProbe() +{ + QMutexLocker locker(&m_mutex); + + // Set/Activate the audio probe. + if (m_graph) { + // If we don't have a audio probe, then stop and release the audio sample grabber + if (!m_audioProbeControl && (m_executedTasks & SetAudioProbe)) { + m_pendingTasks |= ReleaseAudioProbe; + ::SetEvent(m_taskHandle); + m_loop->wait(&m_mutex); + } else if (m_audioProbeControl) { + m_pendingTasks |= SetAudioProbe; + } + } +} + +void DirectShowPlayerService::updateVideoProbe() +{ + QMutexLocker locker(&m_mutex); + + // Set/Activate the video probe. + if (m_graph) { + // If we don't have a video probe, then stop and release the video sample grabber + if (!m_videoProbeControl && (m_executedTasks & SetVideoProbe)) { + m_pendingTasks |= ReleaseVideoProbe; + ::SetEvent(m_taskHandle); + m_loop->wait(&m_mutex); + } else if (m_videoProbeControl){ + m_pendingTasks |= SetVideoProbe; + } + } +} + void DirectShowPlayerService::doReleaseVideoOutput(QMutexLocker *locker) { Q_UNUSED(locker) @@ -1217,6 +1444,97 @@ void DirectShowPlayerService::videoOutputChanged() setVideoOutput(m_videoRendererControl->filter()); } +void DirectShowPlayerService::onAudioBufferAvailable(double time, quint8 *buffer, long len) +{ + QMutexLocker locker(&m_mutex); + if (!m_audioProbeControl || !m_audioSampleGrabber) + return; + + DirectShowMediaType mt(AM_MEDIA_TYPE { GUID_NULL }); + const bool ok = m_audioSampleGrabber->getConnectedMediaType(&mt); + if (!ok) + return; + + if (mt->majortype != MEDIATYPE_Audio) + return; + + if (mt->subtype != MEDIASUBTYPE_PCM) + return; + + const bool isWfx = ((mt->formattype == FORMAT_WaveFormatEx) && (mt->cbFormat >= sizeof(WAVEFORMATEX))); + WAVEFORMATEX *wfx = isWfx ? reinterpret_cast(mt->pbFormat) : nullptr; + + if (!wfx) + return; + + if (wfx->wFormatTag != WAVE_FORMAT_PCM && wfx->wFormatTag != WAVE_FORMAT_EXTENSIBLE) + return; + + if ((wfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE) && (wfx->cbSize >= sizeof(WAVEFORMATEXTENSIBLE))) { + WAVEFORMATEXTENSIBLE *wfxe = reinterpret_cast(wfx); + if (wfxe->SubFormat != KSDATAFORMAT_SUBTYPE_PCM) + return; + } + + QAudioFormat format; + format.setSampleRate(wfx->nSamplesPerSec); + format.setChannelCount(wfx->nChannels); + format.setSampleSize(wfx->wBitsPerSample); + format.setCodec("audio/pcm"); + format.setByteOrder(QAudioFormat::LittleEndian); + if (format.sampleSize() == 8) + format.setSampleType(QAudioFormat::UnSignedInt); + else + format.setSampleType(QAudioFormat::SignedInt); + + const quint64 startTime = quint64(time * 1000.); + QAudioBuffer audioBuffer(QByteArray(reinterpret_cast(buffer), len), + format, + startTime); + + Q_EMIT m_audioProbeControl->audioBufferProbed(audioBuffer); +} + +void DirectShowPlayerService::onVideoBufferAvailable(double time, quint8 *buffer, long len) +{ + Q_UNUSED(time); + Q_UNUSED(buffer); + Q_UNUSED(len); + + QMutexLocker locker(&m_mutex); + if (!m_videoProbeControl || !m_videoSampleGrabber) + return; + + DirectShowMediaType mt(AM_MEDIA_TYPE { GUID_NULL }); + const bool ok = m_videoSampleGrabber->getConnectedMediaType(&mt); + if (!ok) + return; + + if (mt->majortype != MEDIATYPE_Video) + return; + + QVideoFrame::PixelFormat format = DirectShowMediaType::pixelFormatFromType(&mt); + if (format == QVideoFrame::Format_Invalid) { + qCWarning(qtDirectShowPlugin, "Invalid format, stopping video probes!"); + m_videoSampleGrabber->stop(); + return; + } + + const QVideoSurfaceFormat &videoFormat = DirectShowMediaType::videoFormatFromType(&mt); + if (!videoFormat.isValid()) + return; + + const QSize &size = videoFormat.frameSize(); + + const int bytesPerLine = DirectShowMediaType::bytesPerLine(videoFormat); + QByteArray data(reinterpret_cast(buffer), len); + QVideoFrame frame(new QMemoryVideoBuffer(data, bytesPerLine), + size, + format); + + Q_EMIT m_videoProbeControl->videoFrameProbed(frame); +} + void DirectShowPlayerService::graphEvent(QMutexLocker *locker) { Q_UNUSED(locker) @@ -1405,6 +1723,16 @@ void DirectShowPlayerService::run() m_executingTask = ReleaseVideoOutput; doReleaseVideoOutput(&locker); + } else if (m_pendingTasks & ReleaseAudioProbe) { + m_pendingTasks ^= ReleaseAudioProbe; + m_executingTask = ReleaseAudioProbe; + + doReleaseAudioProbe(&locker); + } else if (m_pendingTasks & ReleaseVideoProbe) { + m_pendingTasks ^= ReleaseVideoProbe; + m_executingTask = ReleaseVideoProbe; + + doReleaseVideoProbe(&locker); } else if (m_pendingTasks & SetUrlSource) { m_pendingTasks ^= SetUrlSource; m_executingTask = SetUrlSource; @@ -1415,6 +1743,16 @@ void DirectShowPlayerService::run() m_executingTask = SetStreamSource; doSetStreamSource(&locker); + } else if (m_pendingTasks & SetAudioProbe) { + m_pendingTasks ^= SetAudioProbe; + m_executingTask = SetAudioProbe; + + doSetAudioProbe(&locker); + } else if (m_pendingTasks & SetVideoProbe) { + m_pendingTasks ^= SetVideoProbe; + m_executingTask = SetVideoProbe; + + doSetVideoProbe(&locker); } else if (m_pendingTasks & Render) { m_pendingTasks ^= Render; m_executingTask = Render; diff --git a/src/plugins/directshow/player/directshowplayerservice.h b/src/plugins/directshow/player/directshowplayerservice.h index dc0226001..676d88fb5 100644 --- a/src/plugins/directshow/player/directshowplayerservice.h +++ b/src/plugins/directshow/player/directshowplayerservice.h @@ -61,6 +61,9 @@ class DirectShowAudioEndpointControl; class DirectShowMetaDataControl; class DirectShowPlayerControl; class DirectShowVideoRendererControl; +class DirectShowAudioProbeControl; +class DirectShowVideoProbeControl; +class DirectShowSampleGrabber; class QMediaContent; class QVideoWindowControl; @@ -103,10 +106,16 @@ protected: private Q_SLOTS: void videoOutputChanged(); + void onAudioBufferAvailable(double time, quint8 *buffer, long len); + void onVideoBufferAvailable(double time, quint8 *buffer, long len); + private: void releaseGraph(); void updateStatus(); + void updateAudioProbe(); + void updateVideoProbe(); + int findStreamTypes(IBaseFilter *source) const; int findStreamType(IPin *pin) const; @@ -127,29 +136,40 @@ private: void doReleaseAudioOutput(QMutexLocker *locker); void doReleaseVideoOutput(QMutexLocker *locker); void doReleaseGraph(QMutexLocker *locker); + void doSetVideoProbe(QMutexLocker *locker); + void doSetAudioProbe(QMutexLocker *locker); + void doReleaseVideoProbe(QMutexLocker *locker); + void doReleaseAudioProbe(QMutexLocker *locker); void graphEvent(QMutexLocker *locker); enum Task { - Shutdown = 0x0001, - SetUrlSource = 0x0002, - SetStreamSource = 0x0004, + Shutdown = 0x00001, + SetUrlSource = 0x00002, + SetStreamSource = 0x00004, SetSource = SetUrlSource | SetStreamSource, - SetAudioOutput = 0x0008, - SetVideoOutput = 0x0010, + SetAudioOutput = 0x00008, + SetVideoOutput = 0x00010, SetOutputs = SetAudioOutput | SetVideoOutput, - Render = 0x0020, - FinalizeLoad = 0x0040, - SetRate = 0x0080, - Seek = 0x0100, - Play = 0x0200, - Pause = 0x0400, - Stop = 0x0800, - ReleaseGraph = 0x1000, - ReleaseAudioOutput = 0x2000, - ReleaseVideoOutput = 0x4000, - ReleaseFilters = ReleaseGraph | ReleaseAudioOutput | ReleaseVideoOutput + SetAudioProbe = 0x00020, + SetVideoProbe = 0x00040, + SetProbes = SetAudioProbe | SetVideoProbe, + Render = 0x00080, + FinalizeLoad = 0x00100, + SetRate = 0x00200, + Seek = 0x00400, + Play = 0x00800, + Pause = 0x01000, + Stop = 0x02000, + ReleaseGraph = 0x04000, + ReleaseAudioOutput = 0x08000, + ReleaseVideoOutput = 0x10000, + ReleaseAudioProbe = 0x20000, + ReleaseVideoProbe = 0x40000, + ReleaseFilters = ReleaseGraph | ReleaseAudioOutput + | ReleaseVideoOutput | ReleaseAudioProbe + | ReleaseVideoProbe }; enum Event @@ -178,6 +198,10 @@ private: DirectShowVideoRendererControl *m_videoRendererControl; QVideoWindowControl *m_videoWindowControl; DirectShowAudioEndpointControl *m_audioEndpointControl; + DirectShowAudioProbeControl *m_audioProbeControl; + DirectShowVideoProbeControl *m_videoProbeControl; + DirectShowSampleGrabber *m_audioSampleGrabber; + DirectShowSampleGrabber *m_videoSampleGrabber; QThread *m_taskThread; DirectShowEventLoop *m_loop; diff --git a/src/plugins/directshow/player/player.pri b/src/plugins/directshow/player/player.pri index 59ac5a330..9111cc545 100644 --- a/src/plugins/directshow/player/player.pri +++ b/src/plugins/directshow/player/player.pri @@ -3,6 +3,8 @@ INCLUDEPATH += $$PWD QMAKE_USE += directshow LIBS += -lgdi32 +mingw: LIBS_PRIVATE += -lksuser + qtHaveModule(widgets): QT += widgets HEADERS += \ -- cgit v1.2.3