summaryrefslogtreecommitdiffstats
path: root/src
diff options
context:
space:
mode:
authorChristian Strømme <christian.stromme@qt.io>2016-10-20 18:32:48 +0200
committerChristian Stromme <christian.stromme@qt.io>2017-05-03 13:29:24 +0000
commit8e4d966f4e5546787257de046d2c3af8a19214b4 (patch)
treef1d4506eca47ccdc561540c3d1de9767b55baf6f /src
parent03368d61c2cc809a2a3ba209694b71bb12d1c894 (diff)
DirectShow: Enable audio and video probes in DirectShow
This change enables the video and audio probe functionality, so it can be used with the media player. [ChangeLog][DirectShow] Added support for audio and video probes in the mediaplayer. Task-number: QTBUG-56415 Change-Id: If6f36a693b1d22372eab130df07d435c9df5a796 Reviewed-by: Yoann Lopes <yoann.lopes@qt.io>
Diffstat (limited to 'src')
-rw-r--r--src/plugins/directshow/player/directshowplayerservice.cpp340
-rw-r--r--src/plugins/directshow/player/directshowplayerservice.h56
-rw-r--r--src/plugins/directshow/player/player.pri2
3 files changed, 381 insertions, 17 deletions
diff --git a/src/plugins/directshow/player/directshowplayerservice.cpp b/src/plugins/directshow/player/directshowplayerservice.cpp
index 4a75c955..de427f78 100644
--- a/src/plugins/directshow/player/directshowplayerservice.cpp
+++ b/src/plugins/directshow/player/directshowplayerservice.cpp
@@ -53,7 +53,11 @@
#include "directshowiosource.h"
#include "directshowplayercontrol.h"
#include "directshowvideorenderercontrol.h"
-
+#include "directshowutils.h"
+#include "directshowglobal.h"
+#include "directshowaudioprobecontrol.h"
+#include "directshowvideoprobecontrol.h"
+#include "directshowsamplegrabber.h"
#if QT_CONFIG(evr)
#include "directshowevrvideowindowcontrol.h"
@@ -68,6 +72,11 @@
#include <QtCore/qdir.h>
#include <QtCore/qthread.h>
#include <QtCore/qvarlengtharray.h>
+#include <QtCore/qsize.h>
+
+#include <QtMultimedia/qaudiobuffer.h>
+#include <QtMultimedia/qvideoframe.h>
+#include <QtMultimedia/private/qmemoryvideobuffer_p.h>
#if QT_CONFIG(wmsdk)
# include <wmsdk.h>
@@ -117,6 +126,10 @@ DirectShowPlayerService::DirectShowPlayerService(QObject *parent)
, m_videoRendererControl(0)
, m_videoWindowControl(0)
, m_audioEndpointControl(0)
+ , m_audioProbeControl(nullptr)
+ , m_videoProbeControl(nullptr)
+ , m_audioSampleGrabber(nullptr)
+ , m_videoSampleGrabber(nullptr)
, m_taskThread(0)
, m_loop(qt_directShowEventLoop())
, m_pendingTasks(0)
@@ -176,6 +189,8 @@ DirectShowPlayerService::~DirectShowPlayerService()
delete m_metaDataControl;
delete m_videoRendererControl;
delete m_videoWindowControl;
+ delete m_audioProbeControl;
+ delete m_videoProbeControl;
::CloseHandle(m_taskHandle);
}
@@ -219,6 +234,18 @@ QMediaControl *DirectShowPlayerService::requestControl(const char *name)
return m_videoWindowControl;
}
+ } else if (qstrcmp(name, QMediaAudioProbeControl_iid) == 0) {
+ if (!m_audioProbeControl)
+ m_audioProbeControl = new DirectShowAudioProbeControl();
+ m_audioProbeControl->ref();
+ updateAudioProbe();
+ return m_audioProbeControl;
+ } else if (qstrcmp(name, QMediaVideoProbeControl_iid) == 0) {
+ if (!m_videoProbeControl)
+ m_videoProbeControl = new DirectShowVideoProbeControl();
+ m_videoProbeControl->ref();
+ updateVideoProbe();
+ return m_videoProbeControl;
}
return 0;
}
@@ -240,6 +267,20 @@ void DirectShowPlayerService::releaseControl(QMediaControl *control)
delete m_videoWindowControl;
m_videoWindowControl = 0;
+ } else if (control == m_audioProbeControl) {
+ if (!m_audioProbeControl->deref()) {
+ DirectShowAudioProbeControl *old = m_audioProbeControl;
+ m_audioProbeControl = nullptr;
+ updateAudioProbe();
+ delete old;
+ }
+ } else if (control == m_videoProbeControl) {
+ if (!m_videoProbeControl->deref()) {
+ DirectShowVideoProbeControl *old = m_videoProbeControl;
+ m_videoProbeControl = nullptr;
+ updateVideoProbe();
+ delete old;
+ }
}
}
@@ -349,6 +390,10 @@ void DirectShowPlayerService::doSetUrlSource(QMutexLocker *locker)
m_pendingTasks |= SetAudioOutput;
if (m_videoOutput)
m_pendingTasks |= SetVideoOutput;
+ if (m_audioProbeControl)
+ m_pendingTasks |= SetAudioProbe;
+ if (m_videoProbeControl)
+ m_pendingTasks |= SetVideoProbe;
if (m_rate != 1.0)
m_pendingTasks |= SetRate;
@@ -434,6 +479,18 @@ void DirectShowPlayerService::doRender(QMutexLocker *locker)
m_executedTasks |= SetVideoOutput;
}
+ if (m_pendingTasks & SetAudioProbe) {
+ doSetAudioProbe(locker);
+ m_pendingTasks ^= SetAudioProbe;
+ m_executedTasks |= SetAudioProbe;
+ }
+
+ if (m_pendingTasks & SetVideoProbe) {
+ doSetVideoProbe(locker);
+ m_pendingTasks ^= SetVideoProbe;
+ m_executedTasks |= SetVideoProbe;
+ }
+
IFilterGraph2 *graph = m_graph;
graph->AddRef();
@@ -601,6 +658,9 @@ void DirectShowPlayerService::doReleaseGraph(QMutexLocker *locker)
control->Release();
}
+ doReleaseAudioProbe(locker);
+ doReleaseVideoProbe(locker);
+
if (m_source) {
m_source->Release();
m_source = 0;
@@ -614,6 +674,139 @@ void DirectShowPlayerService::doReleaseGraph(QMutexLocker *locker)
m_loop->wake();
}
+void DirectShowPlayerService::doSetVideoProbe(QMutexLocker *locker)
+{
+ Q_UNUSED(locker);
+
+ if (!m_graph) {
+ qCWarning(qtDirectShowPlugin, "Attempting to set a video probe without a valid graph!");
+ return;
+ }
+
+ // Create the sample grabber, if necessary.
+ if (!m_videoSampleGrabber) {
+ m_videoSampleGrabber = new DirectShowSampleGrabber;
+ connect(m_videoSampleGrabber, &DirectShowSampleGrabber::bufferAvailable, this, &DirectShowPlayerService::onVideoBufferAvailable);
+ }
+
+ if (FAILED(m_graph->AddFilter(m_videoSampleGrabber->filter(), L"Video Sample Grabber"))) {
+ qCWarning(qtDirectShowPlugin, "Failed to add the video sample grabber into the graph!");
+ return;
+ }
+
+ // TODO: Make util function for getting this, so it's easy to keep it in sync.
+ static const GUID subtypes[] = { MEDIASUBTYPE_ARGB32,
+ MEDIASUBTYPE_RGB32,
+ MEDIASUBTYPE_RGB24,
+ MEDIASUBTYPE_RGB565,
+ MEDIASUBTYPE_RGB555,
+ MEDIASUBTYPE_AYUV,
+ MEDIASUBTYPE_I420,
+ MEDIASUBTYPE_IYUV,
+ MEDIASUBTYPE_YV12,
+ MEDIASUBTYPE_UYVY,
+ MEDIASUBTYPE_YUYV,
+ MEDIASUBTYPE_YUY2,
+ MEDIASUBTYPE_NV12,
+ MEDIASUBTYPE_MJPG,
+ MEDIASUBTYPE_IMC1,
+ MEDIASUBTYPE_IMC2,
+ MEDIASUBTYPE_IMC3,
+ MEDIASUBTYPE_IMC4 };
+
+ // Negotiate the subtype
+ DirectShowMediaType mediaType(AM_MEDIA_TYPE { MEDIATYPE_Video });
+ const int items = (sizeof subtypes / sizeof(GUID));
+ bool connected = false;
+ for (int i = 0; i != items; ++i) {
+ mediaType->subtype = subtypes[i];
+ m_videoSampleGrabber->setMediaType(&mediaType);
+ if (SUCCEEDED(DirectShowUtils::connectFilters(m_graph, m_source, m_videoSampleGrabber->filter(), true)))
+ connected = true;
+ break;
+ }
+
+ if (!connected) {
+ qCWarning(qtDirectShowPlugin, "Unable to connect the video probe!");
+ return;
+ }
+
+ m_videoSampleGrabber->start(DirectShowSampleGrabber::CallbackMethod::BufferCB);
+}
+
+void DirectShowPlayerService::doSetAudioProbe(QMutexLocker *locker)
+{
+ Q_UNUSED(locker);
+
+ if (!m_graph) {
+ qCWarning(qtDirectShowPlugin, "Attempting to set an audio probe without a valid graph!");
+ return;
+ }
+
+ // Create the sample grabber, if necessary.
+ if (!m_audioSampleGrabber) {
+ m_audioSampleGrabber = new DirectShowSampleGrabber;
+ connect(m_audioSampleGrabber, &DirectShowSampleGrabber::bufferAvailable, this, &DirectShowPlayerService::onAudioBufferAvailable);
+ }
+
+ static const AM_MEDIA_TYPE mediaType { MEDIATYPE_Audio, MEDIASUBTYPE_PCM };
+ m_audioSampleGrabber->setMediaType(&mediaType);
+
+ if (FAILED(m_graph->AddFilter(m_audioSampleGrabber->filter(), L"Audio Sample Grabber"))) {
+ qCWarning(qtDirectShowPlugin, "Failed to add the audio sample grabber into the graph!");
+ return;
+ }
+
+ if (FAILED(DirectShowUtils::connectFilters(m_graph, m_source, m_audioSampleGrabber->filter(), true))) {
+ qCWarning(qtDirectShowPlugin, "Failed to connect the audio sample grabber");
+ return;
+ }
+
+ m_audioSampleGrabber->start(DirectShowSampleGrabber::CallbackMethod::BufferCB);
+}
+
+void DirectShowPlayerService::doReleaseVideoProbe(QMutexLocker *locker)
+{
+ Q_UNUSED(locker);
+
+ if (!m_graph)
+ return;
+
+ if (!m_videoSampleGrabber)
+ return;
+
+ m_videoSampleGrabber->stop();
+ HRESULT hr = m_graph->RemoveFilter(m_videoSampleGrabber->filter());
+ if (FAILED(hr)) {
+ qCWarning(qtDirectShowPlugin, "Failed to remove the video sample grabber!");
+ return;
+ }
+
+ m_videoSampleGrabber->deleteLater();
+ m_videoSampleGrabber = nullptr;
+}
+
+void DirectShowPlayerService::doReleaseAudioProbe(QMutexLocker *locker)
+{
+ Q_UNUSED(locker);
+
+ if (!m_graph)
+ return;
+
+ if (!m_audioSampleGrabber)
+ return;
+
+ m_audioSampleGrabber->stop();
+ HRESULT hr = m_graph->RemoveFilter(m_audioSampleGrabber->filter());
+ if (FAILED(hr)) {
+ qCWarning(qtDirectShowPlugin, "Failed to remove the audio sample grabber!");
+ return;
+ }
+
+ m_audioSampleGrabber->deleteLater();
+ m_audioSampleGrabber = nullptr;
+}
+
int DirectShowPlayerService::findStreamTypes(IBaseFilter *source) const
{
QVarLengthArray<IBaseFilter *, 16> filters;
@@ -1121,6 +1314,40 @@ void DirectShowPlayerService::setVideoOutput(IBaseFilter *filter)
}
}
+void DirectShowPlayerService::updateAudioProbe()
+{
+ QMutexLocker locker(&m_mutex);
+
+ // Set/Activate the audio probe.
+ if (m_graph) {
+ // If we don't have a audio probe, then stop and release the audio sample grabber
+ if (!m_audioProbeControl && (m_executedTasks & SetAudioProbe)) {
+ m_pendingTasks |= ReleaseAudioProbe;
+ ::SetEvent(m_taskHandle);
+ m_loop->wait(&m_mutex);
+ } else if (m_audioProbeControl) {
+ m_pendingTasks |= SetAudioProbe;
+ }
+ }
+}
+
+void DirectShowPlayerService::updateVideoProbe()
+{
+ QMutexLocker locker(&m_mutex);
+
+ // Set/Activate the video probe.
+ if (m_graph) {
+ // If we don't have a video probe, then stop and release the video sample grabber
+ if (!m_videoProbeControl && (m_executedTasks & SetVideoProbe)) {
+ m_pendingTasks |= ReleaseVideoProbe;
+ ::SetEvent(m_taskHandle);
+ m_loop->wait(&m_mutex);
+ } else if (m_videoProbeControl){
+ m_pendingTasks |= SetVideoProbe;
+ }
+ }
+}
+
void DirectShowPlayerService::doReleaseVideoOutput(QMutexLocker *locker)
{
Q_UNUSED(locker)
@@ -1217,6 +1444,97 @@ void DirectShowPlayerService::videoOutputChanged()
setVideoOutput(m_videoRendererControl->filter());
}
+void DirectShowPlayerService::onAudioBufferAvailable(double time, quint8 *buffer, long len)
+{
+ QMutexLocker locker(&m_mutex);
+ if (!m_audioProbeControl || !m_audioSampleGrabber)
+ return;
+
+ DirectShowMediaType mt(AM_MEDIA_TYPE { GUID_NULL });
+ const bool ok = m_audioSampleGrabber->getConnectedMediaType(&mt);
+ if (!ok)
+ return;
+
+ if (mt->majortype != MEDIATYPE_Audio)
+ return;
+
+ if (mt->subtype != MEDIASUBTYPE_PCM)
+ return;
+
+ const bool isWfx = ((mt->formattype == FORMAT_WaveFormatEx) && (mt->cbFormat >= sizeof(WAVEFORMATEX)));
+ WAVEFORMATEX *wfx = isWfx ? reinterpret_cast<WAVEFORMATEX *>(mt->pbFormat) : nullptr;
+
+ if (!wfx)
+ return;
+
+ if (wfx->wFormatTag != WAVE_FORMAT_PCM && wfx->wFormatTag != WAVE_FORMAT_EXTENSIBLE)
+ return;
+
+ if ((wfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE) && (wfx->cbSize >= sizeof(WAVEFORMATEXTENSIBLE))) {
+ WAVEFORMATEXTENSIBLE *wfxe = reinterpret_cast<WAVEFORMATEXTENSIBLE *>(wfx);
+ if (wfxe->SubFormat != KSDATAFORMAT_SUBTYPE_PCM)
+ return;
+ }
+
+ QAudioFormat format;
+ format.setSampleRate(wfx->nSamplesPerSec);
+ format.setChannelCount(wfx->nChannels);
+ format.setSampleSize(wfx->wBitsPerSample);
+ format.setCodec("audio/pcm");
+ format.setByteOrder(QAudioFormat::LittleEndian);
+ if (format.sampleSize() == 8)
+ format.setSampleType(QAudioFormat::UnSignedInt);
+ else
+ format.setSampleType(QAudioFormat::SignedInt);
+
+ const quint64 startTime = quint64(time * 1000.);
+ QAudioBuffer audioBuffer(QByteArray(reinterpret_cast<const char *>(buffer), len),
+ format,
+ startTime);
+
+ Q_EMIT m_audioProbeControl->audioBufferProbed(audioBuffer);
+}
+
+void DirectShowPlayerService::onVideoBufferAvailable(double time, quint8 *buffer, long len)
+{
+ Q_UNUSED(time);
+ Q_UNUSED(buffer);
+ Q_UNUSED(len);
+
+ QMutexLocker locker(&m_mutex);
+ if (!m_videoProbeControl || !m_videoSampleGrabber)
+ return;
+
+ DirectShowMediaType mt(AM_MEDIA_TYPE { GUID_NULL });
+ const bool ok = m_videoSampleGrabber->getConnectedMediaType(&mt);
+ if (!ok)
+ return;
+
+ if (mt->majortype != MEDIATYPE_Video)
+ return;
+
+ QVideoFrame::PixelFormat format = DirectShowMediaType::pixelFormatFromType(&mt);
+ if (format == QVideoFrame::Format_Invalid) {
+ qCWarning(qtDirectShowPlugin, "Invalid format, stopping video probes!");
+ m_videoSampleGrabber->stop();
+ return;
+ }
+
+ const QVideoSurfaceFormat &videoFormat = DirectShowMediaType::videoFormatFromType(&mt);
+ if (!videoFormat.isValid())
+ return;
+
+ const QSize &size = videoFormat.frameSize();
+
+ const int bytesPerLine = DirectShowMediaType::bytesPerLine(videoFormat);
+ QByteArray data(reinterpret_cast<const char *>(buffer), len);
+ QVideoFrame frame(new QMemoryVideoBuffer(data, bytesPerLine),
+ size,
+ format);
+
+ Q_EMIT m_videoProbeControl->videoFrameProbed(frame);
+}
+
void DirectShowPlayerService::graphEvent(QMutexLocker *locker)
{
Q_UNUSED(locker)
@@ -1405,6 +1723,16 @@ void DirectShowPlayerService::run()
m_executingTask = ReleaseVideoOutput;
doReleaseVideoOutput(&locker);
+ } else if (m_pendingTasks & ReleaseAudioProbe) {
+ m_pendingTasks ^= ReleaseAudioProbe;
+ m_executingTask = ReleaseAudioProbe;
+
+ doReleaseAudioProbe(&locker);
+ } else if (m_pendingTasks & ReleaseVideoProbe) {
+ m_pendingTasks ^= ReleaseVideoProbe;
+ m_executingTask = ReleaseVideoProbe;
+
+ doReleaseVideoProbe(&locker);
} else if (m_pendingTasks & SetUrlSource) {
m_pendingTasks ^= SetUrlSource;
m_executingTask = SetUrlSource;
@@ -1415,6 +1743,16 @@ void DirectShowPlayerService::run()
m_executingTask = SetStreamSource;
doSetStreamSource(&locker);
+ } else if (m_pendingTasks & SetAudioProbe) {
+ m_pendingTasks ^= SetAudioProbe;
+ m_executingTask = SetAudioProbe;
+
+ doSetAudioProbe(&locker);
+ } else if (m_pendingTasks & SetVideoProbe) {
+ m_pendingTasks ^= SetVideoProbe;
+ m_executingTask = SetVideoProbe;
+
+ doSetVideoProbe(&locker);
} else if (m_pendingTasks & Render) {
m_pendingTasks ^= Render;
m_executingTask = Render;
diff --git a/src/plugins/directshow/player/directshowplayerservice.h b/src/plugins/directshow/player/directshowplayerservice.h
index dc022600..676d88fb 100644
--- a/src/plugins/directshow/player/directshowplayerservice.h
+++ b/src/plugins/directshow/player/directshowplayerservice.h
@@ -61,6 +61,9 @@ class DirectShowAudioEndpointControl;
class DirectShowMetaDataControl;
class DirectShowPlayerControl;
class DirectShowVideoRendererControl;
+class DirectShowAudioProbeControl;
+class DirectShowVideoProbeControl;
+class DirectShowSampleGrabber;
class QMediaContent;
class QVideoWindowControl;
@@ -103,10 +106,16 @@ protected:
private Q_SLOTS:
void videoOutputChanged();
+ void onAudioBufferAvailable(double time, quint8 *buffer, long len);
+ void onVideoBufferAvailable(double time, quint8 *buffer, long len);
+
private:
void releaseGraph();
void updateStatus();
+ void updateAudioProbe();
+ void updateVideoProbe();
+
int findStreamTypes(IBaseFilter *source) const;
int findStreamType(IPin *pin) const;
@@ -127,29 +136,40 @@ private:
void doReleaseAudioOutput(QMutexLocker *locker);
void doReleaseVideoOutput(QMutexLocker *locker);
void doReleaseGraph(QMutexLocker *locker);
+ void doSetVideoProbe(QMutexLocker *locker);
+ void doSetAudioProbe(QMutexLocker *locker);
+ void doReleaseVideoProbe(QMutexLocker *locker);
+ void doReleaseAudioProbe(QMutexLocker *locker);
void graphEvent(QMutexLocker *locker);
enum Task
{
- Shutdown = 0x0001,
- SetUrlSource = 0x0002,
- SetStreamSource = 0x0004,
+ Shutdown = 0x00001,
+ SetUrlSource = 0x00002,
+ SetStreamSource = 0x00004,
SetSource = SetUrlSource | SetStreamSource,
- SetAudioOutput = 0x0008,
- SetVideoOutput = 0x0010,
+ SetAudioOutput = 0x00008,
+ SetVideoOutput = 0x00010,
SetOutputs = SetAudioOutput | SetVideoOutput,
- Render = 0x0020,
- FinalizeLoad = 0x0040,
- SetRate = 0x0080,
- Seek = 0x0100,
- Play = 0x0200,
- Pause = 0x0400,
- Stop = 0x0800,
- ReleaseGraph = 0x1000,
- ReleaseAudioOutput = 0x2000,
- ReleaseVideoOutput = 0x4000,
- ReleaseFilters = ReleaseGraph | ReleaseAudioOutput | ReleaseVideoOutput
+ SetAudioProbe = 0x00020,
+ SetVideoProbe = 0x00040,
+ SetProbes = SetAudioProbe | SetVideoProbe,
+ Render = 0x00080,
+ FinalizeLoad = 0x00100,
+ SetRate = 0x00200,
+ Seek = 0x00400,
+ Play = 0x00800,
+ Pause = 0x01000,
+ Stop = 0x02000,
+ ReleaseGraph = 0x04000,
+ ReleaseAudioOutput = 0x08000,
+ ReleaseVideoOutput = 0x10000,
+ ReleaseAudioProbe = 0x20000,
+ ReleaseVideoProbe = 0x40000,
+ ReleaseFilters = ReleaseGraph | ReleaseAudioOutput
+ | ReleaseVideoOutput | ReleaseAudioProbe
+ | ReleaseVideoProbe
};
enum Event
@@ -178,6 +198,10 @@ private:
DirectShowVideoRendererControl *m_videoRendererControl;
QVideoWindowControl *m_videoWindowControl;
DirectShowAudioEndpointControl *m_audioEndpointControl;
+ DirectShowAudioProbeControl *m_audioProbeControl;
+ DirectShowVideoProbeControl *m_videoProbeControl;
+ DirectShowSampleGrabber *m_audioSampleGrabber;
+ DirectShowSampleGrabber *m_videoSampleGrabber;
QThread *m_taskThread;
DirectShowEventLoop *m_loop;
diff --git a/src/plugins/directshow/player/player.pri b/src/plugins/directshow/player/player.pri
index 59ac5a33..9111cc54 100644
--- a/src/plugins/directshow/player/player.pri
+++ b/src/plugins/directshow/player/player.pri
@@ -3,6 +3,8 @@ INCLUDEPATH += $$PWD
QMAKE_USE += directshow
LIBS += -lgdi32
+mingw: LIBS_PRIVATE += -lksuser
+
qtHaveModule(widgets): QT += widgets
HEADERS += \