Support per-plane strides and data offsets in QVideoFrame.

Since just adding a new virtual isn't binary compatible add a new derivative
type with a virtual member and connect it up through a virtual in the private
class.

[ChangeLog] Support for per-plane strides and data offsets in QVideoFrame.

Task-number: QTBUG-38345
Change-Id: I1974c2b0b454d130e17971ce549031259d61f9cd
Reviewed-by: Yoann Lopes <yoann.lopes@digia.com>

5 files changed, 328 insertions, 21 deletions

diff --git a/src/multimedia/video/qabstractvideobuffer.cpp b/src/multimedia/video/qabstractvideobuffer.cpp
index 79da6f9b7..5f29a01a1 100644
--- a/src/multimedia/video/qabstractvideobuffer.cpp
+++ b/src/multimedia/video/qabstractvideobuffer.cpp
@@ -56,6 +56,15 @@ static void qRegisterAbstractVideoBufferMetaTypes()
 
 Q_CONSTRUCTOR_FUNCTION(qRegisterAbstractVideoBufferMetaTypes)
 
+int QAbstractVideoBufferPrivate::map(
+            QAbstractVideoBuffer::MapMode mode,
+            int *numBytes,
+            int bytesPerLine[4],
+            uchar *data[4])
+{
+    data[0] = q_ptr->map(mode, numBytes, bytesPerLine);
+    return data[0] ? 1 : 0;
+}
 
 /*!
     \class QAbstractVideoBuffer
@@ -130,6 +139,7 @@ QAbstractVideoBuffer::QAbstractVideoBuffer(QAbstractVideoBufferPrivate &dd, Hand
     : d_ptr(&dd)
     , m_type(type)
 {
+    d_ptr->q_ptr = this;
 }
 
 /*!
@@ -199,6 +209,44 @@ QAbstractVideoBuffer::HandleType QAbstractVideoBuffer::handleType() const
     \sa unmap(), mapMode()
 */
 
+
+/*!
+    Independently maps the planes of a video buffer to memory.
+
+    The map \a mode indicates whether the contents of the mapped memory should be read from and/or
+    written to the buffer.  If the map mode includes the \c QAbstractVideoBuffer::ReadOnly flag the
+    mapped memory will be populated with the content of the buffer when initially mapped.  If the map
+    mode includes the \c QAbstractVideoBuffer::WriteOnly flag the content of the possibly modified
+    mapped memory will be written back to the buffer when unmapped.
+
+    When access to the data is no longer needed be sure to call the unmap() function to release the
+    mapped memory and possibly update the buffer contents.
+
+    Returns the number of planes in the mapped video data.  For each plane the line stride of that
+    plane will be returned in \a bytesPerLine, and a pointer to the plane data will be returned in
+    \a data.  The accumulative size of the mapped data is returned in \a numBytes.
+
+    Not all buffer implementations will map more than the first plane, if this returns a single
+    plane for a planar format the additional planes will have to be calculated from the line stride
+    of the first plane and the frame height.  Mapping a buffer with QVideoFrame will do this for
+    you.
+
+    To implement this function create a derivative of QAbstractPlanarVideoBuffer and implement
+    its map function instance instead.
+
+    \since 5.4
+*/
+int QAbstractVideoBuffer::mapPlanes(MapMode mode, int *numBytes, int bytesPerLine[4], uchar *data[4])
+{
+    if (d_ptr) {
+        return d_ptr->map(mode, numBytes, bytesPerLine, data);
+    } else {
+        data[0] = map(mode, numBytes, bytesPerLine);
+
+        return data[0] ? 1 : 0;
+    }
+}
+
 /*!
     \fn QAbstractVideoBuffer::unmap()
 
@@ -222,6 +270,90 @@ QVariant QAbstractVideoBuffer::handle() const
     return QVariant();
 }
 
+
+int QAbstractPlanarVideoBufferPrivate::map(
+        QAbstractVideoBuffer::MapMode mode, int *numBytes, int bytesPerLine[4], uchar *data[4])
+{
+    return q_func()->map(mode, numBytes, bytesPerLine, data);
+}
+
+/*!
+    \class QAbstractPlanarVideoBuffer
+    \brief The QAbstractPlanarVideoBuffer class is an abstraction for planar video data.
+    \inmodule QtMultimedia
+    \ingroup QtMultimedia
+    \ingroup multimedia
+    \ingroup multimedia_video
+
+    QAbstractPlanarVideoBuffer extends QAbstractVideoBuffer to support mapping
+    non-continuous planar video data.  Implement this instead of QAbstractVideoBuffer when the
+    abstracted video data stores planes in separate buffers or includes padding between planes
+    which would interfere with calculating offsets from the bytes per line and frame height.
+
+    \sa QAbstractVideoBuffer::mapPlanes()
+    \since 5.4
+*/
+
+/*!
+    Constructs an abstract planar video buffer of the given \a type.
+*/
+QAbstractPlanarVideoBuffer::QAbstractPlanarVideoBuffer(HandleType type)
+    : QAbstractVideoBuffer(*new QAbstractPlanarVideoBufferPrivate, type)
+{
+}
+
+/*!
+    \internal
+*/
+QAbstractPlanarVideoBuffer::QAbstractPlanarVideoBuffer(
+        QAbstractPlanarVideoBufferPrivate &dd, HandleType type)
+    : QAbstractVideoBuffer(dd, type)
+{
+}
+/*!
+    Destroys an abstract planar video buffer.
+*/
+QAbstractPlanarVideoBuffer::~QAbstractPlanarVideoBuffer()
+{
+}
+
+/*!
+    \internal
+*/
+uchar *QAbstractPlanarVideoBuffer::map(MapMode mode, int *numBytes, int *bytesPerLine)
+{
+    uchar *data[4];
+    int strides[4];
+    if (map(mode, numBytes, strides, data) > 0) {
+        if (bytesPerLine)
+            *bytesPerLine = strides[0];
+        return data[0];
+    } else {
+        return 0;
+    }
+}
+
+/*!
+    \fn int QAbstractPlanarVideoBuffer::map(MapMode mode, int *numBytes, int bytesPerLine[4], uchar *data[4])
+
+    Maps the contents of a video buffer to memory.
+
+    The map \a mode indicates whether the contents of the mapped memory should be read from and/or
+    written to the buffer.  If the map mode includes the \c QAbstractVideoBuffer::ReadOnly flag the
+    mapped memory will be populated with the content of the buffer when initially mapped.  If the map
+    mode includes the \c QAbstractVideoBuffer::WriteOnly flag the content of the possibly modified
+    mapped memory will be written back to the buffer when unmapped.
+
+    When access to the data is no longer needed be sure to call the unmap() function to release the
+    mapped memory and possibly update the buffer contents.
+
+    Returns the number of planes in the mapped video data.  For each plane the line stride of that
+    plane will be returned in \a bytesPerLine, and a pointer to the plane data will be returned in
+    \a data.  The accumulative size of the mapped data is returned in \a numBytes.
+
+    \sa QAbstractVideoBuffer::map(), QAbstractVideoBuffer::unmap(), QAbstractVideoBuffer::mapMode()
+*/
+
 #ifndef QT_NO_DEBUG_STREAM
 QDebug operator<<(QDebug dbg, QAbstractVideoBuffer::HandleType type)
 {
diff --git a/src/multimedia/video/qabstractvideobuffer.h b/src/multimedia/video/qabstractvideobuffer.h
index 1be90ff90..7f3edf569 100644
--- a/src/multimedia/video/qabstractvideobuffer.h
+++ b/src/multimedia/video/qabstractvideobuffer.h
@@ -85,6 +85,7 @@ public:
     virtual MapMode mapMode() const = 0;
 
     virtual uchar *map(MapMode mode, int *numBytes, int *bytesPerLine) = 0;
+    int mapPlanes(MapMode mode, int *numBytes, int bytesPerLine[4], uchar *data[4]);
     virtual void unmap() = 0;
 
     virtual QVariant handle() const;
@@ -100,6 +101,23 @@ private:
     Q_DISABLE_COPY(QAbstractVideoBuffer)
 };
 
+class QAbstractPlanarVideoBufferPrivate;
+class Q_MULTIMEDIA_EXPORT QAbstractPlanarVideoBuffer : public QAbstractVideoBuffer
+{
+public:
+    QAbstractPlanarVideoBuffer(HandleType type);
+    virtual ~QAbstractPlanarVideoBuffer();
+
+    uchar *map(MapMode mode, int *numBytes, int *bytesPerLine);
+    virtual int map(MapMode mode, int *numBytes, int bytesPerLine[4], uchar *data[4]) = 0;
+
+protected:
+    QAbstractPlanarVideoBuffer(QAbstractPlanarVideoBufferPrivate &dd, HandleType type);
+
+private:
+    Q_DISABLE_COPY(QAbstractPlanarVideoBuffer)
+};
+
 #ifndef QT_NO_DEBUG_STREAM
 Q_MULTIMEDIA_EXPORT QDebug operator<<(QDebug, QAbstractVideoBuffer::HandleType);
 Q_MULTIMEDIA_EXPORT QDebug operator<<(QDebug, QAbstractVideoBuffer::MapMode);
diff --git a/src/multimedia/video/qabstractvideobuffer_p.h b/src/multimedia/video/qabstractvideobuffer_p.h
index 8841f75d1..dfb1df653 100644
--- a/src/multimedia/video/qabstractvideobuffer_p.h
+++ b/src/multimedia/video/qabstractvideobuffer_p.h
@@ -66,10 +66,31 @@ class QAbstractVideoBufferPrivate
 {
 public:
     QAbstractVideoBufferPrivate()
+        : q_ptr(0)
     {}
 
     virtual ~QAbstractVideoBufferPrivate()
     {}
+
+    virtual int map(
+            QAbstractVideoBuffer::MapMode mode,
+            int *numBytes,
+            int bytesPerLine[4],
+            uchar *data[4]);
+
+    QAbstractVideoBuffer *q_ptr;
+};
+
+class QAbstractPlanarVideoBufferPrivate : QAbstractVideoBufferPrivate
+{
+public:
+    QAbstractPlanarVideoBufferPrivate()
+    {}
+
+    int map(QAbstractVideoBuffer::MapMode mode, int *numBytes, int bytesPerLine[4], uchar *data[4]);
+
+private:
+    Q_DECLARE_PUBLIC(QAbstractPlanarVideoBuffer)
 };
 
 QT_END_NAMESPACE
diff --git a/src/multimedia/video/qvideoframe.cpp b/src/multimedia/video/qvideoframe.cpp
index 4a32bc557..7a9e3e1ba 100644
--- a/src/multimedia/video/qvideoframe.cpp
+++ b/src/multimedia/video/qvideoframe.cpp
@@ -71,28 +71,30 @@ public:
     QVideoFramePrivate()
         : startTime(-1)
         , endTime(-1)
-        , data(0)
         , mappedBytes(0)
-        , bytesPerLine(0)
+        , planeCount(0)
         , pixelFormat(QVideoFrame::Format_Invalid)
         , fieldType(QVideoFrame::ProgressiveFrame)
         , buffer(0)
         , mappedCount(0)
     {
+        memset(data, 0, sizeof(data));
+        memset(bytesPerLine, 0, sizeof(bytesPerLine));
     }
 
     QVideoFramePrivate(const QSize &size, QVideoFrame::PixelFormat format)
         : size(size)
         , startTime(-1)
         , endTime(-1)
-        , data(0)
         , mappedBytes(0)
-        , bytesPerLine(0)
+        , planeCount(0)
         , pixelFormat(format)
         , fieldType(QVideoFrame::ProgressiveFrame)
         , buffer(0)
         , mappedCount(0)
     {
+        memset(data, 0, sizeof(data));
+        memset(bytesPerLine, 0, sizeof(bytesPerLine));
     }
 
     ~QVideoFramePrivate()
@@ -104,9 +106,10 @@ public:
     QSize size;
     qint64 startTime;
     qint64 endTime;
-    uchar *data;
+    uchar *data[4];
+    int bytesPerLine[4];
     int mappedBytes;
-    int bytesPerLine;
+    int planeCount;
     QVideoFrame::PixelFormat pixelFormat;
     QVideoFrame::FieldType fieldType;
     QAbstractVideoBuffer *buffer;
@@ -564,18 +567,88 @@ bool QVideoFrame::map(QAbstractVideoBuffer::MapMode mode)
         }
     }
 
-    Q_ASSERT(d->data == 0);
-    Q_ASSERT(d->bytesPerLine == 0);
+    Q_ASSERT(d->data[0] == 0);
+    Q_ASSERT(d->bytesPerLine[0] == 0);
+    Q_ASSERT(d->planeCount == 0);
     Q_ASSERT(d->mappedBytes == 0);
 
-    d->data = d->buffer->map(mode, &d->mappedBytes, &d->bytesPerLine);
+    d->planeCount = d->buffer->mapPlanes(mode, &d->mappedBytes, d->bytesPerLine, d->data);
+    if (d->planeCount == 0)
+        return false;
 
-    if (d->data) {
-        d->mappedCount++;
-        return true;
+    if (d->planeCount > 1) {
+        // If the plane count is derive the additional planes for planar formats.
+    } else switch (d->pixelFormat) {
+    case Format_Invalid:
+    case Format_ARGB32:
+    case Format_ARGB32_Premultiplied:
+    case Format_RGB32:
+    case Format_RGB24:
+    case Format_RGB565:
+    case Format_RGB555:
+    case Format_ARGB8565_Premultiplied:
+    case Format_BGRA32:
+    case Format_BGRA32_Premultiplied:
+    case Format_BGR32:
+    case Format_BGR24:
+    case Format_BGR565:
+    case Format_BGR555:
+    case Format_BGRA5658_Premultiplied:
+    case Format_AYUV444:
+    case Format_AYUV444_Premultiplied:
+    case Format_YUV444:
+    case Format_UYVY:
+    case Format_YUYV:
+    case Format_Y8:
+    case Format_Y16:
+    case Format_Jpeg:
+    case Format_CameraRaw:
+    case Format_AdobeDng:
+    case Format_User:
+        // Single plane or opaque format.
+        break;
+    case Format_YUV420P:
+    case Format_YV12: {
+        // The UV stride is usually half the Y stride and is 32-bit aligned.
+        // However it's not always the case, at least on Windows where the
+        // UV planes are sometimes not aligned.
+        // We calculate the stride using the UV byte count to always
+        // have a correct stride.
+        const int height = d->size.height();
+        const int yStride = d->bytesPerLine[0];
+        const int uvStride = (d->mappedBytes - (yStride * height)) / height;
+
+        // Three planes, the second and third vertically and horizontally subsampled.
+        d->planeCount = 3;
+        d->bytesPerLine[2] = d->bytesPerLine[1] = uvStride;
+        d->data[1] = d->data[0] + (yStride * height);
+        d->data[2] = d->data[1] + (uvStride * height / 2);
+        break;
+    }
+    case Format_NV12:
+    case Format_NV21:
+    case Format_IMC2:
+    case Format_IMC4: {
+        // Semi planar, Full resolution Y plane with interleaved subsampled U and V planes.
+        d->planeCount = 2;
+        d->bytesPerLine[1] = d->bytesPerLine[0];
+        d->data[1] = d->data[0] + (d->bytesPerLine[0] * d->size.height());
+        break;
+    }
+    case Format_IMC1:
+    case Format_IMC3: {
+        // Three planes, the second and third vertically and horizontally subsumpled,
+        // but with lines padded to the width of the first plane.
+        d->planeCount = 3;
+        d->bytesPerLine[2] = d->bytesPerLine[1] = d->bytesPerLine[0];
+        d->data[1] = d->data[0] + (d->bytesPerLine[0] * d->size.height());
+        d->data[2] = d->data[1] + (d->bytesPerLine[1] * d->size.height() / 2);
+        break;
+    }
     }
 
-    return false;
+    d->mappedCount++;
+    return true;
 }
 
 /*!
@@ -604,8 +677,9 @@ void QVideoFrame::unmap()
 
     if (d->mappedCount == 0) {
         d->mappedBytes = 0;
-        d->bytesPerLine = 0;
-        d->data = 0;
+        d->planeCount = 0;
+        memset(d->bytesPerLine, 0, sizeof(d->bytesPerLine));
+        memset(d->data, 0, sizeof(d->data));
 
         d->buffer->unmap();
     }
@@ -623,7 +697,21 @@ void QVideoFrame::unmap()
 */
 int QVideoFrame::bytesPerLine() const
 {
-    return d->bytesPerLine;
+    return d->bytesPerLine[0];
+}
+
+/*!
+    Returns the number of bytes in a scan line of a \a plane.
+
+    This value is only valid while the frame data is \l {map()}{mapped}.
+
+    \sa bits(), map(), mappedBytes(), planeCount()
+    \since 5.4
+*/
+
+int QVideoFrame::bytesPerLine(int plane) const
+{
+    return plane >= 0 && plane < d->planeCount ? d->bytesPerLine[plane] : 0;
 }
 
 /*!
@@ -639,7 +727,24 @@ int QVideoFrame::bytesPerLine() const
 */
 uchar *QVideoFrame::bits()
 {
-    return d->data;
+    return d->data[0];
+}
+
+/*!
+    Returns a pointer to the start of the frame data buffer for a \a plane.
+
+    This value is only valid while the frame data is \l {map()}{mapped}.
+
+    Changes made to data accessed via this pointer (when mapped with write access)
+    are only guaranteed to have been persisted when unmap() is called and when the
+    buffer has been mapped for writing.
+
+    \sa map(), mappedBytes(), bytesPerLine(), planeCount()
+    \since 5.4
+*/
+uchar *QVideoFrame::bits(int plane)
+{
+    return plane >= 0 && plane < d->planeCount ? d->data[plane] : 0;
 }
 
 /*!
@@ -654,7 +759,23 @@ uchar *QVideoFrame::bits()
 */
 const uchar *QVideoFrame::bits() const
 {
-    return d->data;
+    return d->data[0];
+}
+
+/*!
+    Returns a pointer to the start of the frame data buffer for a \a plane.
+
+    This value is only valid while the frame data is \l {map()}{mapped}.
+
+    If the buffer was not mapped with read access, the contents of this
+    buffer will initially be uninitialized.
+
+    \sa map(), mappedBytes(), bytesPerLine(), planeCount()
+    \since 5.4
+*/
+const uchar *QVideoFrame::bits(int plane) const
+{
+    return plane >= 0 && plane < d->planeCount ?  d->data[plane] : 0;
 }
 
 /*!
@@ -670,6 +791,20 @@ int QVideoFrame::mappedBytes() const
 }
 
 /*!
+    Returns the number of planes in the video frame.
+
+    This value is only valid while the frame data is \l {map()}{mapped}.
+
+    \sa map()
+    \since 5.4
+*/
+
+int QVideoFrame::planeCount() const
+{
+    return d->planeCount;
+}
+
+/*!
     Returns a type specific handle to a video frame's buffer.
 
     For an OpenGL texture this would be the texture ID.
@@ -852,9 +987,6 @@ QImage::Format QVideoFrame::imageFormatFromPixelFormat(PixelFormat format)
     return QImage::Format_Invalid;
 }
 
-
-
-
 #ifndef QT_NO_DEBUG_STREAM
 QDebug operator<<(QDebug dbg, QVideoFrame::PixelFormat pf)
 {
diff --git a/src/multimedia/video/qvideoframe.h b/src/multimedia/video/qvideoframe.h
index 504a0d548..0ca13d886 100644
--- a/src/multimedia/video/qvideoframe.h
+++ b/src/multimedia/video/qvideoframe.h
@@ -139,10 +139,14 @@ public:
     void unmap();
 
     int bytesPerLine() const;
+    int bytesPerLine(int plane) const;
 
     uchar *bits();
+    uchar *bits(int plane);
     const uchar *bits() const;
+    const uchar *bits(int plane) const;
     int mappedBytes() const;
+    int planeCount() const;
 
     QVariant handle() const;