Android: fixed image capture preview.

We used to generate the capture preview from the video output, grabbing
the pixels of the last frame (in a FBO). This is not possible anymore,
we instead query the camera for a preview frame, which is in the NV21
format and needs to be converted to RGB.

Change-Id: I1c728b3a708a6f052a83aebf9f15f511eab7a02f
Reviewed-by: Christian Stromme <christian.stromme@digia.com>

9 files changed, 114 insertions, 16 deletions

diff --git a/src/plugins/android/src/common/qandroidmultimediautils.cpp b/src/plugins/android/src/common/qandroidmultimediautils.cpp
index 7ae40358f..9bf38a869 100644
--- a/src/plugins/android/src/common/qandroidmultimediautils.cpp
+++ b/src/plugins/android/src/common/qandroidmultimediautils.cpp
@@ -76,4 +76,29 @@ bool qt_sizeLessThan(const QSize &s1, const QSize &s2)
     return s1.width() * s1.height() < s2.width() * s2.height();
 }
 
+void qt_convert_NV21_to_ARGB32(const uchar *yuv, quint32 *rgb, int width, int height)
+{
+    const int frameSize = width * height;
+
+    int a = 0;
+    for (int i = 0, ci = 0; i < height; ++i, ci += 1) {
+        for (int j = 0, cj = 0; j < width; ++j, cj += 1) {
+            int y = (0xff & ((int) yuv[ci * width + cj]));
+            int v = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 0]));
+            int u = (0xff & ((int) yuv[frameSize + (ci >> 1) * width + (cj & ~1) + 1]));
+            y = y < 16 ? 16 : y;
+
+            int r = (int) (1.164f * (y - 16) + 1.596f * (v - 128));
+            int g = (int) (1.164f * (y - 16) - 0.813f * (v - 128) - 0.391f * (u - 128));
+            int b = (int) (1.164f * (y - 16) + 2.018f * (u - 128));
+
+            r = qBound(0, r, 255);
+            g = qBound(0, g, 255);
+            b = qBound(0, b, 255);
+
+            rgb[a++] = 0xff000000 | (r << 16) | (g << 8) | b;
+        }
+    }
+}
+
 QT_END_NAMESPACE
diff --git a/src/plugins/android/src/common/qandroidmultimediautils.h b/src/plugins/android/src/common/qandroidmultimediautils.h
index 1996209b0..792ab06db 100644
--- a/src/plugins/android/src/common/qandroidmultimediautils.h
+++ b/src/plugins/android/src/common/qandroidmultimediautils.h
@@ -53,6 +53,8 @@ int qt_findClosestValue(const QList<int> &list, int value);
 
 bool qt_sizeLessThan(const QSize &s1, const QSize &s2);
 
+void qt_convert_NV21_to_ARGB32(const uchar *yuv, quint32 *rgb, int width, int height);
+
 QT_END_NAMESPACE
 
 #endif // QANDROIDMULTIMEDIAUTILS_H
diff --git a/src/plugins/android/src/common/qandroidvideooutput.h b/src/plugins/android/src/common/qandroidvideooutput.h
index 1ad64a4b4..6e4a32e3f 100644
--- a/src/plugins/android/src/common/qandroidvideooutput.h
+++ b/src/plugins/android/src/common/qandroidvideooutput.h
@@ -61,8 +61,6 @@ public:
     virtual void setVideoSize(const QSize &) { }
     virtual void stop() { }
 
-    virtual QImage toImage() = 0;
-
     // signals:
     // void readyChanged(bool);
 };
diff --git a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp
index b9aac6434..5306fe918 100644
--- a/src/plugins/android/src/common/qandroidvideorendercontrol.cpp
+++ b/src/plugins/android/src/common/qandroidvideorendercontrol.cpp
@@ -223,13 +223,6 @@ void QAndroidVideoRendererControl::stop()
     m_nativeSize = QSize();
 }
 
-QImage QAndroidVideoRendererControl::toImage()
-{
-    // FIXME!!! Since we are not using a FBO anymore, we can't grab the pixels. And glGetTexImage
-    // doesn't work on GL_TEXTURE_EXTERNAL_OES
-    return QImage();
-}
-
 void QAndroidVideoRendererControl::onFrameAvailable()
 {
     if (!m_nativeSize.isValid() || !m_surface)
diff --git a/src/plugins/android/src/common/qandroidvideorendercontrol.h b/src/plugins/android/src/common/qandroidvideorendercontrol.h
index e98495ede..5d9130c07 100644
--- a/src/plugins/android/src/common/qandroidvideorendercontrol.h
+++ b/src/plugins/android/src/common/qandroidvideorendercontrol.h
@@ -77,7 +77,6 @@ public:
     bool isReady() Q_DECL_OVERRIDE;
     void setVideoSize(const QSize &size) Q_DECL_OVERRIDE;
     void stop() Q_DECL_OVERRIDE;
-    QImage toImage() Q_DECL_OVERRIDE;
 
     void customEvent(QEvent *) Q_DECL_OVERRIDE;
 
diff --git a/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp b/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp
index ec86322a4..3ee700bf4 100644
--- a/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp
+++ b/src/plugins/android/src/mediacapture/qandroidcamerasession.cpp
@@ -147,8 +147,11 @@ bool QAndroidCameraSession::open()
     if (m_camera) {
         connect(m_camera, SIGNAL(pictureExposed()), this, SLOT(onCameraPictureExposed()));
         connect(m_camera, SIGNAL(pictureCaptured(QByteArray)), this, SLOT(onCameraPictureCaptured(QByteArray)));
+        connect(m_camera, SIGNAL(previewFrameAvailable(QByteArray)), this, SLOT(onCameraPreviewFrameAvailable(QByteArray)));
         m_nativeOrientation = m_camera->getNativeOrientation();
         m_status = QCamera::LoadedStatus;
+        if (m_camera->getPreviewFormat() != JCamera::NV21)
+            m_camera->setPreviewFormat(JCamera::NV21);
         emit opened();
     } else {
         m_status = QCamera::UnavailableStatus;
@@ -422,6 +425,7 @@ int QAndroidCameraSession::capture(const QString &fileName)
         // adjust picture rotation depending on the device orientation
         m_camera->setRotation(currentCameraRotation());
 
+        m_camera->requestPreviewFrame();
         m_camera->takePicture();
     } else {
         emit imageCaptureError(m_lastImageCaptureId, QCameraImageCapture::NotSupportedFeatureError,
@@ -450,10 +454,6 @@ void QAndroidCameraSession::onCameraPictureExposed()
 void QAndroidCameraSession::onCameraPictureCaptured(const QByteArray &data)
 {
     if (!m_captureCanceled) {
-        // generate a preview from the viewport
-        if (m_videoOutput)
-            emit imageCaptured(m_currentImageCaptureId, m_videoOutput->toImage());
-
         // Loading and saving the captured image can be slow, do it in a separate thread
         QtConcurrent::run(this, &QAndroidCameraSession::processCapturedImage,
                           m_currentImageCaptureId,
@@ -517,6 +517,34 @@ void QAndroidCameraSession::processCapturedImage(int id,
     }
 }
 
+void QAndroidCameraSession::onCameraPreviewFrameAvailable(const QByteArray &data)
+{
+    if (m_captureCanceled || m_readyForCapture)
+        return;
+
+    QtConcurrent::run(this, &QAndroidCameraSession::processPreviewImage,
+                      m_currentImageCaptureId,
+                      data);
+}
+
+void QAndroidCameraSession::processPreviewImage(int id, const QByteArray &data)
+{
+    QSize frameSize = m_camera->previewSize();
+    QImage preview(frameSize, QImage::Format_ARGB32);
+    qt_convert_NV21_to_ARGB32((const uchar *)data.constData(),
+                              (quint32 *)preview.bits(),
+                              frameSize.width(),
+                              frameSize.height());
+
+    // Preview display of front-facing cameras is flipped horizontally, but the frame data
+    // we get here is not. Flip it ourselves if the camera is front-facing to match what the user
+    // sees on the viewfinder.
+    if (m_camera->getFacing() == JCamera::CameraFacingFront)
+        preview = preview.transformed(QTransform().scale(-1, 1));
+
+    emit imageCaptured(id, preview);
+}
+
 void QAndroidCameraSession::onVideoOutputReady(bool ready)
 {
     if (m_camera && m_videoOutput && ready)
diff --git a/src/plugins/android/src/mediacapture/qandroidcamerasession.h b/src/plugins/android/src/mediacapture/qandroidcamerasession.h
index 8d5b01c64..de891522b 100644
--- a/src/plugins/android/src/mediacapture/qandroidcamerasession.h
+++ b/src/plugins/android/src/mediacapture/qandroidcamerasession.h
@@ -114,6 +114,7 @@ private Q_SLOTS:
 
     void onCameraPictureExposed();
     void onCameraPictureCaptured(const QByteArray &data);
+    void onCameraPreviewFrameAvailable(const QByteArray &data);
 
 private:
     bool open();
@@ -123,7 +124,7 @@ private:
     void stopPreview();
 
     void applyImageSettings();
-    void processPreviewImage(int id);
+    void processPreviewImage(int id, const QByteArray &data);
     void processCapturedImage(int id,
                               const QByteArray &data,
                               QCameraImageCapture::CaptureDestinations dest,
diff --git a/src/plugins/android/src/wrappers/jcamera.cpp b/src/plugins/android/src/wrappers/jcamera.cpp
index f53fa4936..f858f4702 100644
--- a/src/plugins/android/src/wrappers/jcamera.cpp
+++ b/src/plugins/android/src/wrappers/jcamera.cpp
@@ -102,6 +102,18 @@ static void notifyPictureCaptured(JNIEnv *env, jobject, int id, jbyteArray data)
     }
 }
 
+static void notifyPreviewFrame(JNIEnv *env, jobject, int id, jbyteArray data)
+{
+    JCamera *obj = g_objectMap.value(id, 0);
+    if (obj) {
+        QByteArray bytes;
+        int arrayLength = env->GetArrayLength(data);
+        bytes.resize(arrayLength);
+        env->GetByteArrayRegion(data, 0, arrayLength, (jbyte*)bytes.data());
+        Q_EMIT obj->previewFrameAvailable(bytes);
+    }
+}
+
 JCamera::JCamera(int cameraId, jobject cam)
     : QObject()
     , QJNIObjectPrivate(cam)
@@ -225,6 +237,23 @@ QList<QSize> JCamera::getSupportedPreviewSizes()
     return list;
 }
 
+JCamera::ImageFormat JCamera::getPreviewFormat()
+{
+    if (!m_parameters.isValid())
+        return Unknown;
+
+    return JCamera::ImageFormat(m_parameters.callMethod<jint>("getPreviewFormat"));
+}
+
+void JCamera::setPreviewFormat(ImageFormat fmt)
+{
+    if (!m_parameters.isValid())
+        return;
+
+    m_parameters.callMethod<void>("setPreviewFormat", "(I)V", jint(fmt));
+    applyParameters();
+}
+
 void JCamera::setPreviewSize(const QSize &size)
 {
     if (!m_parameters.isValid())
@@ -624,6 +653,11 @@ void JCamera::setJpegQuality(int quality)
     applyParameters();
 }
 
+void JCamera::requestPreviewFrame()
+{
+    callMethod<void>("requestPreviewFrame");
+}
+
 void JCamera::takePicture()
 {
     callMethod<void>("takePicture");
@@ -672,7 +706,8 @@ QStringList JCamera::callStringListMethod(const char *methodName)
 static JNINativeMethod methods[] = {
     {"notifyAutoFocusComplete", "(IZ)V", (void *)notifyAutoFocusComplete},
     {"notifyPictureExposed", "(I)V", (void *)notifyPictureExposed},
-    {"notifyPictureCaptured", "(I[B)V", (void *)notifyPictureCaptured}
+    {"notifyPictureCaptured", "(I[B)V", (void *)notifyPictureCaptured},
+    {"notifyPreviewFrame", "(I[B)V", (void *)notifyPreviewFrame}
 };
 
 bool JCamera::initJNI(JNIEnv *env)
diff --git a/src/plugins/android/src/wrappers/jcamera.h b/src/plugins/android/src/wrappers/jcamera.h
index 0aea81f38..464ca3cb2 100644
--- a/src/plugins/android/src/wrappers/jcamera.h
+++ b/src/plugins/android/src/wrappers/jcamera.h
@@ -58,6 +58,16 @@ public:
         CameraFacingFront = 1
     };
 
+    enum ImageFormat { // same values as in android.graphics.ImageFormat Java class
+        Unknown = 0,
+        RGB565 = 4,
+        NV16 = 16,
+        NV21 = 17,
+        YUY2 = 20,
+        JPEG = 256,
+        YV12 = 842094169
+    };
+
     ~JCamera();
 
     static JCamera *open(int cameraId);
@@ -75,6 +85,9 @@ public:
     QSize getPreferredPreviewSizeForVideo();
     QList<QSize> getSupportedPreviewSizes();
 
+    ImageFormat getPreviewFormat();
+    void setPreviewFormat(ImageFormat fmt);
+
     QSize previewSize() const { return m_previewSize; }
     void setPreviewSize(const QSize &size);
     void setPreviewTexture(jobject surfaceTexture);
@@ -131,6 +144,8 @@ public:
     void startPreview();
     void stopPreview();
 
+    void requestPreviewFrame();
+
     void takePicture();
 
     static bool initJNI(JNIEnv *env);
@@ -143,6 +158,8 @@ Q_SIGNALS:
 
     void whiteBalanceChanged();
 
+    void previewFrameAvailable(const QByteArray &data);
+
     void pictureExposed();
     void pictureCaptured(const QByteArray &data);