Fix VNC format conversion

Pass the right from depth to the conversion function, and
set the right format before creating the compositor image
for RGB16 support.

Fixes: QTBUG-85621
Change-Id: I76f46a3c2d8f1d2b040b790035dbdb0a960ff1a7
Reviewed-by: Andy Nichols <andy.nichols@qt.io>
Reviewed-by: Eirik Aavitsland <eirik.aavitsland@qt.io>
(cherry picked from commit 47c6b5b91e0d1271075d98ded5aa7a25296b5ee3)
Reviewed-by: Qt Cherry-pick Bot <cherrypick_bot@qt-project.org>

4 files changed, 9 insertions, 8 deletions

diff --git a/src/plugins/platforms/vnc/qvnc.cpp b/src/plugins/platforms/vnc/qvnc.cpp
index d1dc6634af..94541508eb 100644
--- a/src/plugins/platforms/vnc/qvnc.cpp
+++ b/src/plugins/platforms/vnc/qvnc.cpp
@@ -514,8 +514,9 @@ void QRfbRawEncoder::write()
             // convert pixels
             char *b = buffer.data();
             const int bstep = rect.w * bytesPerPixel;
+            const int depth = screenImage.depth();
             for (int i = 0; i < rect.h; ++i) {
-                client->convertPixels(b, (const char*)screendata, rect.w);
+                client->convertPixels(b, (const char*)screendata, rect.w, depth);
                 screendata += linestep;
                 b += bstep;
             }
@@ -568,9 +569,10 @@ void QVncClientCursor::write(QVncClient *client) const
     Q_ASSERT(cursor.hasAlphaChannel());
     const QImage img = cursor.convertToFormat(client->server()->screen()->format());
     const int n = client->clientBytesPerPixel() * img.width();
+    const int depth = img.depth();
     char *buffer = new char[n];
     for (int i = 0; i < img.height(); ++i) {
-        client->convertPixels(buffer, (const char*)img.scanLine(i), img.width());
+        client->convertPixels(buffer, (const char*)img.scanLine(i), img.width(), depth);
         socket->write(buffer, n);
     }
     delete[] buffer;
diff --git a/src/plugins/platforms/vnc/qvncclient.cpp b/src/plugins/platforms/vnc/qvncclient.cpp
index ba0ef90891..3d0c1e48c3 100644
--- a/src/plugins/platforms/vnc/qvncclient.cpp
+++ b/src/plugins/platforms/vnc/qvncclient.cpp
@@ -97,10 +97,8 @@ void QVncClient::setDirty(const QRegion &region)
     }
 }
 
-void QVncClient::convertPixels(char *dst, const char *src, int count) const
+void QVncClient::convertPixels(char *dst, const char *src, int count, int screendepth) const
 {
-    const int screendepth = m_server->screen()->depth();
-
     // cutoffs
 #if Q_BYTE_ORDER == Q_BIG_ENDIAN
     if (!m_swapBytes)
diff --git a/src/plugins/platforms/vnc/qvncclient.h b/src/plugins/platforms/vnc/qvncclient.h
index a7a6b6b361..0117f1c4a0 100644
--- a/src/plugins/platforms/vnc/qvncclient.h
+++ b/src/plugins/platforms/vnc/qvncclient.h
@@ -77,7 +77,7 @@ public:
         return m_pixelFormat.bitsPerPixel / 8;
     }
 
-    void convertPixels(char *dst, const char *src, int count) const;
+    void convertPixels(char *dst, const char *src, int count, int depth) const;
     inline bool doPixelConversion() const { return m_needConversion; }
 
 signals:
diff --git a/src/plugins/platforms/vnc/qvncscreen.cpp b/src/plugins/platforms/vnc/qvncscreen.cpp
index 7ef23601ba..fcec8863fa 100644
--- a/src/plugins/platforms/vnc/qvncscreen.cpp
+++ b/src/plugins/platforms/vnc/qvncscreen.cpp
@@ -86,14 +86,13 @@ bool QVncScreen::initialize()
         }
     }
 
-    QFbScreen::initializeCompositor();
-
     switch (depth()) {
     case 32:
         dirty = new QVncDirtyMapOptimized<quint32>(this);
         break;
     case 16:
         dirty = new QVncDirtyMapOptimized<quint16>(this);
+        mFormat = QImage::Format_RGB16;
         break;
     case 8:
         dirty = new QVncDirtyMapOptimized<quint8>(this);
@@ -105,6 +104,8 @@ bool QVncScreen::initialize()
         return false;
     }
 
+    QFbScreen::initializeCompositor();
+
     setPowerState(PowerStateOff);
 
     return true;