D3D12: Support QQuickItem::grabToImage

that is, the grabbing of layers. This is where things start getting
insane because now the convenient assumptions of updateTexture() for
layers only getting called in the sync phase before the rendering of the
real frame begins breaks down. With item grabbing the call happens on
afterRendering()... the engine will now do a minor hack when doing the
readback to make this work.

This, together with the previous patches provides us with the following
impressive, highly karma-boosting results:

tst_scenegraph 24 passed, 0 failed, 11 skipped
tst_qquickwindow 52 passed, 0 failed, 6 skipped
tst_qquickitem 61 passed, 0 failed, 0 skipped
tst_qquickitem2 84 passed, 0 failed, 6 skipped

Change-Id: I8f160e6848215a3b3304f2e00516eea0d96eb4a0
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

3 files changed, 26 insertions, 9 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index c77e86ca99..1abc529835 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -2815,11 +2815,21 @@ void QSGD3D12EnginePrivate::useRenderTargetAsTexture(uint id)
 
 QImage QSGD3D12EnginePrivate::executeAndWaitReadbackRenderTarget(uint id)
 {
-    if (inFrame) {
+    // Readback due to QQuickWindow::grabWindow() happens outside
+    // begin-endFrame, but QQuickItemGrabResult leads to rendering a layer
+    // without a real frame afterwards and triggering readback. This has to be
+    // supported as well.
+    if (inFrame && (!activeLayers || currentLayerDepth)) {
         qWarning("%s: Cannot be called while frame preparation is active", __FUNCTION__);
         return QImage();
     }
 
+    // Due to the above we insert a fake "real" frame when a layer was just rendered into.
+    if (inFrame) {
+        beginFrame();
+        endFrame();
+    }
+
     frameCommandList->Reset(frameCommandAllocator[frameIndex % frameInFlightCount].Get(), nullptr);
 
     D3D12_RESOURCE_STATES bstate;
@@ -2913,10 +2923,19 @@ QImage QSGD3D12EnginePrivate::executeAndWaitReadbackRenderTarget(uint id)
         qWarning("Mapping the readback buffer failed");
         return QImage();
     }
-    for (UINT y = 0; y < rtDesc.Height; ++y) {
-        quint8 *dst = img.scanLine(y);
-        memcpy(dst, p, rtDesc.Width * 4);
-        p += textureLayout.Footprint.RowPitch;
+    const int bpp = 4; // ###
+    if (id == 0) {
+        for (UINT y = 0; y < rtDesc.Height; ++y) {
+            quint8 *dst = img.scanLine(y);
+            memcpy(dst, p, rtDesc.Width * bpp);
+            p += textureLayout.Footprint.RowPitch;
+        }
+    } else {
+        for (int y = rtDesc.Height - 1; y >= 0; --y) {
+            quint8 *dst = img.scanLine(y);
+            memcpy(dst, p, rtDesc.Width * bpp);
+            p += textureLayout.Footprint.RowPitch;
+        }
     }
     readbackBuf->Unmap(0, nullptr);
 
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp
index ef2f923d5b..faa6f7566a 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12layer.cpp
@@ -186,8 +186,7 @@ void QSGD3D12Layer::scheduleUpdate()
 
 QImage QSGD3D12Layer::toImage() const
 {
-    // ### figure out something for item grab support
-    return QImage();
+    return m_rc->engine()->executeAndWaitReadbackRenderTarget(m_rt);
 }
 
 void QSGD3D12Layer::setLive(bool live)
diff --git a/src/quick/items/qquickitemgrabresult.cpp b/src/quick/items/qquickitemgrabresult.cpp
index 3ff61be241..f69b5b3f4b 100644
--- a/src/quick/items/qquickitemgrabresult.cpp
+++ b/src/quick/items/qquickitemgrabresult.cpp
@@ -240,8 +240,7 @@ void QQuickItemGrabResult::render()
         return;
 
     d->texture->setRect(QRectF(0, d->itemSize.height(), d->itemSize.width(), -d->itemSize.height()));
-    QSGContext *sg = QSGDefaultRenderContext::from(QOpenGLContext::currentContext())->sceneGraphContext();
-    const QSize minSize = sg->minimumFBOSize();
+    const QSize minSize = QQuickWindowPrivate::get(d->window.data())->context->sceneGraphContext()->minimumFBOSize();
     d->texture->setSize(QSize(qMax(minSize.width(), d->textureSize.width()),
                               qMax(minSize.height(), d->textureSize.height())));
     d->texture->scheduleUpdate();