D3D12: Implement grabWindow

Change-Id: Icb8151f26bad68795eb2e1f920297267c880b40b
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

9 files changed, 225 insertions, 28 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index 7445a8bb40..c77e86ca99 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -515,6 +515,11 @@ uint QSGD3D12Engine::activeRenderTarget() const
     return d->activeRenderTarget();
 }
 
+QImage QSGD3D12Engine::executeAndWaitReadbackRenderTarget(uint id)
+{
+    return d->executeAndWaitReadbackRenderTarget(id);
+}
+
 QSGRendererInterface::GraphicsAPI QSGD3D12Engine::graphicsAPI() const
 {
     return Direct3D12;
@@ -1748,7 +1753,12 @@ void QSGD3D12EnginePrivate::queueSetRenderTarget(uint id)
         RenderTarget &rt(renderTargets[idx]);
         rtvHandle = rt.rtv;
         dsvHandle = rt.dsv;
-        rt.flags |= RenderTarget::NeedsReadBarrier;
+        if (!(rt.flags & RenderTarget::NeedsReadBarrier)) {
+            rt.flags |= RenderTarget::NeedsReadBarrier;
+            if (!(rt.flags & RenderTarget::Multisample))
+                transitionResource(rt.color.Get(), commandList, D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE,
+                                   D3D12_RESOURCE_STATE_RENDER_TARGET);
+        }
     }
 
     commandList->OMSetRenderTargets(1, &rtvHandle, FALSE, &dsvHandle);
@@ -2163,6 +2173,27 @@ static inline DXGI_FORMAT textureFormat(QImage::Format format, bool wantsAlpha,
     return f;
 }
 
+static inline QImage::Format imageFormatForTexture(DXGI_FORMAT format)
+{
+    QImage::Format f = QImage::Format_Invalid;
+
+    switch (format) {
+    case DXGI_FORMAT_R8G8B8A8_UNORM:
+        f = QImage::Format_RGBA8888_Premultiplied;
+        break;
+    case DXGI_FORMAT_B8G8R8A8_UNORM:
+        f = QImage::Format_ARGB32_Premultiplied;
+        break;
+    case DXGI_FORMAT_R8_UNORM:
+        f = QImage::Format_Grayscale8;
+        break;
+    default:
+        break;
+    }
+
+    return f;
+}
+
 void QSGD3D12EnginePrivate::createTexture(uint id, const QSize &size, QImage::Format format,
                                           QSGD3D12Engine::TextureCreateFlags createFlags)
 {
@@ -2782,6 +2813,116 @@ void QSGD3D12EnginePrivate::useRenderTargetAsTexture(uint id)
     tframeData.activeTextures.append(TransientFrameData::ActiveTexture::ActiveTexture(TransientFrameData::ActiveTexture::TypeRenderTarget, id));
 }
 
+QImage QSGD3D12EnginePrivate::executeAndWaitReadbackRenderTarget(uint id)
+{
+    if (inFrame) {
+        qWarning("%s: Cannot be called while frame preparation is active", __FUNCTION__);
+        return QImage();
+    }
+
+    frameCommandList->Reset(frameCommandAllocator[frameIndex % frameInFlightCount].Get(), nullptr);
+
+    D3D12_RESOURCE_STATES bstate;
+    bool needsBarrier = false;
+    ID3D12Resource *rtRes;
+    if (id == 0) {
+        const int idx = presentFrameIndex % swapChainBufferCount;
+        if (windowSamples > 1) {
+            resolveMultisampledTarget(defaultRT[idx].Get(), backBufferRT[idx].Get(),
+                                      D3D12_RESOURCE_STATE_COPY_SOURCE, frameCommandList.Get());
+        } else {
+            bstate = D3D12_RESOURCE_STATE_PRESENT;
+            needsBarrier = true;
+        }
+        rtRes = backBufferRT[idx].Get();
+    } else {
+        const int idx = id - 1;
+        Q_ASSERT(idx < renderTargets.count());
+        RenderTarget &rt(renderTargets[idx]);
+        Q_ASSERT(rt.entryInUse() && rt.color);
+
+        if (rt.flags & RenderTarget::Multisample) {
+            resolveMultisampledTarget(rt.color.Get(), rt.colorResolve.Get(),
+                                      D3D12_RESOURCE_STATE_COPY_SOURCE, frameCommandList.Get());
+            rtRes = rt.colorResolve.Get();
+        } else {
+            rtRes = rt.color.Get();
+            bstate = D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE;
+            needsBarrier = true;
+        }
+    }
+
+    ComPtr<ID3D12Resource> readbackBuf;
+
+    D3D12_RESOURCE_DESC rtDesc = rtRes->GetDesc();
+    UINT64 textureByteSize = 0;
+    D3D12_PLACED_SUBRESOURCE_FOOTPRINT textureLayout = {};
+    device->GetCopyableFootprints(&rtDesc, 0, 1, 0, &textureLayout, nullptr, nullptr, &textureByteSize);
+
+    D3D12_HEAP_PROPERTIES heapProp = {};
+    heapProp.Type = D3D12_HEAP_TYPE_READBACK;
+
+    D3D12_RESOURCE_DESC bufDesc = {};
+    bufDesc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
+    bufDesc.Width = textureByteSize;
+    bufDesc.Height = 1;
+    bufDesc.DepthOrArraySize = 1;
+    bufDesc.MipLevels = 1;
+    bufDesc.Format = DXGI_FORMAT_UNKNOWN;
+    bufDesc.SampleDesc.Count = 1;
+    bufDesc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
+
+    if (FAILED(device->CreateCommittedResource(&heapProp, D3D12_HEAP_FLAG_NONE, &bufDesc,
+                                               D3D12_RESOURCE_STATE_COPY_DEST, nullptr, IID_PPV_ARGS(&readbackBuf)))) {
+        qWarning("Failed to create committed resource (readback buffer)");
+        return QImage();
+    }
+
+    D3D12_TEXTURE_COPY_LOCATION dstLoc;
+    dstLoc.pResource = readbackBuf.Get();
+    dstLoc.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
+    dstLoc.PlacedFootprint = textureLayout;
+    D3D12_TEXTURE_COPY_LOCATION srcLoc;
+    srcLoc.pResource = rtRes;
+    srcLoc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+    srcLoc.SubresourceIndex = 0;
+
+    ID3D12GraphicsCommandList *cl = frameCommandList.Get();
+    if (needsBarrier)
+        transitionResource(rtRes, cl, bstate, D3D12_RESOURCE_STATE_COPY_SOURCE);
+    cl->CopyTextureRegion(&dstLoc, 0, 0, 0, &srcLoc, nullptr);
+    if (needsBarrier)
+        transitionResource(rtRes, cl, D3D12_RESOURCE_STATE_COPY_SOURCE, bstate);
+
+    cl->Close();
+    ID3D12CommandList *commandLists[] = { cl };
+    commandQueue->ExecuteCommandLists(_countof(commandLists), commandLists);
+
+    QScopedPointer<QSGD3D12CPUWaitableFence> f(createCPUWaitableFence());
+    waitForGPU(f.data()); // uh oh
+
+    QImage::Format fmt = imageFormatForTexture(rtDesc.Format);
+    if (fmt == QImage::Format_Invalid) {
+        qWarning("Could not map render target format %d to a QImage format", rtDesc.Format);
+        return QImage();
+    }
+    QImage img(rtDesc.Width, rtDesc.Height, fmt);
+    quint8 *p = nullptr;
+    const D3D12_RANGE readRange = { 0, 0 };
+    if (FAILED(readbackBuf->Map(0, &readRange, reinterpret_cast<void **>(&p)))) {
+        qWarning("Mapping the readback buffer failed");
+        return QImage();
+    }
+    for (UINT y = 0; y < rtDesc.Height; ++y) {
+        quint8 *dst = img.scanLine(y);
+        memcpy(dst, p, rtDesc.Width * 4);
+        p += textureLayout.Footprint.RowPitch;
+    }
+    readbackBuf->Unmap(0, nullptr);
+
+    return img;
+}
+
 void *QSGD3D12EnginePrivate::getResource(QSGRendererInterface::Resource resource) const
 {
     switch (resource) {
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
index 6deef9f06b..f117400c25 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
@@ -349,6 +349,8 @@ public:
     void useRenderTargetAsTexture(uint id);
     uint activeRenderTarget() const;
 
+    QImage executeAndWaitReadbackRenderTarget(uint id = 0);
+
     // QSGRendererInterface
     GraphicsAPI graphicsAPI() const override;
     void *getResource(Resource resource) const override;
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
index cedcebd6d2..34e5d2da7c 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
@@ -180,6 +180,8 @@ public:
     void useRenderTargetAsTexture(uint id);
     uint activeRenderTarget() const { return currentRenderTarget; }
 
+    QImage executeAndWaitReadbackRenderTarget(uint id);
+
     void *getResource(QSGRendererInterface::Resource resource) const;
 
     // the device is intentionally hidden here. all resources have to go
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
index 83ba443800..fccb127c4f 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
@@ -367,8 +367,16 @@ bool QSGD3D12RenderThread::event(QEvent *e)
         Q_ASSERT(wme->window == exposedWindow || !exposedWindow);
         mutex.lock();
         if (wme->window) {
-            // ###
-            Q_UNREACHABLE();
+            // Grabbing is generally done by rendering a frame and reading the
+            // color buffer contents back, without presenting, and then
+            // creating a QImage from the returned data. It is terribly
+            // inefficient since it involves a full blocking wait for the GPU.
+            // However, our hands are tied by the existing, synchronous APIs of
+            // QQuickWindow and such.
+            QQuickWindowPrivate *wd = QQuickWindowPrivate::get(wme->window);
+            wd->syncSceneGraph();
+            wd->renderSceneGraph(wme->window->size());
+            *wme->image = engine->executeAndWaitReadbackRenderTarget();
         }
         if (Q_UNLIKELY(debug_loop()))
             qDebug("RT - WM_Grab - waking gui to handle result");
@@ -733,7 +741,14 @@ QImage QSGD3D12RenderLoop::grab(QQuickWindow *window)
         qDebug() << "grab" << window;
 
     WindowData *w = windowFor(windows, window);
-    Q_ASSERT(w);
+    // Have to support invisible (but created()'ed) windows as well.
+    // Unlike with GL, leaving that case for QQuickWindow to handle is not feasible.
+    const bool tempExpose = !w;
+    if (tempExpose) {
+        handleExposure(window);
+        w = windowFor(windows, window);
+        Q_ASSERT(w);
+    }
 
     if (!w->thread->isRunning())
         return QImage();
@@ -752,6 +767,11 @@ QImage QSGD3D12RenderLoop::grab(QQuickWindow *window)
     lockedForSync = false;
     w->thread->mutex.unlock();
 
+    result.setDevicePixelRatio(window->effectiveDevicePixelRatio());
+
+    if (tempExpose)
+        handleObscurity(w);
+
     return result;
 }
 
@@ -841,6 +861,11 @@ bool QSGD3D12RenderLoop::interleaveIncubation() const
     return somethingVisible && anim->isRunning();
 }
 
+int QSGD3D12RenderLoop::flags() const
+{
+    return SupportsGrabWithoutExpose;
+}
+
 bool QSGD3D12RenderLoop::event(QEvent *e)
 {
     if (e->type() == QEvent::Timer) {
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h
index 56dcb1d490..732f8dd5d2 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h
@@ -91,8 +91,8 @@ public:
     void postJob(QQuickWindow *window, QRunnable *job) override;
 
     QSurface::SurfaceType windowSurfaceType() const override;
-
     bool interleaveIncubation() const override;
+    int flags() const override;
 
     bool event(QEvent *e) override;
 
diff --git a/src/quick/items/qquickwindow.cpp b/src/quick/items/qquickwindow.cpp
index 908cbd3a90..5d68646681 100644
--- a/src/quick/items/qquickwindow.cpp
+++ b/src/quick/items/qquickwindow.cpp
@@ -3407,33 +3407,40 @@ QOpenGLFramebufferObject *QQuickWindow::renderTarget() const
 QImage QQuickWindow::grabWindow()
 {
     Q_D(QQuickWindow);
-#ifndef QT_NO_OPENGL
-    auto openglRenderContext = static_cast<QSGDefaultRenderContext *>(d->context);
-    if (!isVisible() && !openglRenderContext->openglContext()) {
 
-        if (!handle() || !size().isValid()) {
-            qWarning("QQuickWindow::grabWindow: window must be created and have a valid size");
-            return QImage();
-        }
+    if (!isVisible() && !d->renderControl) {
+        if (d->windowManager && (d->windowManager->flags() & QSGRenderLoop::SupportsGrabWithoutExpose))
+            return d->windowManager->grab(this);
+    }
 
-        QOpenGLContext context;
-        context.setFormat(requestedFormat());
-        context.setShareContext(qt_gl_global_share_context());
-        context.create();
-        context.makeCurrent(this);
-        d->context->initialize(&context);
+#ifndef QT_NO_OPENGL
+    if (!isVisible() && !d->renderControl) {
+        auto openglRenderContext = static_cast<QSGDefaultRenderContext *>(d->context);
+        if (!openglRenderContext->openglContext()) {
+            if (!handle() || !size().isValid()) {
+                qWarning("QQuickWindow::grabWindow: window must be created and have a valid size");
+                return QImage();
+            }
+
+            QOpenGLContext context;
+            context.setFormat(requestedFormat());
+            context.setShareContext(qt_gl_global_share_context());
+            context.create();
+            context.makeCurrent(this);
+            d->context->initialize(&context);
 
-        d->polishItems();
-        d->syncSceneGraph();
-        d->renderSceneGraph(size());
+            d->polishItems();
+            d->syncSceneGraph();
+            d->renderSceneGraph(size());
 
-        bool alpha = format().alphaBufferSize() > 0 && color().alpha() < 255;
-        QImage image = qt_gl_read_framebuffer(size() * effectiveDevicePixelRatio(), alpha, alpha);
-        d->cleanupNodesOnShutdown();
-        d->context->invalidate();
-        context.doneCurrent();
+            bool alpha = format().alphaBufferSize() > 0 && color().alpha() < 255;
+            QImage image = qt_gl_read_framebuffer(size() * effectiveDevicePixelRatio(), alpha, alpha);
+            d->cleanupNodesOnShutdown();
+            d->context->invalidate();
+            context.doneCurrent();
 
-        return image;
+            return image;
+        }
     }
 #endif
     if (d->renderControl)
diff --git a/src/quick/scenegraph/qsgrenderloop_p.h b/src/quick/scenegraph/qsgrenderloop_p.h
index 9159a2f01d..7bf40ecac4 100644
--- a/src/quick/scenegraph/qsgrenderloop_p.h
+++ b/src/quick/scenegraph/qsgrenderloop_p.h
@@ -69,6 +69,10 @@ class Q_QUICK_PRIVATE_EXPORT QSGRenderLoop : public QObject
     Q_OBJECT
 
 public:
+    enum RenderLoopFlags {
+        SupportsGrabWithoutExpose = 0x01
+    };
+
     virtual ~QSGRenderLoop();
 
     virtual void show(QQuickWindow *window) = 0;
@@ -104,6 +108,8 @@ public:
 
     virtual bool interleaveIncubation() const { return false; }
 
+    virtual int flags() const { return 0; }
+
     static void cleanup();
 
 Q_SIGNALS:
diff --git a/tests/manual/nodetypes/main.qml b/tests/manual/nodetypes/main.qml
index 87d42da3bb..2ec2d37f14 100644
--- a/tests/manual/nodetypes/main.qml
+++ b/tests/manual/nodetypes/main.qml
@@ -74,5 +74,8 @@ Item {
 
         if (event.key === Qt.Key_P)
             loader.source = "qrc:/Painter.qml";
+
+        if (event.key === Qt.Key_G)
+            helper.testGrab()
     }
 }
diff --git a/tests/manual/nodetypes/nodetypes.cpp b/tests/manual/nodetypes/nodetypes.cpp
index ac54e069ed..8dc282894f 100644
--- a/tests/manual/nodetypes/nodetypes.cpp
+++ b/tests/manual/nodetypes/nodetypes.cpp
@@ -151,9 +151,19 @@ class Helper : public QObject
     Q_OBJECT
 
 public:
+    Helper(QQuickWindow *w) : m_window(w) { }
+
     Q_INVOKABLE void sleep(int ms) {
         QThread::msleep(ms);
     }
+
+    Q_INVOKABLE void testGrab() {
+        QImage img = m_window->grabWindow();
+        qDebug() << "Saving image to grab_result.png" << img;
+        img.save("grab_result.png");
+    }
+
+    QQuickWindow *m_window;
 };
 
 int main(int argc, char **argv)
@@ -171,10 +181,11 @@ int main(int argc, char **argv)
     qDebug("  [L] - Layers");
     qDebug("  [E] - Effects");
     qDebug("  [P] - QQuickPaintedItem");
+    qDebug("  [G] - Grab current window");
     qDebug("\nPress S to stop the currently running test\n");
 
-    Helper helper;
     QQuickView view;
+    Helper helper(&view);
     if (app.arguments().contains(QLatin1String("--multisample"))) {
         qDebug("Requesting sample count 4");
         QSurfaceFormat fmt;