D3D12: Implement grabWindow

Change-Id: Icb8151f26bad68795eb2e1f920297267c880b40b
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

5 files changed, 175 insertions, 5 deletions

diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
index 7445a8bb40..c77e86ca99 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine.cpp
@@ -515,6 +515,11 @@ uint QSGD3D12Engine::activeRenderTarget() const
     return d->activeRenderTarget();
 }
 
+QImage QSGD3D12Engine::executeAndWaitReadbackRenderTarget(uint id)
+{
+    return d->executeAndWaitReadbackRenderTarget(id);
+}
+
 QSGRendererInterface::GraphicsAPI QSGD3D12Engine::graphicsAPI() const
 {
     return Direct3D12;
@@ -1748,7 +1753,12 @@ void QSGD3D12EnginePrivate::queueSetRenderTarget(uint id)
         RenderTarget &rt(renderTargets[idx]);
         rtvHandle = rt.rtv;
         dsvHandle = rt.dsv;
-        rt.flags |= RenderTarget::NeedsReadBarrier;
+        if (!(rt.flags & RenderTarget::NeedsReadBarrier)) {
+            rt.flags |= RenderTarget::NeedsReadBarrier;
+            if (!(rt.flags & RenderTarget::Multisample))
+                transitionResource(rt.color.Get(), commandList, D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE,
+                                   D3D12_RESOURCE_STATE_RENDER_TARGET);
+        }
     }
 
     commandList->OMSetRenderTargets(1, &rtvHandle, FALSE, &dsvHandle);
@@ -2163,6 +2173,27 @@ static inline DXGI_FORMAT textureFormat(QImage::Format format, bool wantsAlpha,
     return f;
 }
 
+static inline QImage::Format imageFormatForTexture(DXGI_FORMAT format)
+{
+    QImage::Format f = QImage::Format_Invalid;
+
+    switch (format) {
+    case DXGI_FORMAT_R8G8B8A8_UNORM:
+        f = QImage::Format_RGBA8888_Premultiplied;
+        break;
+    case DXGI_FORMAT_B8G8R8A8_UNORM:
+        f = QImage::Format_ARGB32_Premultiplied;
+        break;
+    case DXGI_FORMAT_R8_UNORM:
+        f = QImage::Format_Grayscale8;
+        break;
+    default:
+        break;
+    }
+
+    return f;
+}
+
 void QSGD3D12EnginePrivate::createTexture(uint id, const QSize &size, QImage::Format format,
                                           QSGD3D12Engine::TextureCreateFlags createFlags)
 {
@@ -2782,6 +2813,116 @@ void QSGD3D12EnginePrivate::useRenderTargetAsTexture(uint id)
     tframeData.activeTextures.append(TransientFrameData::ActiveTexture::ActiveTexture(TransientFrameData::ActiveTexture::TypeRenderTarget, id));
 }
 
+QImage QSGD3D12EnginePrivate::executeAndWaitReadbackRenderTarget(uint id)
+{
+    if (inFrame) {
+        qWarning("%s: Cannot be called while frame preparation is active", __FUNCTION__);
+        return QImage();
+    }
+
+    frameCommandList->Reset(frameCommandAllocator[frameIndex % frameInFlightCount].Get(), nullptr);
+
+    D3D12_RESOURCE_STATES bstate;
+    bool needsBarrier = false;
+    ID3D12Resource *rtRes;
+    if (id == 0) {
+        const int idx = presentFrameIndex % swapChainBufferCount;
+        if (windowSamples > 1) {
+            resolveMultisampledTarget(defaultRT[idx].Get(), backBufferRT[idx].Get(),
+                                      D3D12_RESOURCE_STATE_COPY_SOURCE, frameCommandList.Get());
+        } else {
+            bstate = D3D12_RESOURCE_STATE_PRESENT;
+            needsBarrier = true;
+        }
+        rtRes = backBufferRT[idx].Get();
+    } else {
+        const int idx = id - 1;
+        Q_ASSERT(idx < renderTargets.count());
+        RenderTarget &rt(renderTargets[idx]);
+        Q_ASSERT(rt.entryInUse() && rt.color);
+
+        if (rt.flags & RenderTarget::Multisample) {
+            resolveMultisampledTarget(rt.color.Get(), rt.colorResolve.Get(),
+                                      D3D12_RESOURCE_STATE_COPY_SOURCE, frameCommandList.Get());
+            rtRes = rt.colorResolve.Get();
+        } else {
+            rtRes = rt.color.Get();
+            bstate = D3D12_RESOURCE_STATE_PIXEL_SHADER_RESOURCE;
+            needsBarrier = true;
+        }
+    }
+
+    ComPtr<ID3D12Resource> readbackBuf;
+
+    D3D12_RESOURCE_DESC rtDesc = rtRes->GetDesc();
+    UINT64 textureByteSize = 0;
+    D3D12_PLACED_SUBRESOURCE_FOOTPRINT textureLayout = {};
+    device->GetCopyableFootprints(&rtDesc, 0, 1, 0, &textureLayout, nullptr, nullptr, &textureByteSize);
+
+    D3D12_HEAP_PROPERTIES heapProp = {};
+    heapProp.Type = D3D12_HEAP_TYPE_READBACK;
+
+    D3D12_RESOURCE_DESC bufDesc = {};
+    bufDesc.Dimension = D3D12_RESOURCE_DIMENSION_BUFFER;
+    bufDesc.Width = textureByteSize;
+    bufDesc.Height = 1;
+    bufDesc.DepthOrArraySize = 1;
+    bufDesc.MipLevels = 1;
+    bufDesc.Format = DXGI_FORMAT_UNKNOWN;
+    bufDesc.SampleDesc.Count = 1;
+    bufDesc.Layout = D3D12_TEXTURE_LAYOUT_ROW_MAJOR;
+
+    if (FAILED(device->CreateCommittedResource(&heapProp, D3D12_HEAP_FLAG_NONE, &bufDesc,
+                                               D3D12_RESOURCE_STATE_COPY_DEST, nullptr, IID_PPV_ARGS(&readbackBuf)))) {
+        qWarning("Failed to create committed resource (readback buffer)");
+        return QImage();
+    }
+
+    D3D12_TEXTURE_COPY_LOCATION dstLoc;
+    dstLoc.pResource = readbackBuf.Get();
+    dstLoc.Type = D3D12_TEXTURE_COPY_TYPE_PLACED_FOOTPRINT;
+    dstLoc.PlacedFootprint = textureLayout;
+    D3D12_TEXTURE_COPY_LOCATION srcLoc;
+    srcLoc.pResource = rtRes;
+    srcLoc.Type = D3D12_TEXTURE_COPY_TYPE_SUBRESOURCE_INDEX;
+    srcLoc.SubresourceIndex = 0;
+
+    ID3D12GraphicsCommandList *cl = frameCommandList.Get();
+    if (needsBarrier)
+        transitionResource(rtRes, cl, bstate, D3D12_RESOURCE_STATE_COPY_SOURCE);
+    cl->CopyTextureRegion(&dstLoc, 0, 0, 0, &srcLoc, nullptr);
+    if (needsBarrier)
+        transitionResource(rtRes, cl, D3D12_RESOURCE_STATE_COPY_SOURCE, bstate);
+
+    cl->Close();
+    ID3D12CommandList *commandLists[] = { cl };
+    commandQueue->ExecuteCommandLists(_countof(commandLists), commandLists);
+
+    QScopedPointer<QSGD3D12CPUWaitableFence> f(createCPUWaitableFence());
+    waitForGPU(f.data()); // uh oh
+
+    QImage::Format fmt = imageFormatForTexture(rtDesc.Format);
+    if (fmt == QImage::Format_Invalid) {
+        qWarning("Could not map render target format %d to a QImage format", rtDesc.Format);
+        return QImage();
+    }
+    QImage img(rtDesc.Width, rtDesc.Height, fmt);
+    quint8 *p = nullptr;
+    const D3D12_RANGE readRange = { 0, 0 };
+    if (FAILED(readbackBuf->Map(0, &readRange, reinterpret_cast<void **>(&p)))) {
+        qWarning("Mapping the readback buffer failed");
+        return QImage();
+    }
+    for (UINT y = 0; y < rtDesc.Height; ++y) {
+        quint8 *dst = img.scanLine(y);
+        memcpy(dst, p, rtDesc.Width * 4);
+        p += textureLayout.Footprint.RowPitch;
+    }
+    readbackBuf->Unmap(0, nullptr);
+
+    return img;
+}
+
 void *QSGD3D12EnginePrivate::getResource(QSGRendererInterface::Resource resource) const
 {
     switch (resource) {
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
index 6deef9f06b..f117400c25 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p.h
@@ -349,6 +349,8 @@ public:
     void useRenderTargetAsTexture(uint id);
     uint activeRenderTarget() const;
 
+    QImage executeAndWaitReadbackRenderTarget(uint id = 0);
+
     // QSGRendererInterface
     GraphicsAPI graphicsAPI() const override;
     void *getResource(Resource resource) const override;
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
index cedcebd6d2..34e5d2da7c 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12engine_p_p.h
@@ -180,6 +180,8 @@ public:
     void useRenderTargetAsTexture(uint id);
     uint activeRenderTarget() const { return currentRenderTarget; }
 
+    QImage executeAndWaitReadbackRenderTarget(uint id);
+
     void *getResource(QSGRendererInterface::Resource resource) const;
 
     // the device is intentionally hidden here. all resources have to go
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
index 83ba443800..fccb127c4f 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop.cpp
@@ -367,8 +367,16 @@ bool QSGD3D12RenderThread::event(QEvent *e)
         Q_ASSERT(wme->window == exposedWindow || !exposedWindow);
         mutex.lock();
         if (wme->window) {
-            // ###
-            Q_UNREACHABLE();
+            // Grabbing is generally done by rendering a frame and reading the
+            // color buffer contents back, without presenting, and then
+            // creating a QImage from the returned data. It is terribly
+            // inefficient since it involves a full blocking wait for the GPU.
+            // However, our hands are tied by the existing, synchronous APIs of
+            // QQuickWindow and such.
+            QQuickWindowPrivate *wd = QQuickWindowPrivate::get(wme->window);
+            wd->syncSceneGraph();
+            wd->renderSceneGraph(wme->window->size());
+            *wme->image = engine->executeAndWaitReadbackRenderTarget();
         }
         if (Q_UNLIKELY(debug_loop()))
             qDebug("RT - WM_Grab - waking gui to handle result");
@@ -733,7 +741,14 @@ QImage QSGD3D12RenderLoop::grab(QQuickWindow *window)
         qDebug() << "grab" << window;
 
     WindowData *w = windowFor(windows, window);
-    Q_ASSERT(w);
+    // Have to support invisible (but created()'ed) windows as well.
+    // Unlike with GL, leaving that case for QQuickWindow to handle is not feasible.
+    const bool tempExpose = !w;
+    if (tempExpose) {
+        handleExposure(window);
+        w = windowFor(windows, window);
+        Q_ASSERT(w);
+    }
 
     if (!w->thread->isRunning())
         return QImage();
@@ -752,6 +767,11 @@ QImage QSGD3D12RenderLoop::grab(QQuickWindow *window)
     lockedForSync = false;
     w->thread->mutex.unlock();
 
+    result.setDevicePixelRatio(window->effectiveDevicePixelRatio());
+
+    if (tempExpose)
+        handleObscurity(w);
+
     return result;
 }
 
@@ -841,6 +861,11 @@ bool QSGD3D12RenderLoop::interleaveIncubation() const
     return somethingVisible && anim->isRunning();
 }
 
+int QSGD3D12RenderLoop::flags() const
+{
+    return SupportsGrabWithoutExpose;
+}
+
 bool QSGD3D12RenderLoop::event(QEvent *e)
 {
     if (e->type() == QEvent::Timer) {
diff --git a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h
index 56dcb1d490..732f8dd5d2 100644
--- a/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h
+++ b/src/plugins/scenegraph/d3d12/qsgd3d12renderloop_p.h
@@ -91,8 +91,8 @@ public:
     void postJob(QQuickWindow *window, QRunnable *job) override;
 
     QSurface::SurfaceType windowSurfaceType() const override;
-
     bool interleaveIncubation() const override;
+    int flags() const override;
 
     bool event(QEvent *e) override;