rhi: vk: Alter command buffer strategy

Reports on the Web have it that there's nothing guaranteeing a
driver does any actual freeing of resources then doing
vkFreeCommandBuffer for a command buffer from a command pool that
does not have VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT, thus
leading to continuously growing resource usage with our current
allocate/free pattern. It could be that this is the source of out
of memory problems we are seeing on some Android devices.

Instead of just going straight for said command pool flag and doing
ResetCommandBuffer for the command buffers individually, take one
step further and use per-slot (slot being 0 or 1 if QVK_FRAMES_IN_FLIGHT
is 2) command pools. The current pool is reset in each
beginFrame/beginOffscreenFrame, moving all allocated command buffers
to the initial state (while other command buffers with the other pool
are not affected).

This may be (while impossible to tell from just guessing based on the
spec) our best approach to command buffer allocation since a Vulkan
implementation can likely just use some simple per pool allocator,
knowing that we never want to free or reset individual command buffers,
but we rather only reset the whole pool at once.

The option of importing an existing VkCommandPool when creating the
QRhi instance is now gone, but there was probably no point in offering
that in the first place.

When it comes to VK_COMMAND_POOL_RESET_RELEASE_RESOURCES_BIT it will
not be set unless releaseCachedResources() (in Qt Quick this is hooked
into QQuickWindow::releaseResources()) was called. What this does in
practice is unknown, but have an option to set it now and then if the
application really wants.

While we are at it, rename secondaryCbs to activeSecondaryCbStack to
indicate what it really is. (it's a stack as each call to
beginExternal() pushes a new one, while each endExternal() pops)

Change-Id: I2e5c1fad26d794e1f56c778e38f750998d706d84
Reviewed-by: Andy Nichols <andy.nichols@qt.io>

3 files changed, 158 insertions, 131 deletions

diff --git a/src/gui/rhi/qrhivulkan.cpp b/src/gui/rhi/qrhivulkan.cpp
index d6b1bc0670..bb4b94c6b3 100644
--- a/src/gui/rhi/qrhivulkan.cpp
+++ b/src/gui/rhi/qrhivulkan.cpp
@@ -323,10 +323,6 @@ QRhiVulkan::QRhiVulkan(QRhiVulkanInitParams *params, QRhiVulkanNativeHandles *im
             gfxQueueFamilyIdx = importParams->gfxQueueFamilyIdx;
             gfxQueueIdx = importParams->gfxQueueIdx;
             // gfxQueue is output only, no point in accepting it as input
-            if (importParams->cmdPool) {
-                importedCmdPool = true;
-                cmdPool = importParams->cmdPool;
-            }
             if (importParams->vmemAllocator) {
                 importedAllocator = true;
                 allocator = importParams->vmemAllocator;
@@ -590,12 +586,12 @@ bool QRhiVulkan::create(QRhi::Flags flags)
 
     df = inst->deviceFunctions(dev);
 
-    if (!importedCmdPool) {
-        VkCommandPoolCreateInfo poolInfo;
-        memset(&poolInfo, 0, sizeof(poolInfo));
-        poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
-        poolInfo.queueFamilyIndex = uint32_t(gfxQueueFamilyIdx);
-        VkResult err = df->vkCreateCommandPool(dev, &poolInfo, nullptr, &cmdPool);
+    VkCommandPoolCreateInfo poolInfo;
+    memset(&poolInfo, 0, sizeof(poolInfo));
+    poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+    poolInfo.queueFamilyIndex = uint32_t(gfxQueueFamilyIdx);
+    for (int i = 0; i < QVK_FRAMES_IN_FLIGHT; ++i) {
+        VkResult err = df->vkCreateCommandPool(dev, &poolInfo, nullptr, &cmdPool[i]);
         if (err != VK_SUCCESS) {
             qWarning("Failed to create command pool: %d", err);
             return false;
@@ -695,7 +691,6 @@ bool QRhiVulkan::create(QRhi::Flags flags)
     nativeHandlesStruct.gfxQueueFamilyIdx = gfxQueueFamilyIdx;
     nativeHandlesStruct.gfxQueueIdx = gfxQueueIdx;
     nativeHandlesStruct.gfxQueue = gfxQueue;
-    nativeHandlesStruct.cmdPool = cmdPool;
     nativeHandlesStruct.vmemAllocator = allocator;
 
     return true;
@@ -717,11 +712,6 @@ void QRhiVulkan::destroy()
         ofr.cmdFence = VK_NULL_HANDLE;
     }
 
-    if (ofr.cbWrapper.cb) {
-        df->vkFreeCommandBuffers(dev, cmdPool, 1, &ofr.cbWrapper.cb);
-        ofr.cbWrapper.cb = VK_NULL_HANDLE;
-    }
-
     if (pipelineCache) {
         df->vkDestroyPipelineCache(dev, pipelineCache, nullptr);
         pipelineCache = VK_NULL_HANDLE;
@@ -742,9 +732,13 @@ void QRhiVulkan::destroy()
         allocator = nullptr;
     }
 
-    if (!importedCmdPool && cmdPool) {
-        df->vkDestroyCommandPool(dev, cmdPool, nullptr);
-        cmdPool = VK_NULL_HANDLE;
+    for (int i = 0; i < QVK_FRAMES_IN_FLIGHT; ++i) {
+        if (cmdPool[i]) {
+            df->vkDestroyCommandPool(dev, cmdPool[i], nullptr);
+            cmdPool[i] = VK_NULL_HANDLE;
+        }
+        freeSecondaryCbs[i].clear();
+        ofr.cbWrapper[i]->cb = VK_NULL_HANDLE;
     }
 
     if (!importedDevice && dev) {
@@ -1587,10 +1581,6 @@ void QRhiVulkan::releaseSwapChainResources(QRhiSwapChain *swapChain)
             df->vkDestroySemaphore(dev, frame.drawSem, nullptr);
             frame.drawSem = VK_NULL_HANDLE;
         }
-        if (frame.cmdBuf) {
-            df->vkFreeCommandBuffers(dev, cmdPool, 1, &frame.cmdBuf);
-            frame.cmdBuf = VK_NULL_HANDLE;
-        }
     }
 
     for (int i = 0; i < swapChainD->bufferCount; ++i) {
@@ -1624,6 +1614,20 @@ void QRhiVulkan::releaseSwapChainResources(QRhiSwapChain *swapChain)
     // NB! surface and similar must remain intact
 }
 
+void QRhiVulkan::ensureCommandPoolForNewFrame()
+{
+    VkCommandPoolResetFlags flags = 0;
+
+    // While not clear what "recycles all of the resources from the command
+    // pool back to the system" really means in practice, set it when there was
+    // a call to releaseCachedResources() recently.
+    if (releaseCachedResourcesCalledBeforeFrameStart)
+        flags |= VK_COMMAND_POOL_RESET_RELEASE_RESOURCES_BIT;
+
+    // put all command buffers allocated from this slot's pool to initial state
+    df->vkResetCommandPool(dev, cmdPool[currentFrameSlot], 0);
+}
+
 QRhi::FrameOpResult QRhiVulkan::beginFrame(QRhiSwapChain *swapChain, QRhi::BeginFrameFlags)
 {
     QVkSwapChain *swapChainD = QRHI_RES(QVkSwapChain, swapChain);
@@ -1641,10 +1645,11 @@ QRhi::FrameOpResult QRhiVulkan::beginFrame(QRhiSwapChain *swapChain, QRhi::Begin
         }
 
         // move on to next swapchain image
+        uint32_t imageIndex = 0;
         VkResult err = vkAcquireNextImageKHR(dev, swapChainD->sc, UINT64_MAX,
-                                             frame.imageSem, frame.imageFence, &frame.imageIndex);
+                                             frame.imageSem, frame.imageFence, &imageIndex);
         if (err == VK_SUCCESS || err == VK_SUBOPTIMAL_KHR) {
-            swapChainD->currentImageIndex = frame.imageIndex;
+            swapChainD->currentImageIndex = imageIndex;
             frame.imageSemWaitable = true;
             frame.imageAcquired = true;
             frame.imageFenceWaitable = true;
@@ -1699,7 +1704,15 @@ QRhi::FrameOpResult QRhiVulkan::beginFrame(QRhiSwapChain *swapChain, QRhi::Begin
         }
     }
 
-    // build new draw command buffer
+    currentFrameSlot = int(swapChainD->currentFrameSlot);
+    currentSwapChain = swapChainD;
+    if (swapChainD->ds)
+        swapChainD->ds->lastActiveFrameSlot = currentFrameSlot;
+
+    // reset the command pool
+    ensureCommandPoolForNewFrame();
+
+    // start recording to this frame's command buffer
     QRhi::FrameOpResult cbres = startPrimaryCommandBuffer(&frame.cmdBuf);
     if (cbres != QRhi::FrameOpSuccess)
         return cbres;
@@ -1728,11 +1741,6 @@ QRhi::FrameOpResult QRhiVulkan::beginFrame(QRhiSwapChain *swapChain, QRhi::Begin
     QVkSwapChain::ImageResources &image(swapChainD->imageRes[swapChainD->currentImageIndex]);
     swapChainD->rtWrapper.d.fb = image.fb;
 
-    currentFrameSlot = int(swapChainD->currentFrameSlot);
-    currentSwapChain = swapChainD;
-    if (swapChainD->ds)
-        swapChainD->ds->lastActiveFrameSlot = currentFrameSlot;
-
     QRHI_PROF_F(beginSwapChainFrame(swapChain));
 
     prepareNewFrame(&swapChainD->cbWrapper);
@@ -1868,38 +1876,37 @@ void QRhiVulkan::prepareNewFrame(QRhiCommandBuffer *cb)
     QRHI_RES(QVkCommandBuffer, cb)->resetState();
 
     finishActiveReadbacks(); // last, in case the readback-completed callback issues rhi calls
+
+    releaseCachedResourcesCalledBeforeFrameStart = false;
 }
 
 QRhi::FrameOpResult QRhiVulkan::startPrimaryCommandBuffer(VkCommandBuffer *cb)
 {
-    if (*cb) {
-        df->vkFreeCommandBuffers(dev, cmdPool, 1, cb);
-        *cb = VK_NULL_HANDLE;
-    }
-
-    VkCommandBufferAllocateInfo cmdBufInfo;
-    memset(&cmdBufInfo, 0, sizeof(cmdBufInfo));
-    cmdBufInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
-    cmdBufInfo.commandPool = cmdPool;
-    cmdBufInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
-    cmdBufInfo.commandBufferCount = 1;
+    if (!*cb) {
+        VkCommandBufferAllocateInfo cmdBufInfo;
+        memset(&cmdBufInfo, 0, sizeof(cmdBufInfo));
+        cmdBufInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+        cmdBufInfo.commandPool = cmdPool[currentFrameSlot];
+        cmdBufInfo.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+        cmdBufInfo.commandBufferCount = 1;
 
-    VkResult err = df->vkAllocateCommandBuffers(dev, &cmdBufInfo, cb);
-    if (err != VK_SUCCESS) {
-        if (err == VK_ERROR_DEVICE_LOST) {
-            qWarning("Device loss detected in vkAllocateCommandBuffers()");
-            deviceLost = true;
-            return QRhi::FrameOpDeviceLost;
+        VkResult err = df->vkAllocateCommandBuffers(dev, &cmdBufInfo, cb);
+        if (err != VK_SUCCESS) {
+            if (err == VK_ERROR_DEVICE_LOST) {
+                qWarning("Device loss detected in vkAllocateCommandBuffers()");
+                deviceLost = true;
+                return QRhi::FrameOpDeviceLost;
+            }
+            qWarning("Failed to allocate frame command buffer: %d", err);
+            return QRhi::FrameOpError;
         }
-        qWarning("Failed to allocate frame command buffer: %d", err);
-        return QRhi::FrameOpError;
     }
 
     VkCommandBufferBeginInfo cmdBufBeginInfo;
     memset(&cmdBufBeginInfo, 0, sizeof(cmdBufBeginInfo));
     cmdBufBeginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
 
-    err = df->vkBeginCommandBuffer(*cb, &cmdBufBeginInfo);
+    VkResult err = df->vkBeginCommandBuffer(*cb, &cmdBufBeginInfo);
     if (err != VK_SUCCESS) {
         if (err == VK_ERROR_DEVICE_LOST) {
             qWarning("Device loss detected in vkBeginCommandBuffer()");
@@ -1972,10 +1979,6 @@ void QRhiVulkan::waitCommandCompletion(int frameSlot)
 
 QRhi::FrameOpResult QRhiVulkan::beginOffscreenFrame(QRhiCommandBuffer **cb, QRhi::BeginFrameFlags)
 {
-    QRhi::FrameOpResult cbres = startPrimaryCommandBuffer(&ofr.cbWrapper.cb);
-    if (cbres != QRhi::FrameOpSuccess)
-        return cbres;
-
     // Switch to the next slot manually. Swapchains do not know about this
     // which is good. So for example a - unusual but possible - onscreen,
     // onscreen, offscreen, onscreen, onscreen, onscreen sequence of
@@ -1989,10 +1992,17 @@ QRhi::FrameOpResult QRhiVulkan::beginOffscreenFrame(QRhiCommandBuffer **cb, QRhi
     if (swapchains.count() > 1)
         waitCommandCompletion(currentFrameSlot);
 
-    prepareNewFrame(&ofr.cbWrapper);
+    ensureCommandPoolForNewFrame();
+
+    QVkCommandBuffer *cbWrapper = ofr.cbWrapper[currentFrameSlot];
+    QRhi::FrameOpResult cbres = startPrimaryCommandBuffer(&cbWrapper->cb);
+    if (cbres != QRhi::FrameOpSuccess)
+        return cbres;
+
+    prepareNewFrame(cbWrapper);
     ofr.active = true;
 
-    *cb = &ofr.cbWrapper;
+    *cb = cbWrapper;
     return QRhi::FrameOpSuccess;
 }
 
@@ -2002,7 +2012,8 @@ QRhi::FrameOpResult QRhiVulkan::endOffscreenFrame(QRhi::EndFrameFlags flags)
     Q_ASSERT(ofr.active);
     ofr.active = false;
 
-    recordPrimaryCommandBuffer(&ofr.cbWrapper);
+    QVkCommandBuffer *cbWrapper(ofr.cbWrapper[currentFrameSlot]);
+    recordPrimaryCommandBuffer(cbWrapper);
 
     if (!ofr.cmdFence) {
         VkFenceCreateInfo fenceInfo;
@@ -2015,7 +2026,7 @@ QRhi::FrameOpResult QRhiVulkan::endOffscreenFrame(QRhi::EndFrameFlags flags)
         }
     }
 
-    QRhi::FrameOpResult submitres = endAndSubmitPrimaryCommandBuffer(ofr.cbWrapper.cb, ofr.cmdFence, nullptr, nullptr);
+    QRhi::FrameOpResult submitres = endAndSubmitPrimaryCommandBuffer(cbWrapper->cb, ofr.cmdFence, nullptr, nullptr);
     if (submitres != QRhi::FrameOpSuccess)
         return submitres;
 
@@ -2039,10 +2050,11 @@ QRhi::FrameOpResult QRhiVulkan::finish()
         VkCommandBuffer cb;
         if (ofr.active) {
             Q_ASSERT(!currentSwapChain);
-            Q_ASSERT(ofr.cbWrapper.recordingPass == QVkCommandBuffer::NoPass);
-            recordPrimaryCommandBuffer(&ofr.cbWrapper);
-            ofr.cbWrapper.resetCommands();
-            cb = ofr.cbWrapper.cb;
+            QVkCommandBuffer *cbWrapper(ofr.cbWrapper[currentFrameSlot]);
+            Q_ASSERT(cbWrapper->recordingPass == QVkCommandBuffer::NoPass);
+            recordPrimaryCommandBuffer(cbWrapper);
+            cbWrapper->resetCommands();
+            cb = cbWrapper->cb;
         } else {
             Q_ASSERT(currentSwapChain);
             Q_ASSERT(currentSwapChain->cbWrapper.recordingPass == QVkCommandBuffer::NoPass);
@@ -2059,9 +2071,11 @@ QRhi::FrameOpResult QRhiVulkan::finish()
     df->vkQueueWaitIdle(gfxQueue);
 
     if (inFrame) {
+        // The current frame slot's command pool needs to be reset.
+        ensureCommandPoolForNewFrame();
         // Allocate and begin recording on a new command buffer.
         if (ofr.active) {
-            startPrimaryCommandBuffer(&ofr.cbWrapper.cb);
+            startPrimaryCommandBuffer(&ofr.cbWrapper[currentFrameSlot]->cb);
         } else {
             QVkSwapChain::FrameResources &frame(swapChainD->frameRes[swapChainD->currentFrameSlot]);
             startPrimaryCommandBuffer(&frame.cmdBuf);
@@ -2143,16 +2157,22 @@ VkCommandBuffer QRhiVulkan::startSecondaryCommandBuffer(QVkRenderTargetData *rtD
 {
     VkCommandBuffer secondaryCb;
 
-    VkCommandBufferAllocateInfo cmdBufInfo;
-    memset(&cmdBufInfo, 0, sizeof(cmdBufInfo));
-    cmdBufInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
-    cmdBufInfo.commandPool = cmdPool;
-    cmdBufInfo.level = VK_COMMAND_BUFFER_LEVEL_SECONDARY;
-    cmdBufInfo.commandBufferCount = 1;
-    VkResult err = df->vkAllocateCommandBuffers(dev, &cmdBufInfo, &secondaryCb);
-    if (err != VK_SUCCESS) {
-        qWarning("Failed to create secondary command buffer: %d", err);
-        return VK_NULL_HANDLE;
+    if (!freeSecondaryCbs[currentFrameSlot].isEmpty()) {
+        secondaryCb = freeSecondaryCbs[currentFrameSlot].last();
+        freeSecondaryCbs[currentFrameSlot].removeLast();
+    } else {
+        VkCommandBufferAllocateInfo cmdBufInfo;
+        memset(&cmdBufInfo, 0, sizeof(cmdBufInfo));
+        cmdBufInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+        cmdBufInfo.commandPool = cmdPool[currentFrameSlot];
+        cmdBufInfo.level = VK_COMMAND_BUFFER_LEVEL_SECONDARY;
+        cmdBufInfo.commandBufferCount = 1;
+
+        VkResult err = df->vkAllocateCommandBuffers(dev, &cmdBufInfo, &secondaryCb);
+        if (err != VK_SUCCESS) {
+            qWarning("Failed to create secondary command buffer: %d", err);
+            return VK_NULL_HANDLE;
+        }
     }
 
     VkCommandBufferBeginInfo cmdBufBeginInfo;
@@ -2169,10 +2189,9 @@ VkCommandBuffer QRhiVulkan::startSecondaryCommandBuffer(QVkRenderTargetData *rtD
     }
     cmdBufBeginInfo.pInheritanceInfo = &cmdBufInheritInfo;
 
-    err = df->vkBeginCommandBuffer(secondaryCb, &cmdBufBeginInfo);
+    VkResult err = df->vkBeginCommandBuffer(secondaryCb, &cmdBufBeginInfo);
     if (err != VK_SUCCESS) {
         qWarning("Failed to begin secondary command buffer: %d", err);
-        df->vkFreeCommandBuffers(dev, cmdPool, 1, &secondaryCb);
         return VK_NULL_HANDLE;
     }
 
@@ -2190,15 +2209,10 @@ void QRhiVulkan::endAndEnqueueSecondaryCommandBuffer(VkCommandBuffer cb, QVkComm
     cmd.args.executeSecondary.cb = cb;
     cbD->commands.append(cmd);
 
-    deferredReleaseSecondaryCommandBuffer(cb);
-}
-
-void QRhiVulkan::deferredReleaseSecondaryCommandBuffer(VkCommandBuffer cb)
-{
     QRhiVulkan::DeferredReleaseEntry e;
-    e.type = QRhiVulkan::DeferredReleaseEntry::CommandBuffer;
+    e.type = QRhiVulkan::DeferredReleaseEntry::SecondaryCommandBuffer;
     e.lastActiveFrameSlot = currentFrameSlot;
-    e.commandBuffer.cb = cb;
+    e.secondaryCommandBuffer.cb = cb;
     releaseQueue.append(e);
 }
 
@@ -2286,7 +2300,7 @@ void QRhiVulkan::beginPass(QRhiCommandBuffer *cb,
     cbD->commands.append(cmd);
 
     if (cbD->passUsesSecondaryCb)
-        cbD->secondaryCbs.append(startSecondaryCommandBuffer(rtD));
+        cbD->activeSecondaryCbStack.append(startSecondaryCommandBuffer(rtD));
 }
 
 void QRhiVulkan::endPass(QRhiCommandBuffer *cb, QRhiResourceUpdateBatch *resourceUpdates)
@@ -2295,8 +2309,8 @@ void QRhiVulkan::endPass(QRhiCommandBuffer *cb, QRhiResourceUpdateBatch *resourc
     Q_ASSERT(cbD->recordingPass == QVkCommandBuffer::RenderPass);
 
     if (cbD->passUsesSecondaryCb) {
-        VkCommandBuffer secondaryCb = cbD->secondaryCbs.last();
-        cbD->secondaryCbs.removeLast();
+        VkCommandBuffer secondaryCb = cbD->activeSecondaryCbStack.last();
+        cbD->activeSecondaryCbStack.removeLast();
         endAndEnqueueSecondaryCommandBuffer(secondaryCb, cbD);
         cbD->resetCachedState();
     }
@@ -2330,7 +2344,7 @@ void QRhiVulkan::beginComputePass(QRhiCommandBuffer *cb,
     cbD->computePassState.reset();
 
     if (cbD->passUsesSecondaryCb)
-        cbD->secondaryCbs.append(startSecondaryCommandBuffer());
+        cbD->activeSecondaryCbStack.append(startSecondaryCommandBuffer());
 }
 
 void QRhiVulkan::endComputePass(QRhiCommandBuffer *cb, QRhiResourceUpdateBatch *resourceUpdates)
@@ -2339,8 +2353,8 @@ void QRhiVulkan::endComputePass(QRhiCommandBuffer *cb, QRhiResourceUpdateBatch *
     Q_ASSERT(cbD->recordingPass == QVkCommandBuffer::ComputePass);
 
     if (cbD->passUsesSecondaryCb) {
-        VkCommandBuffer secondaryCb = cbD->secondaryCbs.last();
-        cbD->secondaryCbs.removeLast();
+        VkCommandBuffer secondaryCb = cbD->activeSecondaryCbStack.last();
+        cbD->activeSecondaryCbStack.removeLast();
         endAndEnqueueSecondaryCommandBuffer(secondaryCb, cbD);
         cbD->resetCachedState();
     }
@@ -2360,7 +2374,7 @@ void QRhiVulkan::setComputePipeline(QRhiCommandBuffer *cb, QRhiComputePipeline *
 
     if (cbD->currentComputePipeline != ps || cbD->currentPipelineGeneration != psD->generation) {
         if (cbD->passUsesSecondaryCb) {
-            df->vkCmdBindPipeline(cbD->secondaryCbs.last(), VK_PIPELINE_BIND_POINT_COMPUTE, psD->pipeline);
+            df->vkCmdBindPipeline(cbD->activeSecondaryCbStack.last(), VK_PIPELINE_BIND_POINT_COMPUTE, psD->pipeline);
         } else {
             QVkCommandBuffer::Command cmd;
             cmd.cmd = QVkCommandBuffer::Command::BindPipeline;
@@ -2489,7 +2503,7 @@ void QRhiVulkan::dispatch(QRhiCommandBuffer *cb, int x, int y, int z)
     }
 
     if (cbD->passUsesSecondaryCb) {
-        VkCommandBuffer secondaryCb = cbD->secondaryCbs.last();
+        VkCommandBuffer secondaryCb = cbD->activeSecondaryCbStack.last();
         if (!imageBarriers.isEmpty()) {
             df->vkCmdPipelineBarrier(secondaryCb, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT, VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT,
                                      0, 0, nullptr,
@@ -3519,8 +3533,8 @@ void QRhiVulkan::executeDeferredReleases(bool forced)
             case QRhiVulkan::DeferredReleaseEntry::StagingBuffer:
                 vmaDestroyBuffer(toVmaAllocator(allocator), e.stagingBuffer.stagingBuffer, toVmaAllocation(e.stagingBuffer.stagingAllocation));
                 break;
-            case QRhiVulkan::DeferredReleaseEntry::CommandBuffer:
-                df->vkFreeCommandBuffers(dev, cmdPool, 1, &e.commandBuffer.cb);
+            case QRhiVulkan::DeferredReleaseEntry::SecondaryCommandBuffer:
+                freeSecondaryCbs[e.lastActiveFrameSlot].append(e.secondaryCommandBuffer.cb);
                 break;
             default:
                 Q_UNREACHABLE();
@@ -4174,7 +4188,7 @@ bool QRhiVulkan::makeThreadLocalNativeContextCurrent()
 
 void QRhiVulkan::releaseCachedResources()
 {
-    // nothing to do here
+    releaseCachedResourcesCalledBeforeFrameStart = true;
 }
 
 bool QRhiVulkan::isDeviceLost() const
@@ -4232,7 +4246,7 @@ void QRhiVulkan::setGraphicsPipeline(QRhiCommandBuffer *cb, QRhiGraphicsPipeline
 
     if (cbD->currentGraphicsPipeline != ps || cbD->currentPipelineGeneration != psD->generation) {
         if (cbD->passUsesSecondaryCb) {
-            df->vkCmdBindPipeline(cbD->secondaryCbs.last(), VK_PIPELINE_BIND_POINT_GRAPHICS, psD->pipeline);
+            df->vkCmdBindPipeline(cbD->activeSecondaryCbStack.last(), VK_PIPELINE_BIND_POINT_GRAPHICS, psD->pipeline);
         } else {
             QVkCommandBuffer::Command cmd;
             cmd.cmd = QVkCommandBuffer::Command::BindPipeline;
@@ -4440,7 +4454,7 @@ void QRhiVulkan::setShaderResources(QRhiCommandBuffer *cb, QRhiShaderResourceBin
         }
 
         if (cbD->passUsesSecondaryCb) {
-            df->vkCmdBindDescriptorSets(cbD->secondaryCbs.last(),
+            df->vkCmdBindDescriptorSets(cbD->activeSecondaryCbStack.last(),
                                         gfxPsD ? VK_PIPELINE_BIND_POINT_GRAPHICS : VK_PIPELINE_BIND_POINT_COMPUTE,
                                         gfxPsD ? gfxPsD->layout : compPsD->layout,
                                         0, 1, &srbD->descSets[descSetIdx],
@@ -4514,7 +4528,7 @@ void QRhiVulkan::setVertexInput(QRhiCommandBuffer *cb,
         }
 
         if (cbD->passUsesSecondaryCb) {
-            df->vkCmdBindVertexBuffers(cbD->secondaryCbs.last(), uint32_t(startBinding),
+            df->vkCmdBindVertexBuffers(cbD->activeSecondaryCbStack.last(), uint32_t(startBinding),
                                        uint32_t(bufs.count()), bufs.constData(), ofs.constData());
         } else {
             QVkCommandBuffer::Command cmd;
@@ -4550,7 +4564,7 @@ void QRhiVulkan::setVertexInput(QRhiCommandBuffer *cb,
             cbD->currentIndexFormat = type;
 
             if (cbD->passUsesSecondaryCb) {
-                df->vkCmdBindIndexBuffer(cbD->secondaryCbs.last(), vkindexbuf, indexOffset, type);
+                df->vkCmdBindIndexBuffer(cbD->activeSecondaryCbStack.last(), vkindexbuf, indexOffset, type);
             } else {
                 QVkCommandBuffer::Command cmd;
                 cmd.cmd = QVkCommandBuffer::Command::BindIndexBuffer;
@@ -4588,7 +4602,7 @@ void QRhiVulkan::setViewport(QRhiCommandBuffer *cb, const QRhiViewport &viewport
     vp->maxDepth = viewport.maxDepth();
 
     if (cbD->passUsesSecondaryCb) {
-        df->vkCmdSetViewport(cbD->secondaryCbs.last(), 0, 1, vp);
+        df->vkCmdSetViewport(cbD->activeSecondaryCbStack.last(), 0, 1, vp);
     } else {
         cmd.cmd = QVkCommandBuffer::Command::SetViewport;
         cbD->commands.append(cmd);
@@ -4601,7 +4615,7 @@ void QRhiVulkan::setViewport(QRhiCommandBuffer *cb, const QRhiViewport &viewport
         s->extent.width = uint32_t(w);
         s->extent.height = uint32_t(h);
         if (cbD->passUsesSecondaryCb) {
-            df->vkCmdSetScissor(cbD->secondaryCbs.last(), 0, 1, s);
+            df->vkCmdSetScissor(cbD->activeSecondaryCbStack.last(), 0, 1, s);
         } else {
             cmd.cmd = QVkCommandBuffer::Command::SetScissor;
             cbD->commands.append(cmd);
@@ -4629,7 +4643,7 @@ void QRhiVulkan::setScissor(QRhiCommandBuffer *cb, const QRhiScissor &scissor)
     s->extent.height = uint32_t(h);
 
     if (cbD->passUsesSecondaryCb) {
-        df->vkCmdSetScissor(cbD->secondaryCbs.last(), 0, 1, s);
+        df->vkCmdSetScissor(cbD->activeSecondaryCbStack.last(), 0, 1, s);
     } else {
         cmd.cmd = QVkCommandBuffer::Command::SetScissor;
         cbD->commands.append(cmd);
@@ -4643,7 +4657,7 @@ void QRhiVulkan::setBlendConstants(QRhiCommandBuffer *cb, const QColor &c)
 
     if (cbD->passUsesSecondaryCb) {
         float constants[] = { float(c.redF()), float(c.greenF()), float(c.blueF()), float(c.alphaF()) };
-        df->vkCmdSetBlendConstants(cbD->secondaryCbs.last(), constants);
+        df->vkCmdSetBlendConstants(cbD->activeSecondaryCbStack.last(), constants);
     } else {
         QVkCommandBuffer::Command cmd;
         cmd.cmd = QVkCommandBuffer::Command::SetBlendConstants;
@@ -4661,7 +4675,7 @@ void QRhiVulkan::setStencilRef(QRhiCommandBuffer *cb, quint32 refValue)
     Q_ASSERT(cbD->recordingPass == QVkCommandBuffer::RenderPass);
 
     if (cbD->passUsesSecondaryCb) {
-        df->vkCmdSetStencilReference(cbD->secondaryCbs.last(), VK_STENCIL_FRONT_AND_BACK, refValue);
+        df->vkCmdSetStencilReference(cbD->activeSecondaryCbStack.last(), VK_STENCIL_FRONT_AND_BACK, refValue);
     } else {
         QVkCommandBuffer::Command cmd;
         cmd.cmd = QVkCommandBuffer::Command::SetStencilRef;
@@ -4677,7 +4691,7 @@ void QRhiVulkan::draw(QRhiCommandBuffer *cb, quint32 vertexCount,
     Q_ASSERT(cbD->recordingPass == QVkCommandBuffer::RenderPass);
 
     if (cbD->passUsesSecondaryCb) {
-        df->vkCmdDraw(cbD->secondaryCbs.last(), vertexCount, instanceCount, firstVertex, firstInstance);
+        df->vkCmdDraw(cbD->activeSecondaryCbStack.last(), vertexCount, instanceCount, firstVertex, firstInstance);
     } else {
         QVkCommandBuffer::Command cmd;
         cmd.cmd = QVkCommandBuffer::Command::Draw;
@@ -4696,7 +4710,7 @@ void QRhiVulkan::drawIndexed(QRhiCommandBuffer *cb, quint32 indexCount,
     Q_ASSERT(cbD->recordingPass == QVkCommandBuffer::RenderPass);
 
     if (cbD->passUsesSecondaryCb) {
-        df->vkCmdDrawIndexed(cbD->secondaryCbs.last(), indexCount, instanceCount,
+        df->vkCmdDrawIndexed(cbD->activeSecondaryCbStack.last(), indexCount, instanceCount,
                              firstIndex, vertexOffset, firstInstance);
     } else {
         QVkCommandBuffer::Command cmd;
@@ -4722,7 +4736,7 @@ void QRhiVulkan::debugMarkBegin(QRhiCommandBuffer *cb, const QByteArray &name)
     QVkCommandBuffer *cbD = QRHI_RES(QVkCommandBuffer, cb);
     if (cbD->recordingPass != QVkCommandBuffer::NoPass && cbD->passUsesSecondaryCb) {
         marker.pMarkerName = name.constData();
-        vkCmdDebugMarkerBegin(cbD->secondaryCbs.last(), &marker);
+        vkCmdDebugMarkerBegin(cbD->activeSecondaryCbStack.last(), &marker);
     } else {
         QVkCommandBuffer::Command cmd;
         cmd.cmd = QVkCommandBuffer::Command::DebugMarkerBegin;
@@ -4740,7 +4754,7 @@ void QRhiVulkan::debugMarkEnd(QRhiCommandBuffer *cb)
 
     QVkCommandBuffer *cbD = QRHI_RES(QVkCommandBuffer, cb);
     if (cbD->recordingPass != QVkCommandBuffer::NoPass && cbD->passUsesSecondaryCb) {
-        vkCmdDebugMarkerEnd(cbD->secondaryCbs.last());
+        vkCmdDebugMarkerEnd(cbD->activeSecondaryCbStack.last());
     } else {
         QVkCommandBuffer::Command cmd;
         cmd.cmd = QVkCommandBuffer::Command::DebugMarkerEnd;
@@ -4760,7 +4774,7 @@ void QRhiVulkan::debugMarkMsg(QRhiCommandBuffer *cb, const QByteArray &msg)
     QVkCommandBuffer *cbD = QRHI_RES(QVkCommandBuffer, cb);
     if (cbD->recordingPass != QVkCommandBuffer::NoPass && cbD->passUsesSecondaryCb) {
         marker.pMarkerName = msg.constData();
-        vkCmdDebugMarkerInsert(cbD->secondaryCbs.last(), &marker);
+        vkCmdDebugMarkerInsert(cbD->activeSecondaryCbStack.last(), &marker);
     } else {
         QVkCommandBuffer::Command cmd;
         cmd.cmd = QVkCommandBuffer::Command::DebugMarkerInsert;
@@ -4822,13 +4836,13 @@ void QRhiVulkan::beginExternal(QRhiCommandBuffer *cb)
         return;
     }
 
-    VkCommandBuffer secondaryCb = cbD->secondaryCbs.last();
-    cbD->secondaryCbs.removeLast();
+    VkCommandBuffer secondaryCb = cbD->activeSecondaryCbStack.last();
+    cbD->activeSecondaryCbStack.removeLast();
     endAndEnqueueSecondaryCommandBuffer(secondaryCb, cbD);
 
     VkCommandBuffer extCb = startSecondaryCommandBuffer(maybeRenderTargetData(cbD));
     if (extCb) {
-        cbD->secondaryCbs.append(extCb);
+        cbD->activeSecondaryCbStack.append(extCb);
         cbD->inExternal = true;
     }
 }
@@ -4840,10 +4854,10 @@ void QRhiVulkan::endExternal(QRhiCommandBuffer *cb)
     if (cbD->recordingPass == QVkCommandBuffer::NoPass) {
         Q_ASSERT(cbD->commands.isEmpty() && cbD->currentPassResTrackerIndex == -1);
     } else if (cbD->inExternal) {
-        VkCommandBuffer extCb = cbD->secondaryCbs.last();
-        cbD->secondaryCbs.removeLast();
+        VkCommandBuffer extCb = cbD->activeSecondaryCbStack.last();
+        cbD->activeSecondaryCbStack.removeLast();
         endAndEnqueueSecondaryCommandBuffer(extCb, cbD);
-        cbD->secondaryCbs.append(startSecondaryCommandBuffer(maybeRenderTargetData(cbD)));
+        cbD->activeSecondaryCbStack.append(startSecondaryCommandBuffer(maybeRenderTargetData(cbD)));
     }
 
     cbD->resetCachedState();
@@ -6622,8 +6636,8 @@ const QRhiNativeHandles *QVkCommandBuffer::nativeHandles()
     if (recordingPass == QVkCommandBuffer::NoPass) {
         nativeHandlesStruct.commandBuffer = cb;
     } else {
-        if (passUsesSecondaryCb && !secondaryCbs.isEmpty())
-            nativeHandlesStruct.commandBuffer = secondaryCbs.last();
+        if (passUsesSecondaryCb && !activeSecondaryCbStack.isEmpty())
+            nativeHandlesStruct.commandBuffer = activeSecondaryCbStack.last();
         else
             nativeHandlesStruct.commandBuffer = cb;
     }
@@ -6651,6 +6665,13 @@ void QVkSwapChain::destroy()
     QRHI_RES_RHI(QRhiVulkan);
     rhiD->swapchains.remove(this);
     rhiD->releaseSwapChainResources(this);
+
+    for (int i = 0; i < QVK_FRAMES_IN_FLIGHT; ++i) {
+        QVkSwapChain::FrameResources &frame(frameRes[i]);
+        frame.cmdBuf = VK_NULL_HANDLE;
+        frame.timestampQueryIndex = -1;
+    }
+
     surface = lastConnectedSurface = VK_NULL_HANDLE;
 
     QRHI_PROF;
diff --git a/src/gui/rhi/qrhivulkan_p.h b/src/gui/rhi/qrhivulkan_p.h
index b603b33e6d..4393ef6b36 100644
--- a/src/gui/rhi/qrhivulkan_p.h
+++ b/src/gui/rhi/qrhivulkan_p.h
@@ -69,8 +69,7 @@ struct Q_GUI_EXPORT QRhiVulkanNativeHandles : public QRhiNativeHandles
     int gfxQueueFamilyIdx = -1;
     int gfxQueueIdx = 0;
     VkQueue gfxQueue = VK_NULL_HANDLE;
-    // and optionally, command pool and/or mem allocator
-    VkCommandPool cmdPool = VK_NULL_HANDLE;
+    // and optionally, the mem allocator
     void *vmemAllocator = nullptr;
 };
 
diff --git a/src/gui/rhi/qrhivulkan_p_p.h b/src/gui/rhi/qrhivulkan_p_p.h
index a352f3c2f2..0a1c43f538 100644
--- a/src/gui/rhi/qrhivulkan_p_p.h
+++ b/src/gui/rhi/qrhivulkan_p_p.h
@@ -341,9 +341,7 @@ struct QVkCommandBuffer : public QRhiCommandBuffer
         recordingPass = NoPass;
         passUsesSecondaryCb = false;
         currentTarget = nullptr;
-
-        secondaryCbs.clear();
-
+        activeSecondaryCbStack.clear();
         resetCommands();
         resetCachedState();
     }
@@ -380,7 +378,7 @@ struct QVkCommandBuffer : public QRhiCommandBuffer
     static const int VERTEX_INPUT_RESOURCE_SLOT_COUNT = 32;
     VkBuffer currentVertexBuffers[VERTEX_INPUT_RESOURCE_SLOT_COUNT];
     quint32 currentVertexOffsets[VERTEX_INPUT_RESOURCE_SLOT_COUNT];
-    QVarLengthArray<VkCommandBuffer, 4> secondaryCbs;
+    QVarLengthArray<VkCommandBuffer, 4> activeSecondaryCbStack;
     bool inExternal;
 
     struct {
@@ -636,10 +634,9 @@ struct QVkSwapChain : public QRhiSwapChain
         VkSemaphore drawSem = VK_NULL_HANDLE;
         bool imageAcquired = false;
         bool imageSemWaitable = false;
-        quint32 imageIndex = 0;
-        VkCommandBuffer cmdBuf = VK_NULL_HANDLE; // primary
         VkFence cmdFence = VK_NULL_HANDLE;
         bool cmdFenceWaitable = false;
+        VkCommandBuffer cmdBuf = VK_NULL_HANDLE; // primary
         int timestampQueryIndex = -1;
     } frameRes[QVK_FRAMES_IN_FLIGHT];
 
@@ -784,7 +781,6 @@ public:
     void prepareNewFrame(QRhiCommandBuffer *cb);
     VkCommandBuffer startSecondaryCommandBuffer(QVkRenderTargetData *rtD = nullptr);
     void endAndEnqueueSecondaryCommandBuffer(VkCommandBuffer cb, QVkCommandBuffer *cbD);
-    void deferredReleaseSecondaryCommandBuffer(VkCommandBuffer cb);
     QRhi::FrameOpResult startPrimaryCommandBuffer(VkCommandBuffer *cb);
     QRhi::FrameOpResult endAndSubmitPrimaryCommandBuffer(VkCommandBuffer cb, VkFence cmdFence,
                                                          VkSemaphore *waitSem, VkSemaphore *signalSem);
@@ -825,6 +821,7 @@ public:
                             int startLayer, int layerCount,
                             int startLevel, int levelCount);
     void updateShaderResourceBindings(QRhiShaderResourceBindings *srb, int descSetIdx = -1);
+    void ensureCommandPoolForNewFrame();
 
     QVulkanInstance *inst = nullptr;
     QWindow *maybeWindow = nullptr;
@@ -832,8 +829,7 @@ public:
     bool importedDevice = false;
     VkPhysicalDevice physDev = VK_NULL_HANDLE;
     VkDevice dev = VK_NULL_HANDLE;
-    bool importedCmdPool = false;
-    VkCommandPool cmdPool = VK_NULL_HANDLE;
+    VkCommandPool cmdPool[QVK_FRAMES_IN_FLIGHT] = {};
     int gfxQueueFamilyIdx = -1;
     int gfxQueueIdx = 0;
     VkQueue gfxQueue = VK_NULL_HANDLE;
@@ -849,6 +845,7 @@ public:
     VkDeviceSize texbufAlign;
     bool hasWideLines = false;
     bool deviceLost = false;
+    bool releaseCachedResourcesCalledBeforeFrameStart = false;
 
     bool debugMarkersAvailable = false;
     bool vertexAttribDivisorAvailable = false;
@@ -877,6 +874,7 @@ public:
         int allocedDescSets = 0;
     };
     QVarLengthArray<DescriptorPoolData, 8> descriptorPools;
+    QVarLengthArray<VkCommandBuffer, 4> freeSecondaryCbs[QVK_FRAMES_IN_FLIGHT];
 
     VkQueryPool timestampQueryPool = VK_NULL_HANDLE;
     QBitArray timestampQueryPoolMap;
@@ -889,9 +887,18 @@ public:
     QRhiVulkanNativeHandles nativeHandlesStruct;
 
     struct OffscreenFrame {
-        OffscreenFrame(QRhiImplementation *rhi) : cbWrapper(rhi) { }
+        OffscreenFrame(QRhiImplementation *rhi)
+        {
+            for (int i = 0; i < QVK_FRAMES_IN_FLIGHT; ++i)
+                cbWrapper[i] = new QVkCommandBuffer(rhi);
+        }
+        ~OffscreenFrame()
+        {
+            for (int i = 0; i < QVK_FRAMES_IN_FLIGHT; ++i)
+                delete cbWrapper[i];
+        }
         bool active = false;
-        QVkCommandBuffer cbWrapper;
+        QVkCommandBuffer *cbWrapper[QVK_FRAMES_IN_FLIGHT];
         VkFence cmdFence = VK_NULL_HANDLE;
     } ofr;
 
@@ -926,7 +933,7 @@ public:
             TextureRenderTarget,
             RenderPass,
             StagingBuffer,
-            CommandBuffer
+            SecondaryCommandBuffer
         };
         Type type;
         int lastActiveFrameSlot; // -1 if not used otherwise 0..FRAMES_IN_FLIGHT-1
@@ -975,7 +982,7 @@ public:
             } stagingBuffer;
             struct {
                 VkCommandBuffer cb;
-            } commandBuffer;
+            } secondaryCommandBuffer;
         };
     };
     QList<DeferredReleaseEntry> releaseQueue;