vulkan: Fix handling of presentation semaphores

We need one image-ready semaphore per frame in flight, and one render-complete semaphore per swap chain image

panda/src/vulkandisplay/vulkanFrameData.h

@@ -32,6 +32,10 @@ public:
   VkCommandBuffer _cmd = VK_NULL_HANDLE;
   VkCommandBuffer _transfer_cmd = VK_NULL_HANDLE;
 
+  // The frame data takes ownership of this semaphore, which indicates when the
+  // frame is allowed to start rendering (the image is available).
+  VkSemaphore _wait_semaphore = VK_NULL_HANDLE;
+
   // Keep track of resources that should be deleted after this frame is done.
   pvector<VulkanMemoryBlock> _pending_free;
   pvector<VkBuffer> _pending_destroy_buffers;

panda/src/vulkandisplay/vulkanGraphicsStateGuardian.cxx

@@ -2561,6 +2561,16 @@ end_scene() {
  */
 void VulkanGraphicsStateGuardian::
 end_frame(Thread *current_thread) {
+  end_frame(current_thread, VK_NULL_HANDLE, VK_NULL_HANDLE);
+}
+
+/**
+ * Version of end_frame that waits for a semaphore before rendering, and also
+ * signals a given semaphore when it's done.
+ * Takes ownership of the wait_for semaphore.
+ */
+void VulkanGraphicsStateGuardian::
+end_frame(Thread *current_thread, VkSemaphore wait_for, VkSemaphore signal_done) {
   GraphicsStateGuardian::end_frame(current_thread);
 
   nassertv(_frame_data->_transfer_cmd != VK_NULL_HANDLE);
@@ -2623,18 +2633,19 @@ end_frame(Thread *current_thread) {
   submit_info.signalSemaphoreCount = 0;
   submit_info.pSignalSemaphores = nullptr;
 
-  if (_wait_semaphore != VK_NULL_HANDLE) {
+  if (wait_for != VK_NULL_HANDLE) {
     // We may need to wait until the attachments are available for writing.
     static const VkPipelineStageFlags flags = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
     submit_info.waitSemaphoreCount = 1;
-    submit_info.pWaitSemaphores = &_wait_semaphore;
+    submit_info.pWaitSemaphores = &wait_for;
     submit_info.pWaitDstStageMask = &flags;
+    _frame_data->_wait_semaphore = wait_for;
   }
 
-  if (_signal_semaphore != VK_NULL_HANDLE) {
+  if (signal_done != VK_NULL_HANDLE) {
     // And we were asked to signal a semaphore when we are done rendering.
     submit_info.signalSemaphoreCount = 1;
-    submit_info.pSignalSemaphores = &_signal_semaphore;
+    submit_info.pSignalSemaphores = &signal_done;
   }
 
   VkResult err;
@@ -2644,10 +2655,6 @@ end_frame(Thread *current_thread) {
     return;
   }
 
-  // We're done with these for now.
-  _wait_semaphore = VK_NULL_HANDLE;
-  _signal_semaphore = VK_NULL_HANDLE;
-
   // If we queued up synchronous texture downloads, wait for the queue to finish
   // (slow!) and then copy the data from Vulkan host memory to Panda memory.
   if (_frame_data->_wait_for_finish) {
@@ -2700,6 +2707,11 @@ finish_frame(FrameData &frame_data) {
   ++_last_finished_frame;
   nassertv(frame_data._frame_index == _last_finished_frame);
 
+  if (frame_data._wait_semaphore != VK_NULL_HANDLE) {
+    vkDestroySemaphore(_device, frame_data._wait_semaphore, nullptr);
+    frame_data._wait_semaphore = nullptr;
+  }
+
   for (VkBufferView buffer_view : frame_data._pending_destroy_buffer_views) {
     vkDestroyBufferView(_device, buffer_view, nullptr);
   }

panda/src/vulkandisplay/vulkanGraphicsStateGuardian.h

@@ -94,6 +94,7 @@ public:
   virtual bool begin_scene();
   virtual void end_scene();
   virtual void end_frame(Thread *current_thread);
+  void end_frame(Thread *current_thread, VkSemaphore wait_for, VkSemaphore signal_done);
   void finish_frame(FrameData &frame_data);
 
   virtual bool begin_draw_primitives(const GeomPipelineReader *geom_reader,
@@ -225,8 +226,6 @@ private:
   VulkanTextureContext *_fb_color_tc = nullptr;
   VulkanTextureContext *_fb_depth_tc = nullptr;
   VkSampleCountFlagBits _fb_ms_count = VK_SAMPLE_COUNT_1_BIT;
-  VkSemaphore _wait_semaphore = VK_NULL_HANDLE;
-  VkSemaphore _signal_semaphore = VK_NULL_HANDLE;
 
   // Remembers semaphores created on this device.
   pvector<VkSemaphore> _semaphores;

panda/src/vulkandisplay/vulkanGraphicsWindow.cxx

@@ -226,9 +226,6 @@ begin_frame(FrameMode mode, Thread *current_thread) {
   vkgsg->_fb_color_tc = color_tc;
   vkgsg->_fb_depth_tc = _depth_stencil_tc;
   vkgsg->_fb_ms_count = _ms_count;
-  vkgsg->_wait_semaphore = _image_available;
-  vkgsg->_signal_semaphore = _render_complete;
-
   return true;
 }
 
@@ -248,6 +245,7 @@ end_frame(FrameMode mode, Thread *current_thread) {
   nassertv(cmd != VK_NULL_HANDLE);
   SwapBuffer &buffer = _swap_buffers[_image_index];
 
+  VkSemaphore signal_done = VK_NULL_HANDLE;
   if (mode == FM_render) {
     vkCmdEndRenderPass(cmd);
     vkgsg->_render_pass = VK_NULL_HANDLE;
@@ -258,6 +256,8 @@ end_frame(FrameMode mode, Thread *current_thread) {
 
     // Now we can do copy-to-texture, now that the render pass has ended.
     copy_to_textures();
+
+    signal_done = buffer._render_complete;
   }
 
   // If we copied the textures, transition it back to the present state.
@@ -268,9 +268,13 @@ end_frame(FrameMode mode, Thread *current_thread) {
 
   // Note: this will close the command buffer, and unsignal the previous
   // frame's semaphore.
-  vkgsg->end_frame(current_thread);
+  vkgsg->end_frame(current_thread, _image_available, signal_done);
+
+  // Ownership of this was transferred to the VulkanFrameData.
+  _image_available = VK_NULL_HANDLE;
 
   if (mode == FM_render) {
+    nassertv(!_flip_ready);
     _flip_ready = true;
     clear_cube_map_selection();
   }
@@ -294,12 +298,14 @@ begin_flip() {
   VkQueue queue = vkgsg->_queue;
   VkResult err;
 
+  SwapBuffer &buffer = _swap_buffers[_image_index];
+
   VkResult results[1] = {VK_SUCCESS};
   VkPresentInfoKHR present;
   present.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
   present.pNext = nullptr;
   present.waitSemaphoreCount = 1;
-  present.pWaitSemaphores = &_render_complete;
+  present.pWaitSemaphores = &buffer._render_complete;
   present.swapchainCount = 1;
   present.pSwapchains = &_swapchain;
   present.pImageIndices = &_image_index;
@@ -315,12 +321,16 @@ begin_flip() {
     _swapchain_size.set(-1, -1);
     _flip_ready = false;
     return;
-
-  } else if (err == VK_SUBOPTIMAL_KHR) {
+  }
+  else if (err == VK_SUBOPTIMAL_KHR) {
     std::cerr << "suboptimal.\n";
-
-  } else if (err != VK_SUCCESS) {
+  }
+  else if (err != VK_SUCCESS) {
     vulkan_error(err, "Error presenting queue");
+
+    if (err == VK_ERROR_FULL_SCREEN_EXCLUSIVE_MODE_LOST_EXT || err == VK_ERROR_SURFACE_LOST_KHR) {
+      _flip_ready = false;
+    }
     return;
   }
 }
@@ -352,9 +362,15 @@ end_flip() {
   VulkanGraphicsStateGuardian *vkgsg;
   DCAST_INTO_V(vkgsg, _gsg);
 
+  nassertv(_image_available == VK_NULL_HANDLE);
+
+  VkSemaphoreCreateInfo semaphore_info = {};
+  semaphore_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
+  VkResult err = vkCreateSemaphore(vkgsg->_device, &semaphore_info, nullptr, &_image_available);
+  nassertv_always(err == VK_SUCCESS);
+
   // Get a new image for rendering into.  This may wait until a new image is
   // available.
-  VkResult
   err = vkAcquireNextImageKHR(vkgsg->_device, _swapchain, UINT64_MAX,
                               _image_available, VK_NULL_HANDLE, &_image_index);
 
@@ -977,6 +993,11 @@ create_swapchain() {
     }
 
     buffer._tc->_image_views.push_back(image_view);
+
+    // Create a semaphore that is signalled when we are finished rendering,
+    // to indicate that it is safe to present the image.
+    buffer._render_complete = vkgsg->create_semaphore();
+    nassertr(buffer._render_complete != VK_NULL_HANDLE, false);
   }
 
   // Now create a depth image.
@@ -1103,15 +1124,11 @@ create_swapchain() {
   }
 
   // Create a semaphore for signalling the availability of an image.
-  // It will be signalled in end_flip() and waited upon before submitting the
-  // command buffers that use that image for rendering to.
-  _image_available = vkgsg->create_semaphore();
-  nassertr(_image_available != VK_NULL_HANDLE, false);
-
-  // Now create another one that is signalled when we are finished rendering,
-  // to indicate that it is safe to present the image.
-  _render_complete = vkgsg->create_semaphore();
-  nassertr(_render_complete != VK_NULL_HANDLE, false);
+  VkSemaphoreCreateInfo semaphore_info = {};
+  semaphore_info.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
+
+  err = vkCreateSemaphore(vkgsg->_device, &semaphore_info, nullptr, &_image_available);
+  nassertr_always(err == VK_SUCCESS, false);
 
   // We need to acquire an image before we continue rendering.
   _image_index = 0;

panda/src/vulkandisplay/vulkanGraphicsWindow.h

@@ -70,8 +70,7 @@ private:
   VkRenderPass _render_pass = VK_NULL_HANDLE;
   int _current_clear_mask = -1;
 
-  // We'll need these to synchronize the rendering with the presentation.
-  VkSemaphore _render_complete = VK_NULL_HANDLE;
+  // Synchronizes flip of previous frame with rendering.
   VkSemaphore _image_available = VK_NULL_HANDLE;
 
   LVecBase2i _swapchain_size;
@@ -80,6 +79,7 @@ private:
   struct SwapBuffer {
     VulkanTextureContext *_tc;
     VkFramebuffer _framebuffer;
+    VkSemaphore _render_complete;
   };
   typedef pvector<SwapBuffer> SwapBuffers;
   SwapBuffers _swap_buffers;