Fence API (#45)

Reviewed-on: #45

include/Refresh.h

@@ -76,6 +76,7 @@ typedef struct Refresh_ShaderModule Refresh_ShaderModule;
 typedef struct Refresh_ComputePipeline Refresh_ComputePipeline;
 typedef struct Refresh_GraphicsPipeline Refresh_GraphicsPipeline;
 typedef struct Refresh_CommandBuffer Refresh_CommandBuffer;
+typedef struct Refresh_Fence Refresh_Fence;
 
 typedef enum Refresh_PresentMode
 {
@@ -1206,15 +1207,50 @@ REFRESHAPI Refresh_Texture* Refresh_AcquireSwapchainTexture(
 /* Submits all of the enqueued commands. */
 REFRESHAPI void Refresh_Submit(
 	Refresh_Device* device,
-	uint32_t commandBufferCount,
+	Refresh_CommandBuffer *commandBuffer
-	Refresh_CommandBuffer **pCommandBuffers
 );
 
-/* Waits for all submissions to complete. */
+/* Submits a command buffer and acquires a fence.
+ * You can use the fence to check if or wait until the command buffer has finished processing.
+ * You are responsible for releasing this fence when you are done using it.
+ */
+REFRESHAPI Refresh_Fence* Refresh_SubmitAndAcquireFence(
+	Refresh_Device* device,
+	Refresh_CommandBuffer *commandBuffer
+);
+
+/* Waits for the device to become idle. */
 REFRESHAPI void Refresh_Wait(
 	Refresh_Device *device
 );
 
+/* Waits for given fences to be signaled.
+ *
+ * waitAll: If 0, waits for any fence to be signaled. If 1, waits for all fences to be signaled.
+ * fenceCount: The number of fences being submitted.
+ * pFences: An array of fences to be waited on.
+ */
+REFRESHAPI void Refresh_WaitForFences(
+	Refresh_Device *device,
+	uint8_t waitAll,
+	uint32_t fenceCount,
+	Refresh_Fence **pFences
+);
+
+/* Check the status of a fence. 1 means the fence is signaled. */
+REFRESHAPI int Refresh_QueryFence(
+	Refresh_Device *device,
+	Refresh_Fence *fence
+);
+
+/* Allows the fence to be reused by future command buffer submissions.
+ * If you do not release fences after acquiring them, you will cause unbounded resource growth.
+ */
+REFRESHAPI void Refresh_ReleaseFence(
+	Refresh_Device *device,
+	Refresh_Fence *fence
+);
+
 #ifdef __cplusplus
 }
 #endif /* __cplusplus */

src/Refresh.c

@@ -892,14 +892,23 @@ void Refresh_SetSwapchainPresentMode(
 
 void Refresh_Submit(
 	Refresh_Device *device,
-	uint32_t commandBufferCount,
+	Refresh_CommandBuffer *commandBuffer
-	Refresh_CommandBuffer **pCommandBuffers
 ) {
 	NULL_RETURN(device);
 	device->Submit(
 		device->driverData,
-		commandBufferCount,
+		commandBuffer
-		pCommandBuffers
+	);
+}
+
+Refresh_Fence* Refresh_SubmitAndAcquireFence(
+	Refresh_Device *device,
+	Refresh_CommandBuffer *commandBuffer
+) {
+	NULL_RETURN_NULL(device);
+	return device->SubmitAndAcquireFence(
+		device->driverData,
+		commandBuffer
 	);
 }
 
@@ -912,4 +921,44 @@ void Refresh_Wait(
 	);
 }
 
+void Refresh_WaitForFences(
+	Refresh_Device *device,
+	uint8_t waitAll,
+	uint32_t fenceCount,
+	Refresh_Fence **pFences
+) {
+	NULL_RETURN(device);
+	device->WaitForFences(
+		device->driverData,
+		waitAll,
+		fenceCount,
+		pFences
+	);
+}
+
+int Refresh_QueryFence(
+	Refresh_Device *device,
+	Refresh_Fence *fence
+) {
+	if (device == NULL) {
+		return 0;
+	}
+
+	return device->QueryFence(
+		device->driverData,
+		fence
+	);
+}
+
+void Refresh_ReleaseFence(
+	Refresh_Device *device,
+	Refresh_Fence *fence
+) {
+	NULL_RETURN(device);
+	device->ReleaseFence(
+		device->driverData,
+		fence
+	);
+}
+
 /* vim: set noexpandtab shiftwidth=8 tabstop=8: */

src/Refresh_Driver.h

@@ -268,7 +268,7 @@ struct Refresh_Device
 
 	/* Setters */
 
-	void(*SetTextureData)(
+	void (*SetTextureData)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_TextureSlice *textureSlice,
@@ -276,7 +276,7 @@ struct Refresh_Device
 		uint32_t dataLengthInBytes
 	);
 
-	void(*SetTextureDataYUV)(
+	void (*SetTextureDataYUV)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer* commandBuffer,
 		Refresh_Texture *y,
@@ -295,7 +295,7 @@ struct Refresh_Device
 		uint32_t uvStride
 	);
 
-	void(*CopyTextureToTexture)(
+	void (*CopyTextureToTexture)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_TextureSlice *sourceTextureSlice,
@@ -303,14 +303,14 @@ struct Refresh_Device
 		Refresh_Filter filter
 	);
 
-	void(*CopyTextureToBuffer)(
+	void (*CopyTextureToBuffer)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_TextureSlice *textureSlice,
 		Refresh_Buffer *buffer
 	);
 
-	void(*SetBufferData)(
+	void (*SetBufferData)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Buffer *buffer,
@@ -319,14 +319,14 @@ struct Refresh_Device
 		uint32_t dataLength
 	);
 
-	uint32_t(*PushVertexShaderUniforms)(
+	uint32_t (*PushVertexShaderUniforms)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		void *data,
 		uint32_t dataLengthInBytes
 	);
 
-	uint32_t(*PushFragmentShaderUniforms)(
+	uint32_t (*PushFragmentShaderUniforms)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		void *data,
@@ -340,14 +340,14 @@ struct Refresh_Device
 		uint32_t dataLengthInBytes
 	);
 
-	void(*BindVertexSamplers)(
+	void (*BindVertexSamplers)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Texture **pTextures,
 		Refresh_Sampler **pSamplers
 	);
 
-	void(*BindFragmentSamplers)(
+	void (*BindFragmentSamplers)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Texture **pTextures,
@@ -356,7 +356,7 @@ struct Refresh_Device
 
 	/* Getters */
 
-	void(*GetBufferData)(
+	void (*GetBufferData)(
 		Refresh_Renderer *driverData,
 		Refresh_Buffer *buffer,
 		void *data,
@@ -365,39 +365,39 @@ struct Refresh_Device
 
 	/* Disposal */
 
-	void(*QueueDestroyTexture)(
+	void (*QueueDestroyTexture)(
 		Refresh_Renderer *driverData,
 		Refresh_Texture *texture
 	);
 
-	void(*QueueDestroySampler)(
+	void (*QueueDestroySampler)(
 		Refresh_Renderer *driverData,
 		Refresh_Sampler *sampler
 	);
 
-	void(*QueueDestroyBuffer)(
+	void (*QueueDestroyBuffer)(
 		Refresh_Renderer *driverData,
 		Refresh_Buffer *buffer
 	);
 
-	void(*QueueDestroyShaderModule)(
+	void (*QueueDestroyShaderModule)(
 		Refresh_Renderer *driverData,
 		Refresh_ShaderModule *shaderModule
 	);
 
-	void(*QueueDestroyComputePipeline)(
+	void (*QueueDestroyComputePipeline)(
 		Refresh_Renderer *driverData,
 		Refresh_ComputePipeline *computePipeline
 	);
 
-	void(*QueueDestroyGraphicsPipeline)(
+	void (*QueueDestroyGraphicsPipeline)(
 		Refresh_Renderer *driverData,
 		Refresh_GraphicsPipeline *graphicsPipeline
 	);
 
 	/* Graphics State */
 
-	void(*BeginRenderPass)(
+	void (*BeginRenderPass)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_ColorAttachmentInfo *colorAttachmentInfos,
@@ -405,30 +405,30 @@ struct Refresh_Device
 		Refresh_DepthStencilAttachmentInfo *depthStencilAttachmentInfo
 	);
 
-	void(*EndRenderPass)(
+	void (*EndRenderPass)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer
 	);
 
-	void(*SetViewport)(
+	void (*SetViewport)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Viewport *viewport
 	);
 
-	void(*SetScissor)(
+	void (*SetScissor)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Rect *scissor
 	);
 
-	void(*BindGraphicsPipeline)(
+	void (*BindGraphicsPipeline)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_GraphicsPipeline *graphicsPipeline
 	);
 
-	void(*BindVertexBuffers)(
+	void (*BindVertexBuffers)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		uint32_t firstBinding,
@@ -437,7 +437,7 @@ struct Refresh_Device
 		uint64_t *pOffsets
 	);
 
-	void(*BindIndexBuffer)(
+	void (*BindIndexBuffer)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Buffer *buffer,
@@ -445,19 +445,19 @@ struct Refresh_Device
 		Refresh_IndexElementSize indexElementSize
 	);
 
-	void(*BindComputePipeline)(
+	void (*BindComputePipeline)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_ComputePipeline *computePipeline
 	);
 
-	void(*BindComputeBuffers)(
+	void (*BindComputeBuffers)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Buffer **pBuffers
 	);
 
-	void(*BindComputeTextures)(
+	void (*BindComputeTextures)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
 		Refresh_Texture **pTextures
@@ -469,7 +469,7 @@ struct Refresh_Device
 		Refresh_PresentMode presentMode
 	);
 
-	void(*UnclaimWindow)(
+	void (*UnclaimWindow)(
 		Refresh_Renderer *driverData,
 		void *windowHandle
 	);
@@ -497,16 +497,37 @@ struct Refresh_Device
 		Refresh_PresentMode presentMode
 	);
 
-	void(*Submit)(
+	void (*Submit)(
 		Refresh_Renderer *driverData,
-		uint32_t commandBufferCount,
+		Refresh_CommandBuffer *commandBuffer
-		Refresh_CommandBuffer **pCommandBuffers
 	);
 
-	void(*Wait)(
+	Refresh_Fence* (*SubmitAndAcquireFence)(
+		Refresh_Renderer *driverData,
+		Refresh_CommandBuffer *commandBuffer
+	);
+
+	void (*Wait)(
 		Refresh_Renderer *driverData
 	);
 
+	void (*WaitForFences)(
+		Refresh_Renderer *driverData,
+		uint8_t waitAll,
+		uint32_t fenceCount,
+		Refresh_Fence **pFences
+	);
+
+	int (*QueryFence)(
+		Refresh_Renderer *driverData,
+		Refresh_Fence *fence
+	);
+
+	void (*ReleaseFence)(
+		Refresh_Renderer *driverData,
+		Refresh_Fence *fence
+	);
+
 	/* Opaque pointer for the Driver */
 	Refresh_Renderer *driverData;
 };
@@ -560,7 +581,11 @@ struct Refresh_Device
 	ASSIGN_DRIVER_FUNC(GetSwapchainFormat, name) \
 	ASSIGN_DRIVER_FUNC(SetSwapchainPresentMode, name) \
 	ASSIGN_DRIVER_FUNC(Submit, name) \
-	ASSIGN_DRIVER_FUNC(Wait, name)
+	ASSIGN_DRIVER_FUNC(SubmitAndAcquireFence, name) \
+	ASSIGN_DRIVER_FUNC(Wait, name) \
+	ASSIGN_DRIVER_FUNC(WaitForFences, name) \
+	ASSIGN_DRIVER_FUNC(QueryFence, name) \
+	ASSIGN_DRIVER_FUNC(ReleaseFence, name)
 
 typedef struct Refresh_Driver
 {

src/Refresh_Driver_Vulkan.c

@@ -1533,6 +1533,15 @@ typedef struct VulkanTransferBufferPool
 	uint32_t availableBufferCapacity;
 } VulkanTransferBufferPool;
 
+typedef struct VulkanFencePool
+{
+	SDL_mutex *lock;
+
+	VkFence *availableFences;
+	uint32_t availableFenceCount;
+	uint32_t availableFenceCapacity;
+} VulkanFencePool;
+
 typedef struct VulkanCommandPool VulkanCommandPool;
 
 typedef struct VulkanCommandBuffer
@@ -1624,6 +1633,7 @@ typedef struct VulkanCommandBuffer
 	/* Shader modules have references tracked by pipelines */
 
 	VkFence inFlightFence;
+	uint8_t autoReleaseFence;
 } VulkanCommandBuffer;
 
 struct VulkanCommandPool
@@ -1737,6 +1747,7 @@ typedef struct VulkanRenderer
 	uint32_t submittedCommandBufferCapacity;
 
 	VulkanTransferBufferPool transferBufferPool;
+	VulkanFencePool fencePool;
 
 	CommandPoolHashTable commandPoolHashTable;
 	DescriptorSetLayoutHashTable descriptorSetLayoutHashTable;
@@ -1828,7 +1839,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory(VulkanRenderer *renderer);
 static void VULKAN_INTERNAL_BeginCommandBuffer(VulkanRenderer *renderer, VulkanCommandBuffer *commandBuffer);
 static void VULKAN_UnclaimWindow(Refresh_Renderer *driverData, void *windowHandle);
 static void VULKAN_Wait(Refresh_Renderer *driverData);
-static void VULKAN_Submit(Refresh_Renderer *driverData, uint32_t commandBufferCount, Refresh_CommandBuffer **pCommandBuffers);
+static void VULKAN_Submit(Refresh_Renderer *driverData, Refresh_CommandBuffer *commandBuffer);
 static void VULKAN_INTERNAL_DestroyRenderTarget(VulkanRenderer *renderer, VulkanRenderTarget *renderTarget);
 
 /* Error Handling */
@@ -3589,12 +3600,6 @@ static void VULKAN_INTERNAL_DestroyCommandPool(
 	{
 		commandBuffer = commandPool->inactiveCommandBuffers[i];
 
-		renderer->vkDestroyFence(
-			renderer->logicalDevice,
-			commandBuffer->inFlightFence,
-			NULL
-		);
-
 		SDL_free(commandBuffer->presentDatas);
 		SDL_free(commandBuffer->waitSemaphores);
 		SDL_free(commandBuffer->signalSemaphores);
@@ -5158,6 +5163,14 @@ static void VULKAN_DestroyDevice(
 	SDL_free(renderer->transferBufferPool.availableBuffers);
 	SDL_DestroyMutex(renderer->transferBufferPool.lock);
 
+	for (i = 0; i < renderer->fencePool.availableFenceCount; i += 1)
+	{
+		renderer->vkDestroyFence(renderer->logicalDevice, renderer->fencePool.availableFences[i], NULL);
+	}
+
+	SDL_free(renderer->fencePool.availableFences);
+	SDL_DestroyMutex(renderer->fencePool.lock);
+
 	for (i = 0; i < NUM_COMMAND_POOL_BUCKETS; i += 1)
 	{
 		commandPoolHashArray = renderer->commandPoolHashTable.buckets[i];
@@ -9362,7 +9375,6 @@ static void VULKAN_INTERNAL_AllocateCommandBuffers(
 	uint32_t allocateCount
 ) {
 	VkCommandBufferAllocateInfo allocateInfo;
-	VkFenceCreateInfo fenceCreateInfo;
 	VkResult vulkanResult;
 	uint32_t i;
 	VkCommandBuffer *commandBuffers = SDL_stack_alloc(VkCommandBuffer, allocateCount);
@@ -9401,23 +9413,7 @@ static void VULKAN_INTERNAL_AllocateCommandBuffers(
 		commandBuffer->commandPool = vulkanCommandPool;
 		commandBuffer->commandBuffer = commandBuffers[i];
 
-		/* Create fence */
+		commandBuffer->inFlightFence = VK_NULL_HANDLE;
-		fenceCreateInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-		fenceCreateInfo.pNext = NULL;
-		fenceCreateInfo.flags = 0;
-
-		vulkanResult = renderer->vkCreateFence(
-			renderer->logicalDevice,
-			&fenceCreateInfo,
-			NULL,
-			&commandBuffer->inFlightFence
-		);
-
-		if (vulkanResult != VK_SUCCESS)
-		{
-			LogVulkanResultAsError("vkCreateFence", vulkanResult);
-		}
-
 		commandBuffer->renderPassDepthTexture = NULL;
 
 		/* Presentation tracking */
@@ -9650,17 +9646,6 @@ static Refresh_CommandBuffer* VULKAN_AcquireCommandBuffer(
 		LogVulkanResultAsError("vkResetCommandBuffer", result);
 	}
 
-	result = renderer->vkResetFences(
-		renderer->logicalDevice,
-		1,
-		&commandBuffer->inFlightFence
-	);
-
-	if (result != VK_SUCCESS)
-	{
-		LogVulkanResultAsError("vkResetFences", result);
-	}
-
 	VULKAN_INTERNAL_BeginCommandBuffer(renderer, commandBuffer);
 
 	return (Refresh_CommandBuffer*) commandBuffer;
@@ -9949,6 +9934,77 @@ static void VULKAN_SetSwapchainPresentMode(
 
 /* Submission structure */
 
+static VkFence VULKAN_INTERNAL_AcquireFenceFromPool(
+	VulkanRenderer *renderer
+) {
+	VkFenceCreateInfo fenceCreateInfo;
+	VkFence fence;
+	VkResult vulkanResult;
+
+	if (renderer->fencePool.availableFenceCount == 0)
+	{
+		/* Create fence */
+		fenceCreateInfo.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+		fenceCreateInfo.pNext = NULL;
+		fenceCreateInfo.flags = 0;
+
+		vulkanResult = renderer->vkCreateFence(
+			renderer->logicalDevice,
+			&fenceCreateInfo,
+			NULL,
+			&fence
+		);
+
+		if (vulkanResult != VK_SUCCESS)
+		{
+			LogVulkanResultAsError("vkCreateFence", vulkanResult);
+			return NULL;
+		}
+
+		return fence;
+	}
+
+	SDL_LockMutex(renderer->fencePool.lock);
+
+	fence = renderer->fencePool.availableFences[renderer->fencePool.availableFenceCount - 1];
+	renderer->fencePool.availableFenceCount -= 1;
+
+	vulkanResult = renderer->vkResetFences(
+		renderer->logicalDevice,
+		1,
+		&fence
+	);
+
+	if (vulkanResult != VK_SUCCESS)
+	{
+		LogVulkanResultAsError("vkResetFences", vulkanResult);
+	}
+
+	SDL_UnlockMutex(renderer->fencePool.lock);
+
+	return fence;
+}
+
+static void VULKAN_INTERNAL_ReturnFenceToPool(
+	VulkanRenderer *renderer,
+	VkFence fence
+) {
+	SDL_LockMutex(renderer->fencePool.lock);
+
+	EXPAND_ARRAY_IF_NEEDED(
+		renderer->fencePool.availableFences,
+		VkFence,
+		renderer->fencePool.availableFenceCount + 1,
+		renderer->fencePool.availableFenceCapacity,
+		renderer->fencePool.availableFenceCapacity * 2
+	);
+
+	renderer->fencePool.availableFences[renderer->fencePool.availableFenceCount] = fence;
+	renderer->fencePool.availableFenceCount += 1;
+
+	SDL_UnlockMutex(renderer->fencePool.lock);
+}
+
 static void VULKAN_INTERNAL_PerformPendingDestroys(
 	VulkanRenderer *renderer
 ) {
@@ -10064,6 +10120,16 @@ static void VULKAN_INTERNAL_CleanCommandBuffer(
 	VulkanUniformBuffer *uniformBuffer;
 	DescriptorSetData *descriptorSetData;
 
+	if (commandBuffer->autoReleaseFence)
+	{
+		VULKAN_INTERNAL_ReturnFenceToPool(
+			renderer,
+			commandBuffer->inFlightFence
+		);
+
+		commandBuffer->inFlightFence = VK_NULL_HANDLE;
+	}
+
 	/* Bound uniform buffers are now available */
 
 	for (i = 0; i < commandBuffer->boundUniformBufferCount; i += 1)
@@ -10219,26 +10285,16 @@ static void VULKAN_Wait(
 	VkResult result;
 	int32_t i;
 
-	SDL_LockMutex(renderer->submitLock);
+	result = renderer->vkDeviceWaitIdle(renderer->logicalDevice);
 
-	for (i = renderer->submittedCommandBufferCount - 1; i >= 0; i -= 1)
+	if (result != VK_SUCCESS)
 	{
-		commandBuffer = renderer->submittedCommandBuffers[i];
+		LogVulkanResultAsError("vkDeviceWaitIdle", result);
-
+		return;
-		result = renderer->vkWaitForFences(
-			renderer->logicalDevice,
-			1,
-			&commandBuffer->inFlightFence,
-			VK_TRUE,
-			UINT64_MAX
-		);
-
-		if (result != VK_SUCCESS)
-		{
-			LogVulkanResultAsError("vkWaitForFences", result);
-		}
 	}
 
+	SDL_LockMutex(renderer->submitLock);
+
 	for (i = renderer->submittedCommandBufferCount - 1; i >= 0; i -= 1)
 	{
 		commandBuffer = renderer->submittedCommandBuffers[i];
@@ -10250,17 +10306,30 @@ static void VULKAN_Wait(
 	SDL_UnlockMutex(renderer->submitLock);
 }
 
+static Refresh_Fence* VULKAN_SubmitAndAcquireFence(
+	Refresh_Renderer *driverData,
+	Refresh_CommandBuffer *commandBuffer
+) {
+	VulkanCommandBuffer *vulkanCommandBuffer;
+
+	VULKAN_Submit(driverData, commandBuffer);
+
+	vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer;
+	vulkanCommandBuffer->autoReleaseFence = 0;
+
+	return (Refresh_Fence*) vulkanCommandBuffer->inFlightFence;
+}
+
 static void VULKAN_Submit(
 	Refresh_Renderer *driverData,
-	uint32_t commandBufferCount,
+	Refresh_CommandBuffer *commandBuffer
-	Refresh_CommandBuffer **pCommandBuffers
 ) {
 	VulkanRenderer* renderer = (VulkanRenderer*)driverData;
 	VkSubmitInfo submitInfo;
 	VkPresentInfoKHR presentInfo;
 	VulkanPresentData *presentData;
 	VkResult vulkanResult, presentResult = VK_SUCCESS;
-	VulkanCommandBuffer *currentCommandBuffer;
+	VulkanCommandBuffer *vulkanCommandBuffer;
 	VkPipelineStageFlags waitStages[MAX_PRESENT_COUNT];
 	uint32_t swapchainImageIndex;
 	uint8_t commandBufferCleaned = 0;
@@ -10275,99 +10344,97 @@ static void VULKAN_Submit(
 		waitStages[i] = VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
 	}
 
-	/* Submit the commands finally */
+	vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer;
 
-	for (i = 0; i < commandBufferCount; i += 1)
+	for (j = 0; j < vulkanCommandBuffer->presentDataCount; j += 1)
 	{
-		currentCommandBuffer = (VulkanCommandBuffer*)pCommandBuffers[i];
+		swapchainImageIndex = vulkanCommandBuffer->presentDatas[j].swapchainImageIndex;
 
-		for (j = 0; j < currentCommandBuffer->presentDataCount; j += 1)
+		VULKAN_INTERNAL_ImageMemoryBarrier(
-		{
+			renderer,
-			swapchainImageIndex = currentCommandBuffer->presentDatas[j].swapchainImageIndex;
+			vulkanCommandBuffer->commandBuffer,
-
+			RESOURCE_ACCESS_PRESENT,
-			VULKAN_INTERNAL_ImageMemoryBarrier(
+			VK_IMAGE_ASPECT_COLOR_BIT,
-				renderer,
+			0,
-				currentCommandBuffer->commandBuffer,
-				RESOURCE_ACCESS_PRESENT,
-				VK_IMAGE_ASPECT_COLOR_BIT,
-				0,
-				1,
-				0,
-				1,
-				0,
-				currentCommandBuffer->presentDatas[j].windowData->swapchainData->textureContainers[swapchainImageIndex].vulkanTexture->image,
-				&currentCommandBuffer->presentDatas[j].windowData->swapchainData->textureContainers[swapchainImageIndex].vulkanTexture->resourceAccessType
-			);
-		}
-
-		VULKAN_INTERNAL_EndCommandBuffer(renderer, currentCommandBuffer);
-
-		submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
-		submitInfo.pNext = NULL;
-		submitInfo.commandBufferCount = 1;
-		submitInfo.pCommandBuffers = &currentCommandBuffer->commandBuffer;
-
-		submitInfo.pWaitDstStageMask = waitStages;
-		submitInfo.pWaitSemaphores = currentCommandBuffer->waitSemaphores;
-		submitInfo.waitSemaphoreCount = currentCommandBuffer->waitSemaphoreCount;
-		submitInfo.pSignalSemaphores = currentCommandBuffer->signalSemaphores;
-		submitInfo.signalSemaphoreCount = currentCommandBuffer->signalSemaphoreCount;
-
-		vulkanResult = renderer->vkQueueSubmit(
-			renderer->unifiedQueue,
 			1,
-			&submitInfo,
+			0,
-			currentCommandBuffer->inFlightFence
+			1,
+			0,
+			vulkanCommandBuffer->presentDatas[j].windowData->swapchainData->textureContainers[swapchainImageIndex].vulkanTexture->image,
+			&vulkanCommandBuffer->presentDatas[j].windowData->swapchainData->textureContainers[swapchainImageIndex].vulkanTexture->resourceAccessType
+		);
+	}
+
+	VULKAN_INTERNAL_EndCommandBuffer(renderer, vulkanCommandBuffer);
+
+	vulkanCommandBuffer->autoReleaseFence = 1;
+	vulkanCommandBuffer->inFlightFence = VULKAN_INTERNAL_AcquireFenceFromPool(renderer);
+
+	submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
+	submitInfo.pNext = NULL;
+	submitInfo.commandBufferCount = 1;
+	submitInfo.pCommandBuffers = &vulkanCommandBuffer->commandBuffer;
+
+	submitInfo.pWaitDstStageMask = waitStages;
+	submitInfo.pWaitSemaphores = vulkanCommandBuffer->waitSemaphores;
+	submitInfo.waitSemaphoreCount = vulkanCommandBuffer->waitSemaphoreCount;
+	submitInfo.pSignalSemaphores = vulkanCommandBuffer->signalSemaphores;
+	submitInfo.signalSemaphoreCount = vulkanCommandBuffer->signalSemaphoreCount;
+
+	vulkanResult = renderer->vkQueueSubmit(
+		renderer->unifiedQueue,
+		1,
+		&submitInfo,
+		vulkanCommandBuffer->inFlightFence
+	);
+
+	if (vulkanResult != VK_SUCCESS)
+	{
+		LogVulkanResultAsError("vkQueueSubmit", vulkanResult);
+	}
+
+	/* Mark command buffers as submitted */
+
+	if (renderer->submittedCommandBufferCount + 1 >= renderer->submittedCommandBufferCapacity)
+	{
+		renderer->submittedCommandBufferCapacity = renderer->submittedCommandBufferCount + 1;
+
+		renderer->submittedCommandBuffers = SDL_realloc(
+			renderer->submittedCommandBuffers,
+			sizeof(VulkanCommandBuffer*) * renderer->submittedCommandBufferCapacity
+		);
+	}
+
+	renderer->submittedCommandBuffers[renderer->submittedCommandBufferCount] = vulkanCommandBuffer;
+	renderer->submittedCommandBufferCount += 1;
+
+	/* Present, if applicable */
+
+	for (j = 0; j < vulkanCommandBuffer->presentDataCount; j += 1)
+	{
+		presentData = &vulkanCommandBuffer->presentDatas[j];
+
+		presentInfo.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
+		presentInfo.pNext = NULL;
+		presentInfo.pWaitSemaphores = &presentData->windowData->swapchainData->renderFinishedSemaphore;
+		presentInfo.waitSemaphoreCount = 1;
+		presentInfo.pSwapchains = &presentData->windowData->swapchainData->swapchain;
+		presentInfo.swapchainCount = 1;
+		presentInfo.pImageIndices = &presentData->swapchainImageIndex;
+		presentInfo.pResults = NULL;
+
+		presentResult = renderer->vkQueuePresentKHR(
+			renderer->unifiedQueue,
+			&presentInfo
 		);
 
-		if (vulkanResult != VK_SUCCESS)
+		if (presentResult != VK_SUCCESS)
 		{
-			LogVulkanResultAsError("vkQueueSubmit", vulkanResult);
+			VULKAN_INTERNAL_RecreateSwapchain(
-		}
+				renderer,
-
+				presentData->windowData
-		/* Mark command buffers as submitted */
-
-		if (renderer->submittedCommandBufferCount + 1 >= renderer->submittedCommandBufferCapacity)
-		{
-			renderer->submittedCommandBufferCapacity = renderer->submittedCommandBufferCount + 1;
-
-			renderer->submittedCommandBuffers = SDL_realloc(
-				renderer->submittedCommandBuffers,
-				sizeof(VulkanCommandBuffer*) * renderer->submittedCommandBufferCapacity
 			);
 		}
-
-		renderer->submittedCommandBuffers[renderer->submittedCommandBufferCount] = (VulkanCommandBuffer*) pCommandBuffers[i];
-		renderer->submittedCommandBufferCount += 1;
-
-		/* Present, if applicable */
-
-		for (j = 0; j < currentCommandBuffer->presentDataCount; j += 1)
-		{
-			presentData = &currentCommandBuffer->presentDatas[j];
-
-			presentInfo.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
-			presentInfo.pNext = NULL;
-			presentInfo.pWaitSemaphores = &presentData->windowData->swapchainData->renderFinishedSemaphore;
-			presentInfo.waitSemaphoreCount = 1;
-			presentInfo.pSwapchains = &presentData->windowData->swapchainData->swapchain;
-			presentInfo.swapchainCount = 1;
-			presentInfo.pImageIndices = &presentData->swapchainImageIndex;
-			presentInfo.pResults = NULL;
-
-			presentResult = renderer->vkQueuePresentKHR(
-				renderer->unifiedQueue,
-				&presentInfo
-			);
-
-			if (presentResult != VK_SUCCESS)
-			{
-				VULKAN_INTERNAL_RecreateSwapchain(
-					renderer,
-					presentData->windowData
-				);
-			}
-		}
 	}
 
 	/* Check if we can perform any cleanups */
@@ -10670,8 +10737,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory(
 
 	VULKAN_Submit(
 		(Refresh_Renderer*) renderer,
-		1,
+		(Refresh_CommandBuffer*) commandBuffer
-		(Refresh_CommandBuffer**) &commandBuffer
 	);
 
 	renderer->defragInProgress = 0;
@@ -10679,6 +10745,63 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory(
 	return 1;
 }
 
+static void VULKAN_WaitForFences(
+	Refresh_Renderer *driverData,
+	uint8_t waitAll,
+	uint32_t fenceCount,
+	Refresh_Fence **pFences
+) {
+	VulkanRenderer* renderer = (VulkanRenderer*) driverData;
+	VkResult result;
+
+	result = renderer->vkWaitForFences(
+		renderer->logicalDevice,
+		fenceCount,
+		(VkFence*) pFences,
+		waitAll,
+		UINT64_MAX
+	);
+
+	if (result != VK_SUCCESS)
+	{
+		LogVulkanResultAsError("vkWaitForFences", result);
+	}
+}
+
+static int VULKAN_QueryFence(
+	Refresh_Renderer *driverData,
+	Refresh_Fence *fence
+) {
+	VulkanRenderer* renderer = (VulkanRenderer*) driverData;
+	VkResult result;
+
+	result = renderer->vkGetFenceStatus(
+		renderer->logicalDevice,
+		(VkFence) fence
+	);
+
+	if (result == VK_SUCCESS)
+	{
+		return 1;
+	}
+	else if (result == VK_NOT_READY)
+	{
+		return 0;
+	}
+	else
+	{
+		LogVulkanResultAsError("vkGetFenceStatus", result);
+		return -1;
+	}
+}
+
+static void VULKAN_ReleaseFence(
+	Refresh_Renderer *driverData,
+	Refresh_Fence *fence
+) {
+	VULKAN_INTERNAL_ReturnFenceToPool((VulkanRenderer*) driverData, (VkFence) fence);
+}
+
 /* Device instantiation */
 
 static inline uint8_t CheckDeviceExtensions(
@@ -11966,6 +12089,16 @@ static Refresh_Device* VULKAN_CreateDevice(
 		renderer->transferBufferPool.availableBufferCount += 1;
 	}
 
+	/* Initialize fence pool */
+
+	renderer->fencePool.lock = SDL_CreateMutex();
+
+	renderer->fencePool.availableFenceCapacity = 4;
+	renderer->fencePool.availableFenceCount = 0;
+	renderer->fencePool.availableFences = SDL_malloc(
+		renderer->fencePool.availableFenceCapacity * sizeof(VkFence)
+	);
+
 	/* Some drivers don't support D16, so we have to fall back to D32. */
 
 	vulkanResult = renderer->vkGetPhysicalDeviceImageFormatProperties(