deadlock fix + defrag timing tweak

src/Refresh_Driver_Vulkan.c

@@ -78,6 +78,7 @@ static uint32_t deviceExtensionCount = SDL_arraysize(deviceExtensionNames);
 #define POOLED_TRANSFER_BUFFER_SIZE 16000000    /* 16MB */
 #define UBO_BUFFER_SIZE 16000 			        /* 16KB */
 #define DESCRIPTOR_POOL_STARTING_SIZE 128
+#define DEFRAG_TIME 200
 #define WINDOW_DATA "Refresh_VulkanWindowData"
 
 #define IDENTITY_SWIZZLE 		\
@@ -1810,7 +1811,8 @@ typedef struct VulkanRenderer
 	SDL_mutex *renderTargetFetchLock;
 
 	uint8_t needDefrag;
-	uint32_t defragTimer;
+	uint64_t defragTimestamp;
+	uint8_t defragInProgress;
 
 #define VULKAN_INSTANCE_FUNCTION(ext, ret, func, params) \
 		vkfntype_##func func;
@@ -2356,6 +2358,7 @@ static void VULKAN_INTERNAL_RemoveMemoryUsedRegion(
 	if (!usedRegion->allocation->dedicated)
 	{
 		renderer->needDefrag = 1;
+		renderer->defragTimestamp = SDL_GetTicks64() + DEFRAG_TIME; /* reset timer so we batch defrags */
 	}
 
 	SDL_free(usedRegion);
@@ -3445,8 +3448,6 @@ static void VULKAN_INTERNAL_RemoveFramebuffersContainingView(
 	FramebufferHash *hash;
 	int32_t i, j;
 
-	SDL_LockMutex(renderer->framebufferFetchLock);
-
 	for (i = renderer->framebufferHashArray.count - 1; i >= 0; i -= 1)
 	{
 		hash = &renderer->framebufferHashArray.elements[i].key;
@@ -3469,8 +3470,6 @@ static void VULKAN_INTERNAL_RemoveFramebuffersContainingView(
 			}
 		}
 	}
-
-	SDL_UnlockMutex(renderer->framebufferFetchLock);
 }
 
 static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture(
@@ -3478,8 +3477,15 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture(
 	VulkanTexture *texture
 ) {
 	RenderTargetHash *hash;
+	VkImageView *viewsToCheck;
+	int32_t viewsToCheckCount;
+	int32_t viewsToCheckCapacity;
 	int32_t i;
 
+	viewsToCheckCapacity = 16;
+	viewsToCheckCount = 0;
+	viewsToCheck = SDL_malloc(sizeof(VkImageView) * viewsToCheckCapacity);
+
 	SDL_LockMutex(renderer->renderTargetFetchLock);
 
 	for (i = renderer->renderTargetHashArray.count - 1; i >= 0; i -= 1)
@@ -3488,11 +3494,17 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture(
 
 		if (hash->texture == texture)
 		{
-			VULKAN_INTERNAL_RemoveFramebuffersContainingView(
-				renderer,
-				renderer->renderTargetHashArray.elements[i].value->view
+			EXPAND_ARRAY_IF_NEEDED(
+				viewsToCheck,
+				VkImageView,
+				viewsToCheckCount + 1,
+				viewsToCheckCapacity,
+				viewsToCheckCapacity * 2
 			);
 
+			viewsToCheck[viewsToCheckCount] = renderer->renderTargetHashArray.elements[i].value->view;
+			viewsToCheckCount += 1;
+
 			VULKAN_INTERNAL_DestroyRenderTarget(
 				renderer,
 				renderer->renderTargetHashArray.elements[i].value
@@ -3502,10 +3514,26 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture(
 				&renderer->renderTargetHashArray,
 				i
 			);
+
+			Refresh_LogInfo("destroying render target!");
 		}
 	}
 
 	SDL_UnlockMutex(renderer->renderTargetFetchLock);
+
+	SDL_LockMutex(renderer->framebufferFetchLock);
+
+	for (i = 0; i < viewsToCheckCount; i += 1)
+	{
+		VULKAN_INTERNAL_RemoveFramebuffersContainingView(
+			renderer,
+			viewsToCheck[i]
+		);
+	}
+
+	SDL_UnlockMutex(renderer->framebufferFetchLock);
+
+	SDL_free(viewsToCheck);
 }
 
 static void VULKAN_INTERNAL_DestroyTexture(
@@ -3550,11 +3578,6 @@ static void VULKAN_INTERNAL_DestroyRenderTarget(
 	VulkanRenderer *renderer,
 	VulkanRenderTarget *renderTarget
 ) {
-	VULKAN_INTERNAL_RemoveFramebuffersContainingView(
-		renderer,
-		renderTarget->view
-	);
-
 	renderer->vkDestroyImageView(
 		renderer->logicalDevice,
 		renderTarget->view,
@@ -10430,11 +10453,9 @@ static void VULKAN_Submit(
 	VULKAN_INTERNAL_PerformPendingDestroys(renderer);
 
 	/* Defrag! */
-	if (renderer->needDefrag)
+	if (renderer->needDefrag && !renderer->defragInProgress)
 	{
-		renderer->defragTimer += 1;
-
-		if (renderer->defragTimer == 1)
+		if (SDL_GetTicks64() >= renderer->defragTimestamp)
 		{
 			VULKAN_INTERNAL_DefragmentMemory(renderer);
 		}
@@ -10464,6 +10485,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory(
 	SDL_LockMutex(renderer->allocatorLock);
 
 	renderer->needDefrag = 0;
+	renderer->defragInProgress = 1;
 
 	beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
 	beginInfo.pNext = NULL;
@@ -10679,13 +10701,15 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory(
 
 	SDL_UnlockMutex(renderer->allocatorLock);
 
+	renderer->defragTimestamp = SDL_GetTicks64() + DEFRAG_TIME;
+
 	VULKAN_Submit(
 		(Refresh_Renderer*) renderer,
 		1,
 		(Refresh_CommandBuffer**) &commandBuffer
 	);
 
-	renderer->defragTimer = 0;
+	renderer->defragInProgress = 0;
 
 	return 1;
 }
@@ -11988,7 +12012,8 @@ static Refresh_Device* VULKAN_CreateDevice(
 	);
 
 	renderer->needDefrag = 0;
-	renderer->defragTimer = 0;
+	renderer->defragTimestamp = 0;
+	renderer->defragInProgress = 0;
 
 	return result;
 }