From b5dcddea495bead570393663c3b065b6b9f721ac Mon Sep 17 00:00:00 2001
From: Evan Hemsley <evan@moonside.games>
Date: Thu, 11 Jan 2024 10:46:40 -0800
Subject: [PATCH] try moving layout transition into render pass

---
 src/Refresh_Driver_Vulkan.c | 72 ++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 42 deletions(-)

diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c
index 3e00de1..bdf15f2 100644
--- a/src/Refresh_Driver_Vulkan.c
+++ b/src/Refresh_Driver_Vulkan.c
@@ -5969,6 +5969,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 	VkAttachmentReference depthStencilAttachmentReference;
 	VkRenderPassCreateInfo renderPassCreateInfo;
 	VkSubpassDescription subpass;
+	VkImageLayout finalLayout;
 	VkRenderPass renderPass;
 	uint32_t i;
 
@@ -5983,6 +5984,19 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 	{
 		texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture;
 
+		if (texture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
+		{
+			finalLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+		}
+		else if (texture->usageFlags & VK_IMAGE_USAGE_STORAGE_BIT)
+		{
+			finalLayout = VK_IMAGE_LAYOUT_GENERAL;
+		}
+		else
+		{
+			finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+		}
+
 		if (texture->msaaTex != NULL)
 		{
 			msaaTexture = texture->msaaTex;
@@ -6021,7 +6035,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 			attachmentDescriptions[attachmentDescriptionCount].initialLayout =
 				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 			attachmentDescriptions[attachmentDescriptionCount].finalLayout =
-				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+				finalLayout;
 
 			resolveReferences[resolveReferenceCount].attachment =
 				attachmentDescriptionCount;
@@ -6077,7 +6091,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 			attachmentDescriptions[attachmentDescriptionCount].initialLayout =
 				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 			attachmentDescriptions[attachmentDescriptionCount].finalLayout =
-				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+				finalLayout;
 
 
 			colorAttachmentReferences[colorAttachmentReferenceCount].attachment = attachmentDescriptionCount;
@@ -6106,6 +6120,15 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 	{
 		texture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture;
 
+		if (texture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
+		{
+			finalLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+		}
+		else
+		{
+			finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+		}
+
 		attachmentDescriptions[attachmentDescriptionCount].flags = 0;
 		attachmentDescriptions[attachmentDescriptionCount].format = texture->format;
 		attachmentDescriptions[attachmentDescriptionCount].samples = RefreshToVK_SampleCount[
@@ -6126,7 +6149,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 		attachmentDescriptions[attachmentDescriptionCount].initialLayout =
 			VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
 		attachmentDescriptions[attachmentDescriptionCount].finalLayout =
-			VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+			finalLayout;
 
 		depthStencilAttachmentReference.attachment =
 			attachmentDescriptionCount;
@@ -6195,6 +6218,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateTransientRenderPass(
 	uint32_t resolveReferenceCount = 0;
 	uint32_t i;
 
+	/* Note: Render pass compatibility does not compare about layout */
 	for (i = 0; i < attachmentInfo.colorAttachmentCount; i += 1)
 	{
 		attachmentDescription = attachmentInfo.colorAttachmentDescriptions[i];
@@ -9039,35 +9063,11 @@ static void VULKAN_EndRenderPass(
 
 		if (currentTexture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
 		{
-			VULKAN_INTERNAL_ImageMemoryBarrier(
-				renderer,
-				vulkanCommandBuffer->commandBuffer,
-				RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE,
-				currentTexture->aspectFlags,
-				0,
-				currentTexture->layerCount,
-				0,
-				currentTexture->levelCount,
-				0,
-				currentTexture->image,
-				&currentTexture->resourceAccessType
-			);
+			currentTexture->resourceAccessType = RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE;
 		}
 		else if (currentTexture->usageFlags & VK_IMAGE_USAGE_STORAGE_BIT)
 		{
-			VULKAN_INTERNAL_ImageMemoryBarrier(
-				renderer,
-				vulkanCommandBuffer->commandBuffer,
-				RESOURCE_ACCESS_COMPUTE_SHADER_STORAGE_IMAGE_READ_WRITE,
-				currentTexture->aspectFlags,
-				0,
-				currentTexture->layerCount,
-				0,
-				currentTexture->levelCount,
-				0,
-				currentTexture->image,
-				&currentTexture->resourceAccessType
-			);
+			currentTexture->resourceAccessType = RESOURCE_ACCESS_COMPUTE_SHADER_STORAGE_IMAGE_READ_WRITE;
 		}
 	}
 	vulkanCommandBuffer->renderPassColorTargetCount = 0;
@@ -9078,19 +9078,7 @@ static void VULKAN_EndRenderPass(
 
 		if (currentTexture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
 		{
-			VULKAN_INTERNAL_ImageMemoryBarrier(
-				renderer,
-				vulkanCommandBuffer->commandBuffer,
-				RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE,
-				currentTexture->aspectFlags,
-				0,
-				currentTexture->layerCount,
-				0,
-				currentTexture->levelCount,
-				0,
-				currentTexture->image,
-				&currentTexture->resourceAccessType
-			);
+			currentTexture->resourceAccessType = RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE;
 		}
 	}
 	vulkanCommandBuffer->renderPassDepthTexture = NULL;