From 425694ede5d3a8a2420ade544697546d4ba55ff9 Mon Sep 17 00:00:00 2001
From: Evan Hemsley <evan@moonside.games>
Date: Thu, 11 Jan 2024 10:46:40 -0800
Subject: [PATCH] try moving layout transition into render pass

---
 src/Refresh_Driver_Vulkan.c | 72 ++++++++++++++++---------------------
 1 file changed, 30 insertions(+), 42 deletions(-)

diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c
index 5055166..71a8d29 100644
--- a/src/Refresh_Driver_Vulkan.c
+++ b/src/Refresh_Driver_Vulkan.c
@@ -5956,6 +5956,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 	VkAttachmentReference depthStencilAttachmentReference;
 	VkRenderPassCreateInfo renderPassCreateInfo;
 	VkSubpassDescription subpass;
+	VkImageLayout finalLayout;
 	VkRenderPass renderPass;
 	uint32_t i;
 
@@ -5970,6 +5971,19 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 	{
 		texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture;
 
+		if (texture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
+		{
+			finalLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+		}
+		else if (texture->usageFlags & VK_IMAGE_USAGE_STORAGE_BIT)
+		{
+			finalLayout = VK_IMAGE_LAYOUT_GENERAL;
+		}
+		else
+		{
+			finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+		}
+
 		if (texture->msaaTex != NULL)
 		{
 			msaaTexture = texture->msaaTex;
@@ -6008,7 +6022,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 			attachmentDescriptions[attachmentDescriptionCount].initialLayout =
 				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 			attachmentDescriptions[attachmentDescriptionCount].finalLayout =
-				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+				finalLayout;
 
 			resolveReferences[resolveReferenceCount].attachment =
 				attachmentDescriptionCount;
@@ -6064,7 +6078,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 			attachmentDescriptions[attachmentDescriptionCount].initialLayout =
 				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
 			attachmentDescriptions[attachmentDescriptionCount].finalLayout =
-				VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+				finalLayout;
 
 
 			colorAttachmentReferences[colorAttachmentReferenceCount].attachment = attachmentDescriptionCount;
@@ -6093,6 +6107,15 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 	{
 		texture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture;
 
+		if (texture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
+		{
+			finalLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+		}
+		else
+		{
+			finalLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+		}
+
 		attachmentDescriptions[attachmentDescriptionCount].flags = 0;
 		attachmentDescriptions[attachmentDescriptionCount].format = texture->format;
 		attachmentDescriptions[attachmentDescriptionCount].samples = RefreshToVK_SampleCount[
@@ -6113,7 +6136,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 		attachmentDescriptions[attachmentDescriptionCount].initialLayout =
 			VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
 		attachmentDescriptions[attachmentDescriptionCount].finalLayout =
-			VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+			finalLayout;
 
 		depthStencilAttachmentReference.attachment =
 			attachmentDescriptionCount;
@@ -6182,6 +6205,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateTransientRenderPass(
 	uint32_t resolveReferenceCount = 0;
 	uint32_t i;
 
+	/* Note: Render pass compatibility does not compare about layout */
 	for (i = 0; i < attachmentInfo.colorAttachmentCount; i += 1)
 	{
 		attachmentDescription = attachmentInfo.colorAttachmentDescriptions[i];
@@ -9026,35 +9050,11 @@ static void VULKAN_EndRenderPass(
 
 		if (currentTexture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
 		{
-			VULKAN_INTERNAL_ImageMemoryBarrier(
-				renderer,
-				vulkanCommandBuffer->commandBuffer,
-				RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE,
-				currentTexture->aspectFlags,
-				0,
-				currentTexture->layerCount,
-				0,
-				currentTexture->levelCount,
-				0,
-				currentTexture->image,
-				&currentTexture->resourceAccessType
-			);
+			currentTexture->resourceAccessType = RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE;
 		}
 		else if (currentTexture->usageFlags & VK_IMAGE_USAGE_STORAGE_BIT)
 		{
-			VULKAN_INTERNAL_ImageMemoryBarrier(
-				renderer,
-				vulkanCommandBuffer->commandBuffer,
-				RESOURCE_ACCESS_COMPUTE_SHADER_STORAGE_IMAGE_READ_WRITE,
-				currentTexture->aspectFlags,
-				0,
-				currentTexture->layerCount,
-				0,
-				currentTexture->levelCount,
-				0,
-				currentTexture->image,
-				&currentTexture->resourceAccessType
-			);
+			currentTexture->resourceAccessType = RESOURCE_ACCESS_COMPUTE_SHADER_STORAGE_IMAGE_READ_WRITE;
 		}
 	}
 	vulkanCommandBuffer->renderPassColorTargetCount = 0;
@@ -9065,19 +9065,7 @@ static void VULKAN_EndRenderPass(
 
 		if (currentTexture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT)
 		{
-			VULKAN_INTERNAL_ImageMemoryBarrier(
-				renderer,
-				vulkanCommandBuffer->commandBuffer,
-				RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE,
-				currentTexture->aspectFlags,
-				0,
-				currentTexture->layerCount,
-				0,
-				currentTexture->levelCount,
-				0,
-				currentTexture->image,
-				&currentTexture->resourceAccessType
-			);
+			currentTexture->resourceAccessType = RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE;
 		}
 	}
 	vulkanCommandBuffer->renderPassDepthTexture = NULL;