miscellaneous API changes + more MSAA fixes

- Fixed bug where multisample enum values were not being translated into their equivalent Vulkan enums
- Fixed bug where MSAA attachments in transient render pass creation were using a sample count of 1
- Fixed bug where the clearValues array in BeginRenderPass was not being populated with clear values for multisample attachments. (Modeled the fix after https://github.com/FNA-XNA/FNA3D/blob/master/src/FNA3D_Driver_Vulkan.c#L8723)
- Fixed bug where the multisample texture was not being transitioned when beginning a render pass

Reviewed-on: MoonsideGames/Refresh#25
Co-authored-by: TheSpydog <thespydog@noreply.example.org>
Co-committed-by: TheSpydog <thespydog@noreply.example.org>

include/Refresh.h

@@ -169,10 +169,7 @@ typedef enum Refresh_SampleCount
 	REFRESH_SAMPLECOUNT_1,
 	REFRESH_SAMPLECOUNT_2,
 	REFRESH_SAMPLECOUNT_4,
-	REFRESH_SAMPLECOUNT_8,
+	REFRESH_SAMPLECOUNT_8
-	REFRESH_SAMPLECOUNT_16,
-	REFRESH_SAMPLECOUNT_32,
-	REFRESH_SAMPLECOUNT_64
 } Refresh_SampleCount;
 
 typedef enum Refresh_CubeMapFace
@@ -467,7 +464,6 @@ typedef struct Refresh_TextureCreateInfo
 	uint32_t height;
 	uint32_t depth;
 	uint8_t isCube;
-	Refresh_SampleCount sampleCount;
 	uint32_t levelCount;
 	Refresh_TextureFormat format;
 	Refresh_TextureUsageFlags usageFlags;
@@ -526,7 +522,6 @@ typedef struct Refresh_DepthStencilState
 typedef struct Refresh_ColorAttachmentDescription
 {
 	Refresh_TextureFormat format;
-	Refresh_SampleCount sampleCount;
 	Refresh_ColorAttachmentBlendState blendState;
 } Refresh_ColorAttachmentDescription;
 
@@ -1000,11 +995,6 @@ REFRESHAPI void Refresh_QueueDestroyGraphicsPipeline(
 /* Begins a render pass.
  * This will also set a default viewport and scissor state.
  *
- * renderArea:
- * 		The area affected by the render pass.
- * 		All load, store and resolve operations are restricted
- * 		to the given rectangle.
- * 		If NULL, a sensible default will be chosen.
  * colorAttachmentInfos:
  * 		A pointer to an array of Refresh_ColorAttachmentInfo structures
  * 		that contains render targets and clear values. May be NULL.
@@ -1014,7 +1004,6 @@ REFRESHAPI void Refresh_QueueDestroyGraphicsPipeline(
 REFRESHAPI void Refresh_BeginRenderPass(
 	Refresh_Device *device,
 	Refresh_CommandBuffer *commandBuffer,
-	Refresh_Rect *renderArea,
 	Refresh_ColorAttachmentInfo *colorAttachmentInfos,
 	uint32_t colorAttachmentCount,
 	Refresh_DepthStencilAttachmentInfo *depthStencilAttachmentInfo

src/Refresh.c

@@ -625,7 +625,6 @@ void Refresh_QueueDestroyGraphicsPipeline(
 void Refresh_BeginRenderPass(
 	Refresh_Device *device,
 	Refresh_CommandBuffer *commandBuffer,
-	Refresh_Rect *renderArea,
 	Refresh_ColorAttachmentInfo *colorAttachmentInfos,
 	uint32_t colorAttachmentCount,
 	Refresh_DepthStencilAttachmentInfo *depthStencilAttachmentInfo
@@ -634,7 +633,6 @@ void Refresh_BeginRenderPass(
 	device->BeginRenderPass(
 		device->driverData,
 		commandBuffer,
-		renderArea,
 		colorAttachmentInfos,
 		colorAttachmentCount,
 		depthStencilAttachmentInfo

src/Refresh_Driver.h

@@ -395,7 +395,6 @@ struct Refresh_Device
 	void(*BeginRenderPass)(
 		Refresh_Renderer *driverData,
 		Refresh_CommandBuffer *commandBuffer,
-		Refresh_Rect *renderArea,
 		Refresh_ColorAttachmentInfo *colorAttachmentInfos,
 		uint32_t colorAttachmentCount,
 		Refresh_DepthStencilAttachmentInfo *depthStencilAttachmentInfo

src/Refresh_Driver_Vulkan.c

@@ -971,6 +971,7 @@ typedef struct RenderPassHash
 	RenderPassColorTargetDescription colorTargetDescriptions[MAX_COLOR_TARGET_BINDINGS];
 	uint32_t colorAttachmentCount;
 	RenderPassDepthStencilTargetDescription depthStencilTargetDescription;
+	Refresh_SampleCount colorAttachmentSampleCount;
 } RenderPassHash;
 
 typedef struct RenderPassHashMap
@@ -997,6 +998,11 @@ static inline uint8_t RenderPassHash_Compare(
 		return 0;
 	}
 
+	if (a->colorAttachmentSampleCount != b->colorAttachmentSampleCount)
+	{
+		return 0;
+	}
+
 	for (i = 0; i < a->colorAttachmentCount; i += 1)
 	{
 		if (a->colorTargetDescriptions[i].format != b->colorTargetDescriptions[i].format)
@@ -2023,6 +2029,29 @@ static inline VkDeviceSize VULKAN_INTERNAL_BytesPerImage(
 	return blocksPerRow * blocksPerColumn * VULKAN_INTERNAL_BytesPerPixel(format);
 }
 
+static inline Refresh_SampleCount VULKAN_INTERNAL_GetMaxMultiSampleCount(
+	VulkanRenderer *renderer,
+	Refresh_SampleCount multiSampleCount
+) {
+	VkSampleCountFlags flags = renderer->physicalDeviceProperties.properties.limits.framebufferColorSampleCounts;
+	Refresh_SampleCount maxSupported = REFRESH_SAMPLECOUNT_1;
+
+	if (flags & VK_SAMPLE_COUNT_8_BIT)
+	{
+		maxSupported = REFRESH_SAMPLECOUNT_8;
+	}
+	else if (flags & VK_SAMPLE_COUNT_4_BIT)
+	{
+		maxSupported = REFRESH_SAMPLECOUNT_4;
+	}
+	else if (flags & VK_SAMPLE_COUNT_2_BIT)
+	{
+		maxSupported = REFRESH_SAMPLECOUNT_2;
+	}
+
+	return SDL_min(multiSampleCount, maxSupported);
+}
+
 /* Memory Management */
 
 static inline VkDeviceSize VULKAN_INTERNAL_NextHighestAlignment(
@@ -5431,6 +5460,12 @@ static VulkanRenderTarget* VULKAN_INTERNAL_CreateRenderTarget(
 	/* create resolve target for multisample */
 	if (multisampleCount > REFRESH_SAMPLECOUNT_1)
 	{
+		/* Find a compatible sample count to use */
+		multisampleCount = VULKAN_INTERNAL_GetMaxMultiSampleCount(
+			renderer,
+			multisampleCount
+		);
+
 		renderTarget->multisampleTexture =
 			VULKAN_INTERNAL_CreateTexture(
 				renderer,
@@ -5446,7 +5481,7 @@ static VulkanRenderTarget* VULKAN_INTERNAL_CreateRenderTarget(
 				VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT | VK_IMAGE_USAGE_TRANSFER_DST_BIT
 			);
 
-		renderTarget->multisampleCount = multisampleCount;
+		renderTarget->multisampleCount = RefreshToVK_SampleCount[multisampleCount];
 	}
 
 	/* create framebuffer compatible views for RenderTarget */
@@ -5540,6 +5575,7 @@ static VulkanRenderTarget* VULKAN_INTERNAL_FetchRenderTarget(
 
 static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 	VulkanRenderer *renderer,
+	VulkanCommandBuffer *commandBuffer,
 	Refresh_ColorAttachmentInfo *colorAttachmentInfos,
 	uint32_t colorAttachmentCount,
 	Refresh_DepthStencilAttachmentInfo *depthStencilAttachmentInfo
@@ -5579,6 +5615,22 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 		{
 			multisampling = 1;
 
+			/* Transition the multisample attachment */
+
+			VULKAN_INTERNAL_ImageMemoryBarrier(
+				renderer,
+				commandBuffer->commandBuffer,
+				RESOURCE_ACCESS_COLOR_ATTACHMENT_WRITE,
+				VK_IMAGE_ASPECT_COLOR_BIT,
+				0,
+				renderTarget->multisampleTexture->layerCount,
+				0,
+				renderTarget->multisampleTexture->levelCount,
+				0,
+				renderTarget->multisampleTexture->image,
+				&renderTarget->multisampleTexture->resourceAccessType
+			);
+
 			/* Resolve attachment and multisample attachment */
 
 			attachmentDescriptions[attachmentDescriptionCount].flags = 0;
@@ -5588,9 +5640,8 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 			attachmentDescriptions[attachmentDescriptionCount].loadOp = RefreshToVK_LoadOp[
 				colorAttachmentInfos[i].loadOp
 			];
-			attachmentDescriptions[attachmentDescriptionCount].storeOp = RefreshToVK_StoreOp[
+			attachmentDescriptions[attachmentDescriptionCount].storeOp =
-				colorAttachmentInfos[i].storeOp
+				VK_ATTACHMENT_STORE_OP_STORE; /* Always store the resolve texture */
-			];
 			attachmentDescriptions[attachmentDescriptionCount].stencilLoadOp =
 				VK_ATTACHMENT_LOAD_OP_DONT_CARE;
 			attachmentDescriptions[attachmentDescriptionCount].stencilStoreOp =
@@ -5643,9 +5694,8 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 			attachmentDescriptions[attachmentDescriptionCount].loadOp = RefreshToVK_LoadOp[
 				colorAttachmentInfos[i].loadOp
 			];
-			attachmentDescriptions[attachmentDescriptionCount].storeOp = RefreshToVK_StoreOp[
+			attachmentDescriptions[attachmentDescriptionCount].storeOp =
-				colorAttachmentInfos[i].storeOp
+				VK_ATTACHMENT_STORE_OP_STORE; /* Always store non-MSAA textures */
-			];
 			attachmentDescriptions[attachmentDescriptionCount].stencilLoadOp =
 				VK_ATTACHMENT_LOAD_OP_DONT_CARE;
 			attachmentDescriptions[attachmentDescriptionCount].stencilStoreOp =
@@ -5760,7 +5810,8 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass(
 
 static VkRenderPass VULKAN_INTERNAL_CreateTransientRenderPass(
 	VulkanRenderer *renderer,
-	Refresh_GraphicsPipelineAttachmentInfo attachmentInfo
+	Refresh_GraphicsPipelineAttachmentInfo attachmentInfo,
+	Refresh_SampleCount sampleCount
 ) {
 	VkAttachmentDescription attachmentDescriptions[2 * MAX_COLOR_TARGET_BINDINGS + 1];
 	VkAttachmentReference colorAttachmentReferences[MAX_COLOR_TARGET_BINDINGS];
@@ -5782,7 +5833,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateTransientRenderPass(
 	{
 		attachmentDescription = attachmentInfo.colorAttachmentDescriptions[i];
 
-		if (attachmentDescription.sampleCount > REFRESH_SAMPLECOUNT_1)
+		if (sampleCount > REFRESH_SAMPLECOUNT_1)
 		{
 			multisampling = 1;
 
@@ -5810,7 +5861,9 @@ static VkRenderPass VULKAN_INTERNAL_CreateTransientRenderPass(
 			attachmentDescriptions[attachmentDescriptionCount].format = RefreshToVK_SurfaceFormat[
 				attachmentDescription.format
 			];
-			attachmentDescriptions[attachmentDescriptionCount].samples = VK_SAMPLE_COUNT_1_BIT;
+			attachmentDescriptions[attachmentDescriptionCount].samples = RefreshToVK_SampleCount[
+				sampleCount
+			];
 			attachmentDescriptions[attachmentDescriptionCount].loadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
 			attachmentDescriptions[attachmentDescriptionCount].storeOp = VK_ATTACHMENT_STORE_OP_DONT_CARE;
 			attachmentDescriptions[attachmentDescriptionCount].stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE;
@@ -5940,6 +5993,7 @@ static Refresh_GraphicsPipeline* VULKAN_CreateGraphicsPipeline(
 ) {
 	VkResult vulkanResult;
 	uint32_t i;
+	Refresh_SampleCount actualSampleCount;
 
 	VulkanGraphicsPipeline *graphicsPipeline = (VulkanGraphicsPipeline*) SDL_malloc(sizeof(VulkanGraphicsPipeline));
 	VkGraphicsPipelineCreateInfo vkPipelineCreateInfo;
@@ -5977,11 +6031,19 @@ static Refresh_GraphicsPipeline* VULKAN_CreateGraphicsPipeline(
 
 	VulkanRenderer *renderer = (VulkanRenderer*) driverData;
 
+	/* Find a compatible sample count to use */
+
+	actualSampleCount = VULKAN_INTERNAL_GetMaxMultiSampleCount(
+		renderer,
+		pipelineCreateInfo->multisampleState.multisampleCount
+	);
+
 	/* Create a "compatible" render pass */
 
 	VkRenderPass transientRenderPass = VULKAN_INTERNAL_CreateTransientRenderPass(
 		renderer,
-		pipelineCreateInfo->attachmentInfo
+		pipelineCreateInfo->attachmentInfo,
+		actualSampleCount
 	);
 
 	/* Dynamic state */
@@ -6112,9 +6174,7 @@ static Refresh_GraphicsPipeline* VULKAN_CreateGraphicsPipeline(
 	multisampleStateCreateInfo.sType = VK_STRUCTURE_TYPE_PIPELINE_MULTISAMPLE_STATE_CREATE_INFO;
 	multisampleStateCreateInfo.pNext = NULL;
 	multisampleStateCreateInfo.flags = 0;
-	multisampleStateCreateInfo.rasterizationSamples = RefreshToVK_SampleCount[
+	multisampleStateCreateInfo.rasterizationSamples = RefreshToVK_SampleCount[actualSampleCount];
-		pipelineCreateInfo->multisampleState.multisampleCount
-	];
 	multisampleStateCreateInfo.sampleShadingEnable = VK_FALSE;
 	multisampleStateCreateInfo.minSampleShading = 1.0f;
 	multisampleStateCreateInfo.pSampleMask =
@@ -7980,6 +8040,7 @@ static void VULKAN_QueueDestroyGraphicsPipeline(
 
 static VkRenderPass VULKAN_INTERNAL_FetchRenderPass(
 	VulkanRenderer *renderer,
+	VulkanCommandBuffer *commandBuffer,
 	Refresh_ColorAttachmentInfo *colorAttachmentInfos,
 	uint32_t colorAttachmentCount,
 	Refresh_DepthStencilAttachmentInfo *depthStencilAttachmentInfo
@@ -7998,6 +8059,11 @@ static VkRenderPass VULKAN_INTERNAL_FetchRenderPass(
 		hash.colorTargetDescriptions[i].storeOp = colorAttachmentInfos[i].storeOp;
 	}
 
+	if (colorAttachmentCount > 0)
+	{
+		hash.colorAttachmentSampleCount = colorAttachmentInfos[0].sampleCount;
+	}
+
 	hash.colorAttachmentCount = colorAttachmentCount;
 
 	if (depthStencilAttachmentInfo == NULL)
@@ -8030,6 +8096,7 @@ static VkRenderPass VULKAN_INTERNAL_FetchRenderPass(
 
 	renderPass = VULKAN_INTERNAL_CreateRenderPass(
 		renderer,
+		commandBuffer,
 		colorAttachmentInfos,
 		colorAttachmentCount,
 		depthStencilAttachmentInfo
@@ -8294,7 +8361,6 @@ static void VULKAN_SetScissor(
 static void VULKAN_BeginRenderPass(
 	Refresh_Renderer *driverData,
 	Refresh_CommandBuffer *commandBuffer,
-	Refresh_Rect *renderArea,
 	Refresh_ColorAttachmentInfo *colorAttachmentInfos,
 	uint32_t colorAttachmentCount,
 	Refresh_DepthStencilAttachmentInfo *depthStencilAttachmentInfo
@@ -8307,6 +8373,7 @@ static void VULKAN_BeginRenderPass(
 	VulkanTexture *texture;
 	VkClearValue *clearValues;
 	uint32_t clearCount = colorAttachmentCount;
+	uint32_t multisampleAttachmentCount = 0;
 	uint32_t i;
 	VkImageAspectFlags depthAspectFlags;
 	Refresh_Viewport defaultViewport;
@@ -8356,6 +8423,7 @@ static void VULKAN_BeginRenderPass(
 
 	renderPass = VULKAN_INTERNAL_FetchRenderPass(
 		renderer,
+		vulkanCommandBuffer,
 		colorAttachmentInfos,
 		colorAttachmentCount,
 		depthStencilAttachmentInfo
@@ -8393,6 +8461,12 @@ static void VULKAN_BeginRenderPass(
 			&texture->resourceAccessType
 		);
 
+		if (colorAttachmentInfos[i].sampleCount > REFRESH_SAMPLECOUNT_1)
+		{
+			clearCount += 1;
+			multisampleAttachmentCount += 1;
+		}
+
 		VULKAN_INTERNAL_TrackTexture(renderer, vulkanCommandBuffer, texture);
 	}
 
@@ -8430,12 +8504,21 @@ static void VULKAN_BeginRenderPass(
 
 	clearValues = SDL_stack_alloc(VkClearValue, clearCount);
 
-	for (i = 0; i < colorAttachmentCount; i += 1)
+	for (i = 0; i < colorAttachmentCount + multisampleAttachmentCount; i += 1)
 	{
 		clearValues[i].color.float32[0] = colorAttachmentInfos[i].clearColor.x;
 		clearValues[i].color.float32[1] = colorAttachmentInfos[i].clearColor.y;
 		clearValues[i].color.float32[2] = colorAttachmentInfos[i].clearColor.z;
 		clearValues[i].color.float32[3] = colorAttachmentInfos[i].clearColor.w;
+
+		if (colorAttachmentInfos[i].sampleCount > REFRESH_SAMPLECOUNT_1)
+		{
+			i += 1;
+			clearValues[i].color.float32[0] = colorAttachmentInfos[i].clearColor.x;
+			clearValues[i].color.float32[1] = colorAttachmentInfos[i].clearColor.y;
+			clearValues[i].color.float32[2] = colorAttachmentInfos[i].clearColor.z;
+			clearValues[i].color.float32[3] = colorAttachmentInfos[i].clearColor.w;
+		}
 	}
 
 	if (depthStencilAttachmentInfo != NULL)
@@ -8453,21 +8536,10 @@ static void VULKAN_BeginRenderPass(
 	renderPassBeginInfo.framebuffer = framebuffer->framebuffer;
 	renderPassBeginInfo.pClearValues = clearValues;
 	renderPassBeginInfo.clearValueCount = clearCount;
-
-	if (renderArea != NULL)
-	{
-		renderPassBeginInfo.renderArea.extent.width = renderArea->w;
-		renderPassBeginInfo.renderArea.extent.height = renderArea->h;
-		renderPassBeginInfo.renderArea.offset.x = renderArea->x;
-		renderPassBeginInfo.renderArea.offset.y = renderArea->y;
-	}
-	else
-	{
 	renderPassBeginInfo.renderArea.extent.width = framebufferWidth;
 	renderPassBeginInfo.renderArea.extent.height = framebufferHeight;
 	renderPassBeginInfo.renderArea.offset.x = 0;
 	renderPassBeginInfo.renderArea.offset.y = 0;
-	}
 
 	renderer->vkCmdBeginRenderPass(
 		vulkanCommandBuffer->commandBuffer,