From 746ad10c7baa4599055812eba2d35ff183c043fa Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Fri, 12 May 2023 15:11:18 -0700 Subject: [PATCH 01/12] start on memory management rewrite --- src/Refresh_Driver_Vulkan.c | 1515 +++++++++++++++++++++++++---------- 1 file changed, 1095 insertions(+), 420 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index 2deae02..95913ff 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -71,12 +71,12 @@ static uint32_t deviceExtensionCount = SDL_arraysize(deviceExtensionNames); /* Defines */ -#define STARTING_ALLOCATION_SIZE 64000000 /* 64MB */ -#define MAX_ALLOCATION_SIZE 256000000 /* 256MB */ +#define STARTING_ALLOCATION_SIZE 64000000 /* 64MB */ +#define MAX_ALLOCATION_SIZE 256000000 /* 256MB */ +#define ALLOCATION_INCREMENT 16000000 /* 16MB */ #define TRANSFER_BUFFER_STARTING_SIZE 8000000 /* 8MB */ -#define UBO_BUFFER_SIZE 16000 /* 16KB */ +#define UBO_BUFFER_SIZE 16000 /* 16KB */ #define DESCRIPTOR_POOL_STARTING_SIZE 128 -#define DESCRIPTOR_SET_DEACTIVATE_FRAMES 10 #define WINDOW_DATA "Refresh_VulkanWindowData" #define IDENTITY_SWIZZLE \ @@ -389,6 +389,8 @@ static VkBorderColor RefreshToVK_BorderColor[] = /* Memory Allocation */ typedef struct VulkanMemoryAllocation VulkanMemoryAllocation; +typedef struct VulkanBuffer VulkanBuffer; +typedef struct VulkanTexture VulkanTexture; typedef struct VulkanMemoryFreeRegion { @@ -399,8 +401,25 @@ typedef struct VulkanMemoryFreeRegion uint32_t sortedIndex; } VulkanMemoryFreeRegion; +typedef struct VulkanMemoryUsedRegion +{ + VulkanMemoryAllocation *allocation; + VkDeviceSize offset; + VkDeviceSize size; + VkDeviceSize resourceOffset; /* differs from offset based on alignment*/ + VkDeviceSize resourceSize; /* differs from size based on alignment */ + VkDeviceSize alignment; + uint8_t isBuffer; + REFRESHNAMELESS union + { + VulkanBuffer *vulkanBuffer; + VulkanTexture *vulkanTexture; + }; +} VulkanMemoryUsedRegion; + typedef struct VulkanMemorySubAllocator { + uint32_t memoryTypeIndex; VkDeviceSize nextAllocationSize; VulkanMemoryAllocation **allocations; uint32_t allocationCount; @@ -414,10 +433,16 @@ struct VulkanMemoryAllocation VulkanMemorySubAllocator *allocator; VkDeviceMemory memory; VkDeviceSize size; + VulkanMemoryUsedRegion **usedRegions; + uint32_t usedRegionCount; + uint32_t usedRegionCapacity; VulkanMemoryFreeRegion **freeRegions; uint32_t freeRegionCount; uint32_t freeRegionCapacity; uint8_t dedicated; + uint8_t availableForAllocation; + VkDeviceSize freeSpace; + VkDeviceSize usedSpace; uint8_t *mapPointer; SDL_mutex *memoryLock; }; @@ -664,18 +689,16 @@ static const VulkanResourceAccessInfo AccessMap[RESOURCE_ACCESS_TYPES_COUNT] = /* Memory structures */ -typedef struct VulkanBuffer /* cast from Refresh_Buffer */ +struct VulkanBuffer /* cast from Refresh_Buffer */ { VkBuffer buffer; VkDeviceSize size; - VkDeviceSize offset; /* move this to UsedMemoryRegion system */ - VkDeviceSize memorySize; /* move this to UsedMemoryRegion system */ - VulkanMemoryAllocation *allocation; /* see above */ + VulkanMemoryUsedRegion *usedRegion; VulkanResourceAccessType resourceAccessType; VkBufferUsageFlags usage; SDL_atomic_t referenceCount; /* Tracks command buffer usage */ -} VulkanBuffer; +}; typedef struct VulkanUniformBufferPool VulkanUniformBufferPool; @@ -742,11 +765,9 @@ typedef struct VulkanShaderModule SDL_atomic_t referenceCount; } VulkanShaderModule; -typedef struct VulkanTexture +struct VulkanTexture { - VulkanMemoryAllocation *allocation; - VkDeviceSize offset; - VkDeviceSize memorySize; + VulkanMemoryUsedRegion *usedRegion; VkImage image; VkImageView view; @@ -768,7 +789,7 @@ typedef struct VulkanTexture struct VulkanTexture *msaaTex; SDL_atomic_t referenceCount; -} VulkanTexture; +}; typedef struct VulkanRenderTarget { @@ -1481,15 +1502,6 @@ typedef struct VulkanTransferBuffer VkDeviceSize offset; } VulkanTransferBuffer; -typedef struct VulkanTransferBufferPool -{ - SDL_mutex *lock; - - VulkanTransferBuffer **availableBuffers; - uint32_t availableBufferCount; - uint32_t availableBufferCapacity; -} VulkanTransferBufferPool; - typedef struct VulkanCommandPool VulkanCommandPool; typedef struct VulkanCommandBuffer @@ -1673,6 +1685,7 @@ typedef struct VulkanRenderer VkPhysicalDeviceProperties2 physicalDeviceProperties; VkPhysicalDeviceDriverPropertiesKHR physicalDeviceDriverProperties; VkDevice logicalDevice; + uint8_t unifiedMemoryWarning; uint8_t supportsDebugUtils; uint8_t debugMode; @@ -1694,8 +1707,6 @@ typedef struct VulkanRenderer uint32_t submittedCommandBufferCount; uint32_t submittedCommandBufferCapacity; - VulkanTransferBufferPool transferBufferPool; - CommandPoolHashTable commandPoolHashTable; DescriptorSetLayoutHashTable descriptorSetLayoutHashTable; GraphicsPipelineLayoutHashTable graphicsPipelineLayoutHashTable; @@ -1769,7 +1780,11 @@ typedef struct VulkanRenderer SDL_mutex *framebufferFetchLock; SDL_mutex *renderTargetFetchLock; - #define VULKAN_INSTANCE_FUNCTION(ext, ret, func, params) \ + uint8_t needDefrag; + uint32_t defragTimer; + uint8_t resourceFreed; + +#define VULKAN_INSTANCE_FUNCTION(ext, ret, func, params) \ vkfntype_##func func; #define VULKAN_DEVICE_FUNCTION(ext, ret, func, params) \ vkfntype_##func func; @@ -1840,6 +1855,13 @@ static inline void LogVulkanResultAsWarn( } } +#define VULKAN_ERROR_CHECK(res, fn, ret) \ + if (res != VK_SUCCESS) \ + { \ + FNA3D_LogError("%s %s", #fn, VkErrorMessages(res)); \ + return ret; \ + } + /* Utility */ static inline VkFormat RefreshToVK_DepthFormat( @@ -2043,6 +2065,8 @@ static inline Refresh_SampleCount VULKAN_INTERNAL_GetMaxMultiSampleCount( /* Memory Management */ +/* Vulkan: Memory Allocation */ + static inline VkDeviceSize VULKAN_INTERNAL_NextHighestAlignment( VkDeviceSize n, VkDeviceSize align @@ -2050,24 +2074,59 @@ static inline VkDeviceSize VULKAN_INTERNAL_NextHighestAlignment( return align * ((n + align - 1) / align); } +static void VULKAN_INTERNAL_MakeMemoryUnavailable( + VulkanRenderer* renderer, + VulkanMemoryAllocation *allocation +) { + uint32_t i, j; + VulkanMemoryFreeRegion *freeRegion; + + allocation->availableForAllocation = 0; + + for (i = 0; i < allocation->freeRegionCount; i += 1) + { + freeRegion = allocation->freeRegions[i]; + + /* close the gap in the sorted list */ + if (allocation->allocator->sortedFreeRegionCount > 1) + { + for (j = freeRegion->sortedIndex; j < allocation->allocator->sortedFreeRegionCount - 1; j += 1) + { + allocation->allocator->sortedFreeRegions[j] = + allocation->allocator->sortedFreeRegions[j + 1]; + + allocation->allocator->sortedFreeRegions[j]->sortedIndex = j; + } + } + + allocation->allocator->sortedFreeRegionCount -= 1; + } +} + static void VULKAN_INTERNAL_RemoveMemoryFreeRegion( + VulkanRenderer *renderer, VulkanMemoryFreeRegion *freeRegion ) { uint32_t i; - /* close the gap in the sorted list */ - if (freeRegion->allocation->allocator->sortedFreeRegionCount > 1) + SDL_LockMutex(renderer->allocatorLock); + + if (freeRegion->allocation->availableForAllocation) { - for (i = freeRegion->sortedIndex; i < freeRegion->allocation->allocator->sortedFreeRegionCount - 1; i += 1) + /* close the gap in the sorted list */ + if (freeRegion->allocation->allocator->sortedFreeRegionCount > 1) { - freeRegion->allocation->allocator->sortedFreeRegions[i] = - freeRegion->allocation->allocator->sortedFreeRegions[i + 1]; + for (i = freeRegion->sortedIndex; i < freeRegion->allocation->allocator->sortedFreeRegionCount - 1; i += 1) + { + freeRegion->allocation->allocator->sortedFreeRegions[i] = + freeRegion->allocation->allocator->sortedFreeRegions[i + 1]; - freeRegion->allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; + freeRegion->allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; + } } - } - freeRegion->allocation->allocator->sortedFreeRegionCount -= 1; + freeRegion->allocation->allocator->sortedFreeRegionCount -= 1; + } /* close the gap in the buffer list */ if (freeRegion->allocation->freeRegionCount > 1 && freeRegion->allocationIndex != freeRegion->allocation->freeRegionCount - 1) @@ -2081,10 +2140,15 @@ static void VULKAN_INTERNAL_RemoveMemoryFreeRegion( freeRegion->allocation->freeRegionCount -= 1; + freeRegion->allocation->freeSpace -= freeRegion->size; + SDL_free(freeRegion); + + SDL_UnlockMutex(renderer->allocatorLock); } static void VULKAN_INTERNAL_NewMemoryFreeRegion( + VulkanRenderer *renderer, VulkanMemoryAllocation *allocation, VkDeviceSize offset, VkDeviceSize size @@ -2094,6 +2158,8 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( int32_t insertionIndex = 0; int32_t i; + SDL_LockMutex(renderer->allocatorLock); + /* look for an adjacent region to merge */ for (i = allocation->freeRegionCount - 1; i >= 0; i -= 1) { @@ -2103,8 +2169,10 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( newOffset = allocation->freeRegions[i]->offset; newSize = allocation->freeRegions[i]->size + size; - VULKAN_INTERNAL_RemoveMemoryFreeRegion(allocation->freeRegions[i]); - VULKAN_INTERNAL_NewMemoryFreeRegion(allocation, newOffset, newSize); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, allocation->freeRegions[i]); + VULKAN_INTERNAL_NewMemoryFreeRegion(renderer, allocation, newOffset, newSize); + + SDL_UnlockMutex(renderer->allocatorLock); return; } @@ -2114,8 +2182,10 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( newOffset = offset; newSize = allocation->freeRegions[i]->size + size; - VULKAN_INTERNAL_RemoveMemoryFreeRegion(allocation->freeRegions[i]); - VULKAN_INTERNAL_NewMemoryFreeRegion(allocation, newOffset, newSize); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, allocation->freeRegions[i]); + VULKAN_INTERNAL_NewMemoryFreeRegion(renderer, allocation, newOffset, newSize); + + SDL_UnlockMutex(renderer->allocatorLock); return; } } @@ -2136,71 +2206,140 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( newFreeRegion->size = size; newFreeRegion->allocation = allocation; + allocation->freeSpace += size; + allocation->freeRegions[allocation->freeRegionCount - 1] = newFreeRegion; newFreeRegion->allocationIndex = allocation->freeRegionCount - 1; - for (i = 0; i < allocation->allocator->sortedFreeRegionCount; i += 1) + if (allocation->availableForAllocation) { - if (allocation->allocator->sortedFreeRegions[i]->size < size) + for (i = 0; i < allocation->allocator->sortedFreeRegionCount; i += 1) { - /* this is where the new region should go */ - break; + if (allocation->allocator->sortedFreeRegions[i]->size < size) + { + /* this is where the new region should go */ + break; + } + + insertionIndex += 1; } - insertionIndex += 1; + if (allocation->allocator->sortedFreeRegionCount + 1 > allocation->allocator->sortedFreeRegionCapacity) + { + allocation->allocator->sortedFreeRegionCapacity *= 2; + allocation->allocator->sortedFreeRegions = SDL_realloc( + allocation->allocator->sortedFreeRegions, + sizeof(VulkanMemoryFreeRegion*) * allocation->allocator->sortedFreeRegionCapacity + ); + } + + /* perform insertion sort */ + if (allocation->allocator->sortedFreeRegionCount > 0 && insertionIndex != allocation->allocator->sortedFreeRegionCount) + { + for (i = allocation->allocator->sortedFreeRegionCount; i > insertionIndex && i > 0; i -= 1) + { + allocation->allocator->sortedFreeRegions[i] = allocation->allocator->sortedFreeRegions[i - 1]; + allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; + } + } + + allocation->allocator->sortedFreeRegionCount += 1; + allocation->allocator->sortedFreeRegions[insertionIndex] = newFreeRegion; + newFreeRegion->sortedIndex = insertionIndex; } - if (allocation->allocator->sortedFreeRegionCount + 1 > allocation->allocator->sortedFreeRegionCapacity) + SDL_UnlockMutex(renderer->allocatorLock); +} + +static VulkanMemoryUsedRegion* VULKAN_INTERNAL_NewMemoryUsedRegion( + VulkanRenderer *renderer, + VulkanMemoryAllocation *allocation, + VkDeviceSize offset, + VkDeviceSize size, + VkDeviceSize resourceOffset, + VkDeviceSize resourceSize, + VkDeviceSize alignment +) { + VulkanMemoryUsedRegion *memoryUsedRegion; + + SDL_LockMutex(renderer->allocatorLock); + + if (allocation->usedRegionCount == allocation->usedRegionCapacity) { - allocation->allocator->sortedFreeRegionCapacity *= 2; - allocation->allocator->sortedFreeRegions = SDL_realloc( - allocation->allocator->sortedFreeRegions, - sizeof(VulkanMemoryFreeRegion*) * allocation->allocator->sortedFreeRegionCapacity + allocation->usedRegionCapacity *= 2; + allocation->usedRegions = SDL_realloc( + allocation->usedRegions, + allocation->usedRegionCapacity * sizeof(VulkanMemoryUsedRegion*) ); } - /* perform insertion sort */ - if (allocation->allocator->sortedFreeRegionCount > 0 && insertionIndex != allocation->allocator->sortedFreeRegionCount) - { - for (i = allocation->allocator->sortedFreeRegionCount; i > insertionIndex && i > 0; i -= 1) - { - allocation->allocator->sortedFreeRegions[i] = allocation->allocator->sortedFreeRegions[i - 1]; - allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; - } - } + memoryUsedRegion = SDL_malloc(sizeof(VulkanMemoryUsedRegion)); + memoryUsedRegion->allocation = allocation; + memoryUsedRegion->offset = offset; + memoryUsedRegion->size = size; + memoryUsedRegion->resourceOffset = resourceOffset; + memoryUsedRegion->resourceSize = resourceSize; + memoryUsedRegion->alignment = alignment; - allocation->allocator->sortedFreeRegionCount += 1; - allocation->allocator->sortedFreeRegions[insertionIndex] = newFreeRegion; - newFreeRegion->sortedIndex = insertionIndex; + allocation->usedSpace += size; + + allocation->usedRegions[allocation->usedRegionCount] = memoryUsedRegion; + allocation->usedRegionCount += 1; + + SDL_UnlockMutex(renderer->allocatorLock); + + return memoryUsedRegion; } -static uint8_t VULKAN_INTERNAL_FindMemoryType( +static void VULKAN_INTERNAL_RemoveMemoryUsedRegion( VulkanRenderer *renderer, - uint32_t typeFilter, - VkMemoryPropertyFlags requiredProperties, - VkMemoryPropertyFlags ignoredProperties, - uint32_t *memoryTypeIndex + VulkanMemoryUsedRegion *usedRegion ) { uint32_t i; - for (i = *memoryTypeIndex; i < renderer->memoryProperties.memoryTypeCount; i += 1) + SDL_LockMutex(renderer->allocatorLock); + + for (i = 0; i < usedRegion->allocation->usedRegionCount; i += 1) { - if ( (typeFilter & (1 << i)) && - (renderer->memoryProperties.memoryTypes[i].propertyFlags & requiredProperties) == requiredProperties && - (renderer->memoryProperties.memoryTypes[i].propertyFlags & ignoredProperties) == 0 ) + if (usedRegion->allocation->usedRegions[i] == usedRegion) { - *memoryTypeIndex = i; - return 1; + /* plug the hole */ + if (i != usedRegion->allocation->usedRegionCount - 1) + { + usedRegion->allocation->usedRegions[i] = usedRegion->allocation->usedRegions[usedRegion->allocation->usedRegionCount - 1]; + } + + break; } } - Refresh_LogError("Failed to find memory properties %X, required %X, ignored %X", typeFilter, requiredProperties, ignoredProperties); - return 0; + usedRegion->allocation->usedSpace -= usedRegion->size; + + usedRegion->allocation->usedRegionCount -= 1; + + VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, + usedRegion->allocation, + usedRegion->offset, + usedRegion->size + ); + + if (!usedRegion->allocation->dedicated) + { + renderer->needDefrag = 1; + } + + SDL_free(usedRegion); + + renderer->resourceFreed = 1; + SDL_UnlockMutex(renderer->allocatorLock); } static uint8_t VULKAN_INTERNAL_FindBufferMemoryRequirements( VulkanRenderer *renderer, VkBuffer buffer, + VkMemoryPropertyFlags requiredMemoryProperties, + VkMemoryPropertyFlags ignoredMemoryProperties, VkMemoryRequirements2KHR *pMemoryRequirements, uint32_t *pMemoryTypeIndex ) { @@ -2216,20 +2355,13 @@ static uint8_t VULKAN_INTERNAL_FindBufferMemoryRequirements( pMemoryRequirements ); - if (!VULKAN_INTERNAL_FindMemoryType( + return VULKAN_INTERNAL_FindMemoryType( renderer, pMemoryRequirements->memoryRequirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, - 0, + requiredMemoryProperties, + ignoredMemoryProperties, pMemoryTypeIndex - )) { - Refresh_LogError( - "Could not find valid memory type for buffer creation" - ); - return 0; - } - - return 1; + ); } static uint8_t VULKAN_INTERNAL_FindImageMemoryRequirements( @@ -2252,20 +2384,60 @@ static uint8_t VULKAN_INTERNAL_FindImageMemoryRequirements( pMemoryRequirements ); - if (!VULKAN_INTERNAL_FindMemoryType( + return VULKAN_INTERNAL_FindMemoryType( renderer, pMemoryRequirements->memoryRequirements.memoryTypeBits, requiredMemoryPropertyFlags, ignoredMemoryPropertyFlags, pMemoryTypeIndex - )) { - Refresh_LogError( - "Could not find valid memory type for image creation" + ); +} + +static void VULKAN_INTERNAL_DeallocateMemory( + VulkanRenderer *renderer, + VulkanMemorySubAllocator *allocator, + uint32_t allocationIndex +) { + uint32_t i; + uint8_t isDeviceLocal = + (renderer->memoryProperties.memoryTypes[allocator->memoryTypeIndex].propertyFlags & + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0; + + VulkanMemoryAllocation *allocation = allocator->allocations[allocationIndex]; + + SDL_LockMutex(renderer->allocatorLock); + + for (i = 0; i < allocation->freeRegionCount; i += 1) + { + VULKAN_INTERNAL_RemoveMemoryFreeRegion( + renderer, + allocation->freeRegions[i] ); - return 0; + } + SDL_free(allocation->freeRegions); + + /* no need to iterate used regions because deallocate + * only happens when there are 0 used regions + */ + SDL_free(allocation->usedRegions); + + renderer->vkFreeMemory( + renderer->logicalDevice, + allocation->memory, + NULL + ); + + SDL_DestroyMutex(allocation->memoryLock); + SDL_free(allocation); + + if (allocationIndex != allocator->allocationCount - 1) + { + allocator->allocations[allocationIndex] = allocator->allocations[allocator->allocationCount - 1]; } - return 1; + allocator->allocationCount -= 1; + + SDL_UnlockMutex(renderer->allocatorLock); } static uint8_t VULKAN_INTERNAL_AllocateMemory( @@ -2290,8 +2462,20 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( allocation = SDL_malloc(sizeof(VulkanMemoryAllocation)); allocation->size = allocationSize; + allocation->freeSpace = 0; /* added by FreeRegions */ + allocation->usedSpace = 0; /* added by UsedRegions */ allocation->memoryLock = SDL_CreateMutex(); + allocator->allocationCount += 1; + allocator->allocations = SDL_realloc( + allocator->allocations, + sizeof(VulkanMemoryAllocation*) * allocator->allocationCount + ); + + allocator->allocations[ + allocator->allocationCount - 1 + ] = allocation; + if (dedicated) { dedicatedInfo.sType = @@ -2301,29 +2485,24 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( dedicatedInfo.image = image; allocInfo.pNext = &dedicatedInfo; - allocation->dedicated = 1; + allocation->availableForAllocation = 0; } else { allocInfo.pNext = NULL; - - allocator->allocationCount += 1; - allocator->allocations = SDL_realloc( - allocator->allocations, - sizeof(VulkanMemoryAllocation*) * allocator->allocationCount - ); - - allocator->allocations[ - allocator->allocationCount - 1 - ] = allocation; - allocation->dedicated = 0; + allocation->availableForAllocation = 1; } + allocation->usedRegions = SDL_malloc(sizeof(VulkanMemoryUsedRegion*)); + allocation->usedRegionCount = 0; + allocation->usedRegionCapacity = 1; + allocation->freeRegions = SDL_malloc(sizeof(VulkanMemoryFreeRegion*)); allocation->freeRegionCount = 0; allocation->freeRegionCapacity = 1; + allocation->allocator = allocator; result = renderer->vkAllocateMemory( @@ -2346,7 +2525,7 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( SDL_free(allocation); - LogVulkanResultAsWarn("vkAllocateMemory", result); + FNA3D_LogWarn("vkAllocateMemory: %s", VkErrorMessages(result)); return 0; } @@ -2357,16 +2536,11 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( renderer->logicalDevice, allocation->memory, 0, - allocation->size, + VK_WHOLE_SIZE, 0, (void**) &allocation->mapPointer ); - - if (result != VK_SUCCESS) - { - LogVulkanResultAsError("vkMapMemory", result); - return 0; - } + VULKAN_ERROR_CHECK(result, vkMapMemory, 0) } else { @@ -2374,6 +2548,7 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( } VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, allocation, 0, allocation->size @@ -2383,40 +2558,101 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( return 1; } -static uint8_t VULKAN_INTERNAL_FindAvailableMemory( +static uint8_t VULKAN_INTERNAL_BindBufferMemory( VulkanRenderer *renderer, + VulkanMemoryUsedRegion *usedRegion, + VkDeviceSize alignedOffset, + VkBuffer buffer +) { + VkResult vulkanResult; + + SDL_LockMutex(usedRegion->allocation->memoryLock); + + vulkanResult = renderer->vkBindBufferMemory( + renderer->logicalDevice, + buffer, + usedRegion->allocation->memory, + alignedOffset + ); + + SDL_UnlockMutex(usedRegion->allocation->memoryLock); + + VULKAN_ERROR_CHECK(vulkanResult, vkBindBufferMemory, 0) + + return 1; +} + +static uint8_t VULKAN_INTERNAL_BindImageMemory( + VulkanRenderer *renderer, + VulkanMemoryUsedRegion *usedRegion, + VkDeviceSize alignedOffset, + VkImage image +) { + VkResult vulkanResult; + + SDL_LockMutex(usedRegion->allocation->memoryLock); + + vulkanResult = renderer->vkBindImageMemory( + renderer->logicalDevice, + image, + usedRegion->allocation->memory, + alignedOffset + ); + + SDL_UnlockMutex(usedRegion->allocation->memoryLock); + + VULKAN_ERROR_CHECK(vulkanResult, vkBindBufferMemory, 0) + + return 1; +} + +static uint8_t VULKAN_INTERNAL_BindResourceMemory( + VulkanRenderer* renderer, uint32_t memoryTypeIndex, - VkMemoryRequirements2KHR *memoryRequirements, - VkMemoryDedicatedRequirementsKHR *dedicatedRequirements, + VkMemoryRequirements2KHR* memoryRequirements, + VkMemoryDedicatedRequirementsKHR* dedicatedRequirements, + uint8_t forceDedicated, + VkDeviceSize resourceSize, /* may be different from requirements size! */ VkBuffer buffer, /* may be VK_NULL_HANDLE */ VkImage image, /* may be VK_NULL_HANDLE */ - VulkanMemoryAllocation **pMemoryAllocation, - VkDeviceSize *pOffset, - VkDeviceSize *pSize + VulkanMemoryUsedRegion** pMemoryUsedRegion ) { VulkanMemoryAllocation *allocation; VulkanMemorySubAllocator *allocator; VulkanMemoryFreeRegion *region; + VulkanMemoryUsedRegion *usedRegion; VkDeviceSize requiredSize, allocationSize; VkDeviceSize alignedOffset; uint32_t newRegionSize, newRegionOffset; uint8_t shouldAllocDedicated = + forceDedicated || dedicatedRequirements->prefersDedicatedAllocation || dedicatedRequirements->requiresDedicatedAllocation; - uint8_t isHostVisible, allocationResult; + uint8_t isDeviceLocal, isHostVisible, allocationResult; isHostVisible = (renderer->memoryProperties.memoryTypes[memoryTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0; + isDeviceLocal = + (renderer->memoryProperties.memoryTypes[memoryTypeIndex].propertyFlags & + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0; + allocator = &renderer->memoryAllocator->subAllocators[memoryTypeIndex]; requiredSize = memoryRequirements->memoryRequirements.size; + if ( (buffer == VK_NULL_HANDLE && image == VK_NULL_HANDLE) || + (buffer != VK_NULL_HANDLE && image != VK_NULL_HANDLE) ) + { + FNA3D_LogError("BindResourceMemory must be given either a VulkanBuffer or a VulkanTexture"); + return 0; + } + SDL_LockMutex(renderer->allocatorLock); /* find the largest free region and use it */ - if (allocator->sortedFreeRegionCount > 0) + if (!shouldAllocDedicated && allocator->sortedFreeRegionCount > 0) { region = allocator->sortedFreeRegions[0]; allocation = region->allocation; @@ -2428,31 +2664,29 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( if (alignedOffset + requiredSize <= region->offset + region->size) { - *pMemoryAllocation = allocation; + usedRegion = VULKAN_INTERNAL_NewMemoryUsedRegion( + renderer, + allocation, + region->offset, + requiredSize + (alignedOffset - region->offset), + alignedOffset, + resourceSize, + memoryRequirements->memoryRequirements.alignment + ); - /* not aligned - create a new free region */ - if (region->offset != alignedOffset) - { - VULKAN_INTERNAL_NewMemoryFreeRegion( - allocation, - region->offset, - alignedOffset - region->offset - ); - } - - *pOffset = alignedOffset; - *pSize = requiredSize; + usedRegion->isBuffer = buffer != VK_NULL_HANDLE; newRegionSize = region->size - ((alignedOffset - region->offset) + requiredSize); newRegionOffset = alignedOffset + requiredSize; /* remove and add modified region to re-sort */ - VULKAN_INTERNAL_RemoveMemoryFreeRegion(region); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, region); /* if size is 0, no need to re-insert */ if (newRegionSize != 0) { VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, allocation, newRegionOffset, newRegionSize @@ -2461,6 +2695,40 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( SDL_UnlockMutex(renderer->allocatorLock); + if (buffer != VK_NULL_HANDLE) + { + if (!VULKAN_INTERNAL_BindBufferMemory( + renderer, + usedRegion, + alignedOffset, + buffer + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); + + return 0; + } + } + else if (image != VK_NULL_HANDLE) + { + if (!VULKAN_INTERNAL_BindImageMemory( + renderer, + usedRegion, + alignedOffset, + image + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); + + return 0; + } + } + + *pMemoryUsedRegion = usedRegion; return 1; } } @@ -2473,14 +2741,13 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( } else if (requiredSize > allocator->nextAllocationSize) { - /* allocate a page of required size aligned to STARTING_ALLOCATION_SIZE increments */ + /* allocate a page of required size aligned to ALLOCATION_INCREMENT increments */ allocationSize = - VULKAN_INTERNAL_NextHighestAlignment(requiredSize, STARTING_ALLOCATION_SIZE); + VULKAN_INTERNAL_NextHighestAlignment(requiredSize, ALLOCATION_INCREMENT); } else { allocationSize = allocator->nextAllocationSize; - allocator->nextAllocationSize = SDL_min(allocator->nextAllocationSize * 2, MAX_ALLOCATION_SIZE); } allocationResult = VULKAN_INTERNAL_AllocateMemory( @@ -2500,24 +2767,33 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( SDL_UnlockMutex(renderer->allocatorLock); /* Responsibility of the caller to handle being out of memory */ - Refresh_LogWarn("Failed to allocate memory!"); + FNA3D_LogWarn("Failed to allocate memory!"); return 2; } - *pMemoryAllocation = allocation; - *pOffset = 0; - *pSize = requiredSize; + usedRegion = VULKAN_INTERNAL_NewMemoryUsedRegion( + renderer, + allocation, + 0, + requiredSize, + 0, + resourceSize, + memoryRequirements->memoryRequirements.alignment + ); + + usedRegion->isBuffer = buffer != VK_NULL_HANDLE; region = allocation->freeRegions[0]; newRegionOffset = region->offset + requiredSize; newRegionSize = region->size - requiredSize; - VULKAN_INTERNAL_RemoveMemoryFreeRegion(region); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, region); if (newRegionSize != 0) { VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, allocation, newRegionOffset, newRegionSize @@ -2526,68 +2802,50 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( SDL_UnlockMutex(renderer->allocatorLock); - return 1; -} - -static uint8_t VULKAN_INTERNAL_FindAvailableBufferMemory( - VulkanRenderer *renderer, - VkBuffer buffer, - VulkanMemoryAllocation **pMemoryAllocation, - VkDeviceSize *pOffset, - VkDeviceSize *pSize -) { - uint32_t memoryTypeIndex = 0; - VkMemoryDedicatedRequirementsKHR dedicatedRequirements = + if (buffer != VK_NULL_HANDLE) { - VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR, - NULL - }; - VkMemoryRequirements2KHR memoryRequirements = - { - VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR, - &dedicatedRequirements - }; - uint8_t findResult = 0; - - while (VULKAN_INTERNAL_FindBufferMemoryRequirements( - renderer, - buffer, - &memoryRequirements, - &memoryTypeIndex - )) { - findResult = VULKAN_INTERNAL_FindAvailableMemory( + if (!VULKAN_INTERNAL_BindBufferMemory( renderer, - memoryTypeIndex, - &memoryRequirements, - &dedicatedRequirements, - buffer, - VK_NULL_HANDLE, - pMemoryAllocation, - pOffset, - pSize - ); + usedRegion, + 0, + buffer + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); - if (findResult == 1) - { - break; + return 0; } - else - { - memoryTypeIndex += 1; + } + else if (image != VK_NULL_HANDLE) + { + if (!VULKAN_INTERNAL_BindImageMemory( + renderer, + usedRegion, + 0, + image + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); + + return 0; } } - return findResult; + *pMemoryUsedRegion = usedRegion; + return 1; } -static uint8_t VULKAN_INTERNAL_FindAvailableTextureMemory( - VulkanRenderer *renderer, +static uint8_t VULKAN_INTERNAL_BindMemoryForImage( + VulkanRenderer* renderer, VkImage image, - uint8_t cpuAllocation, - VulkanMemoryAllocation **pMemoryAllocation, - VkDeviceSize *pOffset, - VkDeviceSize *pSize + uint8_t isRenderTarget, + VulkanMemoryUsedRegion** usedRegion ) { + uint8_t bindResult = 0; uint32_t memoryTypeIndex = 0; VkMemoryPropertyFlags requiredMemoryPropertyFlags; VkMemoryPropertyFlags ignoredMemoryPropertyFlags; @@ -2601,18 +2859,10 @@ static uint8_t VULKAN_INTERNAL_FindAvailableTextureMemory( VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR, &dedicatedRequirements }; - uint8_t findResult = 0; - if (cpuAllocation) - { - requiredMemoryPropertyFlags = 0; - ignoredMemoryPropertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - } - else - { - requiredMemoryPropertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - ignoredMemoryPropertyFlags = 0; - } + /* Prefer GPU allocation */ + requiredMemoryPropertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + ignoredMemoryPropertyFlags = 0; while (VULKAN_INTERNAL_FindImageMemoryRequirements( renderer, @@ -2622,29 +2872,599 @@ static uint8_t VULKAN_INTERNAL_FindAvailableTextureMemory( &memoryRequirements, &memoryTypeIndex )) { - findResult = VULKAN_INTERNAL_FindAvailableMemory( + bindResult = VULKAN_INTERNAL_BindResourceMemory( renderer, memoryTypeIndex, &memoryRequirements, &dedicatedRequirements, + isRenderTarget, + memoryRequirements.memoryRequirements.size, VK_NULL_HANDLE, image, - pMemoryAllocation, - pOffset, - pSize + usedRegion ); - if (findResult == 1) + if (bindResult == 1) { break; } - else + else /* Bind failed, try the next device-local heap */ { memoryTypeIndex += 1; } } - return findResult; + /* Bind _still_ failed, try again without device local */ + if (bindResult != 1) + { + memoryTypeIndex = 0; + requiredMemoryPropertyFlags = 0; + ignoredMemoryPropertyFlags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; + + if (isRenderTarget) + { + FNA3D_LogWarn("RenderTarget is allocated in host memory, pre-allocate your targets!"); + } + + FNA3D_LogWarn("Out of device local memory, falling back to host memory"); + + while (VULKAN_INTERNAL_FindImageMemoryRequirements( + renderer, + image, + requiredMemoryPropertyFlags, + ignoredMemoryPropertyFlags, + &memoryRequirements, + &memoryTypeIndex + )) { + bindResult = VULKAN_INTERNAL_BindResourceMemory( + renderer, + memoryTypeIndex, + &memoryRequirements, + &dedicatedRequirements, + isRenderTarget, + memoryRequirements.memoryRequirements.size, + VK_NULL_HANDLE, + image, + usedRegion + ); + + if (bindResult == 1) + { + break; + } + else /* Bind failed, try the next heap */ + { + memoryTypeIndex += 1; + } + } + } + + return bindResult; +} + +static uint8_t VULKAN_INTERNAL_BindMemoryForBuffer( + VulkanRenderer* renderer, + VkBuffer buffer, + VkDeviceSize size, + uint8_t preferDeviceLocal, + uint8_t isTransferBuffer, + VulkanMemoryUsedRegion** usedRegion +) { + uint8_t bindResult = 0; + uint32_t memoryTypeIndex = 0; + VkMemoryPropertyFlags requiredMemoryPropertyFlags; + VkMemoryPropertyFlags ignoredMemoryPropertyFlags; + VkMemoryDedicatedRequirementsKHR dedicatedRequirements = + { + VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR, + NULL + }; + VkMemoryRequirements2KHR memoryRequirements = + { + VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR, + &dedicatedRequirements + }; + + requiredMemoryPropertyFlags = + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + + if (preferDeviceLocal) + { + requiredMemoryPropertyFlags |= + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + } + ignoredMemoryPropertyFlags = 0; + + while (VULKAN_INTERNAL_FindBufferMemoryRequirements( + renderer, + buffer, + requiredMemoryPropertyFlags, + ignoredMemoryPropertyFlags, + &memoryRequirements, + &memoryTypeIndex + )) { + bindResult = VULKAN_INTERNAL_BindResourceMemory( + renderer, + memoryTypeIndex, + &memoryRequirements, + &dedicatedRequirements, + isTransferBuffer, + size, + buffer, + VK_NULL_HANDLE, + usedRegion + ); + + if (bindResult == 1) + { + break; + } + else /* Bind failed, try the next device-local heap */ + { + memoryTypeIndex += 1; + } + } + + /* Bind failed, try again if originally preferred device local */ + if (bindResult != 1 && preferDeviceLocal) + { + memoryTypeIndex = 0; + requiredMemoryPropertyFlags = + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + + /* Follow-up for the warning logged by FindMemoryType */ + if (!renderer->unifiedMemoryWarning) + { + FNA3D_LogWarn("No unified memory found, falling back to host memory"); + renderer->unifiedMemoryWarning = 1; + } + + while (VULKAN_INTERNAL_FindBufferMemoryRequirements( + renderer, + buffer, + requiredMemoryPropertyFlags, + ignoredMemoryPropertyFlags, + &memoryRequirements, + &memoryTypeIndex + )) { + bindResult = VULKAN_INTERNAL_BindResourceMemory( + renderer, + memoryTypeIndex, + &memoryRequirements, + &dedicatedRequirements, + isTransferBuffer, + size, + buffer, + VK_NULL_HANDLE, + usedRegion + ); + + if (bindResult == 1) + { + break; + } + else /* Bind failed, try the next heap */ + { + memoryTypeIndex += 1; + } + } + } + + return bindResult; +} + +static uint8_t VULKAN_INTERNAL_FindAllocationToDefragment( + VulkanRenderer *renderer, + VulkanMemorySubAllocator *allocator, + uint32_t *allocationIndexToDefrag +) { + uint32_t i, j; + + for (i = 0; i < VK_MAX_MEMORY_TYPES; i += 1) + { + *allocator = renderer->memoryAllocator->subAllocators[i]; + + for (j = 0; j < allocator->allocationCount; j += 1) + { + if (allocator->allocations[j]->freeRegionCount > 1) + { + *allocationIndexToDefrag = j; + return 1; + } + } + } + + return 0; +} + +static uint8_t VULKAN_INTERNAL_DefragmentMemory( + VulkanRenderer *renderer +) { + VulkanMemorySubAllocator allocator; + VulkanMemoryAllocation *allocation; + uint32_t allocationIndexToDefrag; + VulkanMemoryUsedRegion *currentRegion; + VulkanMemoryUsedRegion *newRegion; + VkBuffer copyBuffer; + VkBufferCopy bufferCopy; + VkImage copyImage; + VkImageCopy *imageCopyRegions; + VkImageAspectFlags aspectFlags; + VulkanCommandBuffer *commandBuffer; + VkCommandBufferBeginInfo beginInfo; + VkPipelineStageFlags waitFlags = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + VkSubmitInfo submitInfo; + VkResult result; + uint32_t i, level; + VulkanResourceAccessType copyResourceAccessType = RESOURCE_ACCESS_NONE; + VulkanResourceAccessType originalResourceAccessType; + + SDL_LockMutex(renderer->commandLock); + SDL_LockMutex(renderer->allocatorLock); + + renderer->needDefrag = 0; + + beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + beginInfo.pNext = NULL; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + beginInfo.pInheritanceInfo = NULL; + + commandBuffer = VULKAN_AcquireCommandBuffer((Refresh_Renderer *) renderer); + + if (VULKAN_INTERNAL_FindAllocationToDefragment( + renderer, + &allocator, + &allocationIndexToDefrag + )) { + allocation = allocator.allocations[allocationIndexToDefrag]; + + VULKAN_INTERNAL_MakeMemoryUnavailable( + renderer, + allocation + ); + + for (i = 0; i < allocation->usedRegionCount; i += 1) + { + currentRegion = allocation->usedRegions[i]; + copyResourceAccessType = RESOURCE_ACCESS_NONE; + + if (currentRegion->isBuffer) + { + currentRegion->vulkanBuffer->bufferCreateInfo.usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + copyBuffer = VULKAN_INTERNAL_CreateBuffer( + renderer, + currentRegion->vulkanBuffer->bufferCreateInfo.size, + RESOURCE_ACCESS_NONE, + currentRegion->vulkanBuffer->bufferCreateInfo.usage | VK_BUFFER_USAGE_TRANSFER_DST_BIT + ); + + result = renderer->vkCreateBuffer( + renderer->logicalDevice, + ¤tRegion->vulkanBuffer->bufferCreateInfo, + NULL, + ©Buffer + ); + VULKAN_ERROR_CHECK(result, vkCreateBuffer, 0) + + if ( + VULKAN_INTERNAL_BindMemoryForBuffer( + renderer, + copyBuffer, + currentRegion->resourceSize, + currentRegion->vulkanBuffer->preferDeviceLocal, + 0, + &newRegion + ) != 1) + { + /* Out of memory, abort */ + renderer->vkDestroyBuffer( + renderer->logicalDevice, + copyBuffer, + NULL + ); + break; + } + + originalResourceAccessType = currentRegion->vulkanBuffer->resourceAccessType; + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + RESOURCE_ACCESS_TRANSFER_READ, + currentRegion->vulkanBuffer->buffer, + ¤tRegion->vulkanBuffer->resourceAccessType + ); + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + RESOURCE_ACCESS_TRANSFER_WRITE, + copyBuffer, + ©ResourceAccessType + ); + + bufferCopy.srcOffset = 0; + bufferCopy.dstOffset = 0; + bufferCopy.size = currentRegion->resourceSize; + + renderer->vkCmdCopyBuffer( + commandBuffer->commandBuffer, + currentRegion->vulkanBuffer->buffer, + copyBuffer, + 1, + &bufferCopy + ); + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + originalResourceAccessType, + copyBuffer, + ©ResourceAccessType + ); + + if (renderer->defragmentedBuffersToDestroyCount >= renderer->defragmentedBuffersToDestroyCapacity) + { + renderer->defragmentedBuffersToDestroyCapacity *= 2; + renderer->defragmentedBuffersToDestroy = SDL_realloc( + renderer->defragmentedBuffersToDestroy, + sizeof(VkBuffer) * renderer->defragmentedBuffersToDestroyCapacity + ); + } + + if (renderer->usedRegionsToDestroyCount >= renderer->usedRegionsToDestroyCapacity) + { + renderer->usedRegionsToDestroyCapacity *= 2; + renderer->usedRegionsToDestroy = SDL_realloc( + renderer->usedRegionsToDestroy, + sizeof(VulkanMemoryUsedRegion*) * renderer->usedRegionsToDestroyCapacity + ); + } + + renderer->defragmentedBuffersToDestroy[ + renderer->defragmentedBuffersToDestroyCount + ] = currentRegion->vulkanBuffer->buffer; + + renderer->defragmentedBuffersToDestroyCount += 1; + + renderer->usedRegionsToDestroy[ + renderer->usedRegionsToDestroyCount + ] = currentRegion; + + renderer->usedRegionsToDestroyCount += 1; + + newRegion->isBuffer = 1; + newRegion->vulkanBuffer = currentRegion->vulkanBuffer; + newRegion->vulkanBuffer->usedRegion = newRegion; /* lol */ + newRegion->vulkanBuffer->buffer = copyBuffer; + newRegion->vulkanBuffer->resourceAccessType = copyResourceAccessType; + + renderer->needDefrag = 1; + } + else + { + result = renderer->vkCreateImage( + renderer->logicalDevice, + ¤tRegion->vulkanTexture->imageCreateInfo, + NULL, + ©Image + ); + + VULKAN_ERROR_CHECK(result, vkCreateImage, 0) + + if (VULKAN_INTERNAL_BindMemoryForImage( + renderer, + copyImage, + 0, + &newRegion + ) != 1) + { + /* Out of memory, abort */ + renderer->vkDestroyImage( + renderer->logicalDevice, + copyImage, + NULL + ); + + break; + } + + if (IsDepthFormat(currentRegion->vulkanTexture->surfaceFormat)) + { + aspectFlags = VK_IMAGE_ASPECT_DEPTH_BIT; + + if (DepthFormatContainsStencil(currentRegion->vulkanTexture->surfaceFormat)) + { + aspectFlags |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + } + else + { + aspectFlags = VK_IMAGE_ASPECT_COLOR_BIT; + } + + originalResourceAccessType = currentRegion->vulkanTexture->resourceAccessType; + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_READ, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + currentRegion->vulkanTexture->image, + ¤tRegion->vulkanTexture->resourceAccessType + ); + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_WRITE, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + copyImage, + ©ResourceAccessType + ); + + imageCopyRegions = SDL_stack_alloc(VkImageCopy, currentRegion->vulkanTexture->levelCount); + + for (level = 0; level < currentRegion->vulkanTexture->levelCount; level += 1) + { + imageCopyRegions[level].srcOffset.x = 0; + imageCopyRegions[level].srcOffset.y = 0; + imageCopyRegions[level].srcOffset.z = 0; + imageCopyRegions[level].srcSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].srcSubresource.baseArrayLayer = 0; + imageCopyRegions[level].srcSubresource.layerCount = currentRegion->vulkanTexture->layerCount; + imageCopyRegions[level].srcSubresource.mipLevel = level; + imageCopyRegions[level].extent.width = SDL_max(1, currentRegion->vulkanTexture->dimensions.width >> level); + imageCopyRegions[level].extent.height = SDL_max(1, currentRegion->vulkanTexture->dimensions.height >> level); + imageCopyRegions[level].extent.depth = currentRegion->vulkanTexture->depth; + imageCopyRegions[level].dstOffset.x = 0; + imageCopyRegions[level].dstOffset.y = 0; + imageCopyRegions[level].dstOffset.z = 0; + imageCopyRegions[level].dstSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].dstSubresource.baseArrayLayer = 0; + imageCopyRegions[level].dstSubresource.layerCount = currentRegion->vulkanTexture->layerCount; + imageCopyRegions[level].dstSubresource.mipLevel = level; + } + + renderer->vkCmdCopyImage( + commandBuffer->commandBuffer, + currentRegion->vulkanTexture->image, + AccessMap[currentRegion->vulkanTexture->resourceAccessType].imageLayout, + copyImage, + AccessMap[copyResourceAccessType].imageLayout, + currentRegion->vulkanTexture->levelCount, + imageCopyRegions + ); + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + originalResourceAccessType, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + copyImage, + ©ResourceAccessType + ); + + SDL_stack_free(imageCopyRegions); + + if (renderer->defragmentedImagesToDestroyCount >= renderer->defragmentedImagesToDestroyCapacity) + { + renderer->defragmentedImagesToDestroyCapacity *= 2; + renderer->defragmentedImagesToDestroy = SDL_realloc( + renderer->defragmentedImagesToDestroy, + sizeof(VkImage) * renderer->defragmentedImagesToDestroyCapacity + ); + } + + if (renderer->defragmentedImageViewsToDestroyCount >= renderer->defragmentedImageViewsToDestroyCapacity) + { + renderer->defragmentedImageViewsToDestroyCapacity *= 2; + renderer->defragmentedImageViewsToDestroy = SDL_realloc( + renderer->defragmentedImageViewsToDestroy, + sizeof(VkImageView) * renderer->defragmentedImageViewsToDestroyCapacity + ); + } + + if (renderer->usedRegionsToDestroyCount >= renderer->usedRegionsToDestroyCapacity) + { + renderer->usedRegionsToDestroyCapacity *= 2; + renderer->usedRegionsToDestroy = SDL_realloc( + renderer->usedRegionsToDestroy, + sizeof(VulkanMemoryUsedRegion*) * renderer->usedRegionsToDestroyCapacity + ); + } + + renderer->defragmentedImagesToDestroy[ + renderer->defragmentedImagesToDestroyCount + ] = currentRegion->vulkanTexture->image; + + renderer->defragmentedImagesToDestroyCount += 1; + + renderer->defragmentedImageViewsToDestroy[ + renderer->defragmentedImageViewsToDestroyCount + ] = currentRegion->vulkanTexture->view; + + renderer->defragmentedImageViewsToDestroyCount += 1; + + renderer->usedRegionsToDestroy[ + renderer->usedRegionsToDestroyCount + ] = currentRegion; + + renderer->usedRegionsToDestroyCount += 1; + + currentRegion->vulkanTexture->viewCreateInfo.image = copyImage; + + renderer->vkCreateImageView( + renderer->logicalDevice, + ¤tRegion->vulkanTexture->viewCreateInfo, + NULL, + ¤tRegion->vulkanTexture->view + ); + + newRegion->isBuffer = 0; + + newRegion->vulkanTexture = currentRegion->vulkanTexture; + newRegion->vulkanTexture->usedRegion = newRegion; /* lol */ + newRegion->vulkanTexture->image = copyImage; + newRegion->vulkanTexture->resourceAccessType = copyResourceAccessType; + + renderer->needDefrag = 1; + } + } + } + + renderer->vkEndCommandBuffer( + renderer->defragCommandBufferContainer->commandBuffer + ); + + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submitInfo.pNext = NULL; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &renderer->defragCommandBufferContainer->commandBuffer; + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = &renderer->defragSemaphore; + submitInfo.pWaitDstStageMask = &waitFlags; + submitInfo.signalSemaphoreCount = 0; + submitInfo.pSignalSemaphores = NULL; + + result = renderer->vkResetFences( + renderer->logicalDevice, + 1, + &renderer->defragCommandBufferContainer->inFlightFence + ); + VULKAN_ERROR_CHECK(result, vkResetFences, 0) + + result = renderer->vkQueueSubmit( + renderer->unifiedQueue, + 1, + &submitInfo, + renderer->defragCommandBufferContainer->inFlightFence + ); + VULKAN_ERROR_CHECK(result, vkQueueSubmit, 0) + + renderer->currentCommandBufferContainer = NULL; + renderer->numActiveCommands = 0; + + renderer->defragTimer = 0; + + SDL_UnlockMutex(renderer->commandLock); + SDL_UnlockMutex(renderer->allocatorLock); + + return 1; } /* Memory Barriers */ @@ -3023,31 +3843,6 @@ static void VULKAN_INTERNAL_DestroyTexture( VulkanRenderer* renderer, VulkanTexture* texture ) { - if (texture->allocation->dedicated) - { - renderer->vkFreeMemory( - renderer->logicalDevice, - texture->allocation->memory, - NULL - ); - - SDL_DestroyMutex(texture->allocation->memoryLock); - SDL_free(texture->allocation->freeRegions); - SDL_free(texture->allocation); - } - else - { - SDL_LockMutex(renderer->allocatorLock); - - VULKAN_INTERNAL_NewMemoryFreeRegion( - texture->allocation, - texture->offset, - texture->memorySize - ); - - SDL_UnlockMutex(renderer->allocatorLock); - } - VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( renderer, texture @@ -3065,6 +3860,11 @@ static void VULKAN_INTERNAL_DestroyTexture( NULL ); + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + texture->usedRegion + ); + /* destroy the msaa texture, if there is one */ if (texture->msaaTex != NULL) { @@ -3099,37 +3899,17 @@ static void VULKAN_INTERNAL_DestroyBuffer( VulkanRenderer* renderer, VulkanBuffer* buffer ) { - if (buffer->allocation->dedicated) - { - renderer->vkFreeMemory( - renderer->logicalDevice, - buffer->allocation->memory, - NULL - ); - - SDL_DestroyMutex(buffer->allocation->memoryLock); - SDL_free(buffer->allocation->freeRegions); - SDL_free(buffer->allocation); - } - else - { - SDL_LockMutex(renderer->allocatorLock); - - VULKAN_INTERNAL_NewMemoryFreeRegion( - buffer->allocation, - buffer->offset, - buffer->memorySize - ); - - SDL_UnlockMutex(renderer->allocatorLock); - } - renderer->vkDestroyBuffer( renderer->logicalDevice, buffer->buffer, NULL ); + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + buffer->usedRegion + ); + SDL_free(buffer); } @@ -3667,7 +4447,7 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( VulkanBuffer* buffer; VkResult vulkanResult; VkBufferCreateInfo bufferCreateInfo; - uint8_t findMemoryResult; + uint8_t bindResult; buffer = SDL_malloc(sizeof(VulkanBuffer)); @@ -3690,51 +4470,30 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( NULL, &buffer->buffer ); + VULKAN_ERROR_CHECK(vulkanResult, vkCreateBuffer, 0) - if (vulkanResult != VK_SUCCESS) - { - SDL_free(buffer); - LogVulkanResultAsError("vkCreateBuffer", vulkanResult); - Refresh_LogError("Failed to create VkBuffer"); - return NULL; - } + buffer->bufferCreateInfo = bufferCreateInfo; - findMemoryResult = VULKAN_INTERNAL_FindAvailableBufferMemory( + bindResult = VULKAN_INTERNAL_BindMemoryForBuffer( renderer, buffer->buffer, - &buffer->allocation, - &buffer->offset, - &buffer->memorySize + buffer->size, + buffer->preferDeviceLocal, + buffer->isTransferBuffer, + &buffer->usedRegion ); - /* We're out of available memory */ - if (findMemoryResult == 2) + if (bindResult != 1) { - Refresh_LogWarn("Out of buffer memory!"); - return NULL; - } - else if (findMemoryResult == 0) - { - Refresh_LogError("Failed to find buffer memory!"); + renderer->vkDestroyBuffer( + renderer->logicalDevice, + buffer->buffer, + NULL); + return NULL; } - SDL_LockMutex(buffer->allocation->memoryLock); - - vulkanResult = renderer->vkBindBufferMemory( - renderer->logicalDevice, - buffer->buffer, - buffer->allocation->memory, - buffer->offset - ); - - SDL_UnlockMutex(buffer->allocation->memoryLock); - - if (vulkanResult != VK_SUCCESS) - { - Refresh_LogError("Failed to bind buffer memory!"); - return NULL; - } + buffer->usedRegion->vulkanBuffer = buffer; /* lol */ buffer->resourceAccessType = resourceAccessType; @@ -4726,15 +5485,6 @@ static void VULKAN_DestroyDevice( SDL_free(renderer->dummyFragmentUniformBuffer); SDL_free(renderer->dummyComputeUniformBuffer); - for (i = 0; i < renderer->transferBufferPool.availableBufferCount; i += 1) - { - VULKAN_INTERNAL_DestroyBuffer(renderer, renderer->transferBufferPool.availableBuffers[i]->buffer); - SDL_free(renderer->transferBufferPool.availableBuffers[i]); - } - - SDL_free(renderer->transferBufferPool.availableBuffers); - SDL_DestroyMutex(renderer->transferBufferPool.lock); - for (i = 0; i < NUM_COMMAND_POOL_BUCKETS; i += 1) { commandPoolHashArray = renderer->commandPoolHashTable.buckets[i]; @@ -5223,7 +5973,7 @@ static VulkanTexture* VULKAN_INTERNAL_CreateTexture( VkImageCreateInfo imageCreateInfo; VkImageCreateFlags imageCreateFlags = 0; VkImageViewCreateInfo imageViewCreateInfo; - uint8_t findMemoryResult; + uint8_t bindResult; uint8_t is3D = depth > 1 ? 1 : 0; uint8_t layerCount = isCube ? 6 : 1; uint8_t isRenderTarget = @@ -5271,73 +6021,29 @@ static VulkanTexture* VULKAN_INTERNAL_CreateTexture( NULL, &texture->image ); + VULKAN_ERROR_CHECK(vulkanResult, vkCreateImage, 0) - if (vulkanResult != VK_SUCCESS) - { - LogVulkanResultAsError("vkCreateImage", vulkanResult); - Refresh_LogError("Failed to create texture!"); - } + texture->isRenderTarget = isRenderTarget; + texture->imageCreateInfo = imageCreateInfo; - /* Prefer GPU allocation */ - findMemoryResult = VULKAN_INTERNAL_FindAvailableTextureMemory( + bindResult = VULKAN_INTERNAL_BindMemoryForImage( renderer, texture->image, - 0, - &texture->allocation, - &texture->offset, - &texture->memorySize + isRenderTarget, + &texture->usedRegion ); - /* No device local memory available */ - if (findMemoryResult == 2) + if (bindResult != 1) { - if (isRenderTarget) - { - Refresh_LogWarn("RenderTarget is allocated in host memory, pre-allocate your targets!"); - } - - Refresh_LogWarn("Out of device local memory, falling back to host memory"); - - /* Attempt CPU allocation */ - findMemoryResult = VULKAN_INTERNAL_FindAvailableTextureMemory( - renderer, + renderer->vkDestroyImage( + renderer->logicalDevice, texture->image, - 1, - &texture->allocation, - &texture->offset, - &texture->memorySize - ); + NULL); - /* Memory alloc completely failed, time to die */ - if (findMemoryResult == 0) - { - Refresh_LogError("Something went very wrong allocating memory!"); - return 0; - } - else if (findMemoryResult == 2) - { - Refresh_LogError("Out of memory!"); - return 0; - } + return bindResult; } - SDL_LockMutex(texture->allocation->memoryLock); - - vulkanResult = renderer->vkBindImageMemory( - renderer->logicalDevice, - texture->image, - texture->allocation->memory, - texture->offset - ); - - SDL_UnlockMutex(texture->allocation->memoryLock); - - if (vulkanResult != VK_SUCCESS) - { - LogVulkanResultAsError("vkBindImageMemory", vulkanResult); - Refresh_LogError("Failed to bind texture memory!"); - return 0; - } + texture->usedRegion->vulkanTexture = texture; /* lol */ imageViewCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; imageViewCreateInfo.pNext = NULL; @@ -6715,41 +7421,7 @@ static VulkanTransferBuffer* VULKAN_INTERNAL_AcquireTransferBuffer( } } - /* Nothing fits, so let's get a transfer buffer from the pool */ - - SDL_LockMutex(renderer->transferBufferPool.lock); - - for (i = 0; i < renderer->transferBufferPool.availableBufferCount; i += 1) - { - transferBuffer = renderer->transferBufferPool.availableBuffers[i]; - offset = transferBuffer->offset + alignment - (transferBuffer->offset % alignment); - - if (offset + requiredSize <= transferBuffer->buffer->size) - { - if (commandBuffer->transferBufferCount == commandBuffer->transferBufferCapacity) - { - commandBuffer->transferBufferCapacity *= 2; - commandBuffer->transferBuffers = SDL_realloc( - commandBuffer->transferBuffers, - commandBuffer->transferBufferCapacity * sizeof(VulkanTransferBuffer*) - ); - } - - commandBuffer->transferBuffers[commandBuffer->transferBufferCount] = transferBuffer; - commandBuffer->transferBufferCount += 1; - - renderer->transferBufferPool.availableBuffers[i] = renderer->transferBufferPool.availableBuffers[renderer->transferBufferPool.availableBufferCount - 1]; - renderer->transferBufferPool.availableBufferCount -= 1; - SDL_UnlockMutex(renderer->transferBufferPool.lock); - - transferBuffer->offset = offset; - return transferBuffer; - } - } - - SDL_UnlockMutex(renderer->transferBufferPool.lock); - - /* Nothing fits still, so let's create a new transfer buffer */ + /* Nothing fits, so let's create a new transfer buffer */ size = TRANSFER_BUFFER_STARTING_SIZE; @@ -9663,26 +10335,13 @@ static void VULKAN_INTERNAL_CleanCommandBuffer( commandBuffer->boundUniformBufferCount = 0; - SDL_LockMutex(renderer->transferBufferPool.lock); - - if (renderer->transferBufferPool.availableBufferCount + commandBuffer->transferBufferCount >= renderer->transferBufferPool.availableBufferCapacity) - { - renderer->transferBufferPool.availableBufferCapacity = renderer->transferBufferPool.availableBufferCount + commandBuffer->transferBufferCount; - renderer->transferBufferPool.availableBuffers = SDL_realloc( - renderer->transferBufferPool.availableBuffers, - renderer->transferBufferPool.availableBufferCapacity * sizeof(VulkanTransferBuffer*) - ); - } - for (i = 0; i < commandBuffer->transferBufferCount; i += 1) { - commandBuffer->transferBuffers[i]->offset = 0; - renderer->transferBufferPool.availableBuffers[renderer->transferBufferPool.availableBufferCount] = commandBuffer->transferBuffers[i]; - renderer->transferBufferPool.availableBufferCount += 1; + VULKAN_INTERNAL_DestroyBuffer(renderer, commandBuffer->transferBuffers[i]->buffer); + SDL_free(commandBuffer->transferBuffers[i]); + commandBuffer->transferBuffers[i] = NULL; } - SDL_UnlockMutex(renderer->transferBufferPool.lock); - commandBuffer->transferBufferCount = 0; /* Bound descriptor sets are now available */ @@ -10350,6 +11009,9 @@ static uint8_t VULKAN_INTERNAL_IsDeviceSuitable( static void VULKAN_INTERNAL_GetPhysicalDeviceProperties( VulkanRenderer *renderer ) { + VkDeviceSize deviceLocalHeapSize; + int32_t i; + renderer->physicalDeviceDriverProperties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES_KHR; renderer->physicalDeviceDriverProperties.pNext = NULL; @@ -10368,6 +11030,21 @@ static void VULKAN_INTERNAL_GetPhysicalDeviceProperties( renderer->physicalDevice, &renderer->memoryProperties ); + + deviceLocalHeapSize = 0; + for (i = 0; i < renderer->memoryProperties.memoryHeapCount; i += 1) + { + if ( renderer->memoryProperties.memoryHeaps[i].flags & + VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ) + { + if (renderer->memoryProperties.memoryHeaps[i].size > deviceLocalHeapSize) + { + deviceLocalHeapSize = renderer->memoryProperties.memoryHeaps[i].size; + } + } + } + + renderer->maxDeviceLocalHeapUsage = deviceLocalHeapSize; } static uint8_t VULKAN_INTERNAL_DeterminePhysicalDevice( @@ -10744,6 +11421,7 @@ static Refresh_Device* VULKAN_CreateDevice( /* Variables: Image Format Detection */ VkImageFormatProperties imageFormatProperties; + SDL_memset(renderer, '\0', sizeof(VulkanRenderer)); renderer->debugMode = debugMode; if (!VULKAN_INTERNAL_PrepareVulkan(renderer)) @@ -10828,6 +11506,7 @@ static Refresh_Device* VULKAN_CreateDevice( for (i = 0; i < VK_MAX_MEMORY_TYPES; i += 1) { + renderer->memoryAllocator->subAllocators[i].memoryTypeIndex = i; renderer->memoryAllocator->subAllocators[i].nextAllocationSize = STARTING_ALLOCATION_SIZE; renderer->memoryAllocator->subAllocators[i].allocations = NULL; renderer->memoryAllocator->subAllocators[i].allocationCount = 0; @@ -11124,14 +11803,6 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->renderTargetHashArray.count = 0; renderer->renderTargetHashArray.capacity = 0; - /* Initialize transfer buffer pool */ - - renderer->transferBufferPool.lock = SDL_CreateMutex(); - - renderer->transferBufferPool.availableBufferCapacity = 4; - renderer->transferBufferPool.availableBufferCount = 0; - renderer->transferBufferPool.availableBuffers = SDL_malloc(renderer->transferBufferPool.availableBufferCapacity * sizeof(VulkanTransferBuffer*)); - /* Some drivers don't support D16, so we have to fall back to D32. */ vulkanResult = renderer->vkGetPhysicalDeviceImageFormatProperties( @@ -11229,6 +11900,10 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->framebuffersToDestroyCapacity ); + renderer->needDefrag = 0; + renderer->defragTimer = 0; + renderer->resourceFreed = 0; + return result; } -- 2.25.1 From 19ebe7284d54168418d81e7039b9bdc9fee98df8 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Mon, 15 May 2023 11:34:57 -0700 Subject: [PATCH 02/12] implement buffer defrag --- src/Refresh_Driver_Vulkan.c | 204 +++++++++++++++++------------------- 1 file changed, 98 insertions(+), 106 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index 95913ff..cade7fa 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -689,7 +689,12 @@ static const VulkanResourceAccessInfo AccessMap[RESOURCE_ACCESS_TYPES_COUNT] = /* Memory structures */ -struct VulkanBuffer /* cast from Refresh_Buffer */ +typedef struct VulkanBufferContainer /* cast from Refresh_Buffer */ +{ + VulkanBuffer *vulkanBuffer; +} VulkanBufferContainer; + +struct VulkanBuffer { VkBuffer buffer; VkDeviceSize size; @@ -698,6 +703,8 @@ struct VulkanBuffer /* cast from Refresh_Buffer */ VkBufferUsageFlags usage; SDL_atomic_t referenceCount; /* Tracks command buffer usage */ + + VulkanBufferContainer *container; }; typedef struct VulkanUniformBufferPool VulkanUniformBufferPool; @@ -1752,6 +1759,10 @@ typedef struct VulkanRenderer uint32_t buffersToDestroyCount; uint32_t buffersToDestroyCapacity; + VulkanBufferContainer **bufferContainersToDestroy; + uint32_t bufferContainersToDestroyCount; + uint32_t bufferContainersToDestroyCapacity; + VulkanSampler **samplersToDestroy; uint32_t samplersToDestroyCount; uint32_t samplersToDestroyCapacity; @@ -3086,8 +3097,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( VulkanMemoryAllocation *allocation; uint32_t allocationIndexToDefrag; VulkanMemoryUsedRegion *currentRegion; - VulkanMemoryUsedRegion *newRegion; - VkBuffer copyBuffer; + VulkanBuffer* newBuffer; VkBufferCopy bufferCopy; VkImage copyImage; VkImageCopy *imageCopyRegions; @@ -3101,7 +3111,6 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( VulkanResourceAccessType copyResourceAccessType = RESOURCE_ACCESS_NONE; VulkanResourceAccessType originalResourceAccessType; - SDL_LockMutex(renderer->commandLock); SDL_LockMutex(renderer->allocatorLock); renderer->needDefrag = 0; @@ -3125,6 +3134,10 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( allocation ); + /* For each used region in the allocation + * create a new resource, copy the data + * and re-point the resource containers + */ for (i = 0; i < allocation->usedRegionCount; i += 1) { currentRegion = allocation->usedRegions[i]; @@ -3132,40 +3145,20 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( if (currentRegion->isBuffer) { - currentRegion->vulkanBuffer->bufferCreateInfo.usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; + currentRegion->vulkanBuffer->usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; - copyBuffer = VULKAN_INTERNAL_CreateBuffer( + // TODO: modify this stuff to change the buffer pointers around + newBuffer = VULKAN_INTERNAL_CreateBuffer( renderer, - currentRegion->vulkanBuffer->bufferCreateInfo.size, + currentRegion->vulkanBuffer->size, RESOURCE_ACCESS_NONE, - currentRegion->vulkanBuffer->bufferCreateInfo.usage | VK_BUFFER_USAGE_TRANSFER_DST_BIT + currentRegion->vulkanBuffer->usage ); - result = renderer->vkCreateBuffer( - renderer->logicalDevice, - ¤tRegion->vulkanBuffer->bufferCreateInfo, - NULL, - ©Buffer - ); - VULKAN_ERROR_CHECK(result, vkCreateBuffer, 0) - - if ( - VULKAN_INTERNAL_BindMemoryForBuffer( - renderer, - copyBuffer, - currentRegion->resourceSize, - currentRegion->vulkanBuffer->preferDeviceLocal, - 0, - &newRegion - ) != 1) + if (newBuffer == NULL) { - /* Out of memory, abort */ - renderer->vkDestroyBuffer( - renderer->logicalDevice, - copyBuffer, - NULL - ); - break; + Refresh_LogError("Failed to create defrag buffer!"); + return 0; } originalResourceAccessType = currentRegion->vulkanBuffer->resourceAccessType; @@ -3180,7 +3173,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( VULKAN_INTERNAL_BufferMemoryBarrier( renderer, RESOURCE_ACCESS_TRANSFER_WRITE, - copyBuffer, + newBuffer, ©ResourceAccessType ); @@ -3191,7 +3184,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( renderer->vkCmdCopyBuffer( commandBuffer->commandBuffer, currentRegion->vulkanBuffer->buffer, - copyBuffer, + newBuffer->buffer, 1, &bufferCopy ); @@ -3199,45 +3192,18 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( VULKAN_INTERNAL_BufferMemoryBarrier( renderer, originalResourceAccessType, - copyBuffer, + newBuffer, ©ResourceAccessType ); - if (renderer->defragmentedBuffersToDestroyCount >= renderer->defragmentedBuffersToDestroyCapacity) - { - renderer->defragmentedBuffersToDestroyCapacity *= 2; - renderer->defragmentedBuffersToDestroy = SDL_realloc( - renderer->defragmentedBuffersToDestroy, - sizeof(VkBuffer) * renderer->defragmentedBuffersToDestroyCapacity - ); - } + SDL_AtomicIncRef(&newBuffer->referenceCount); + SDL_AtomicIncRef(¤tRegion->vulkanBuffer->referenceCount); - if (renderer->usedRegionsToDestroyCount >= renderer->usedRegionsToDestroyCapacity) - { - renderer->usedRegionsToDestroyCapacity *= 2; - renderer->usedRegionsToDestroy = SDL_realloc( - renderer->usedRegionsToDestroy, - sizeof(VulkanMemoryUsedRegion*) * renderer->usedRegionsToDestroyCapacity - ); - } + /* re-point original container to new buffer */ + newBuffer->container = currentRegion->vulkanBuffer->container; + newBuffer->container->vulkanBuffer = newBuffer; - renderer->defragmentedBuffersToDestroy[ - renderer->defragmentedBuffersToDestroyCount - ] = currentRegion->vulkanBuffer->buffer; - - renderer->defragmentedBuffersToDestroyCount += 1; - - renderer->usedRegionsToDestroy[ - renderer->usedRegionsToDestroyCount - ] = currentRegion; - - renderer->usedRegionsToDestroyCount += 1; - - newRegion->isBuffer = 1; - newRegion->vulkanBuffer = currentRegion->vulkanBuffer; - newRegion->vulkanBuffer->usedRegion = newRegion; /* lol */ - newRegion->vulkanBuffer->buffer = copyBuffer; - newRegion->vulkanBuffer->resourceAccessType = copyResourceAccessType; + VULKAN_INTERNAL_QueueDestroyBuffer(renderer, currentRegion->vulkanBuffer); renderer->needDefrag = 1; } @@ -3461,7 +3427,6 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( renderer->defragTimer = 0; - SDL_UnlockMutex(renderer->commandLock); SDL_UnlockMutex(renderer->allocatorLock); return 1; @@ -5825,7 +5790,7 @@ static void VULKAN_DrawPrimitivesIndirect( ) { VulkanRenderer* renderer = (VulkanRenderer*) driverData; VulkanCommandBuffer *vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer; - VulkanBuffer *vulkanBuffer = (VulkanBuffer*) buffer; + VulkanBuffer *vulkanBuffer = ((VulkanBufferContainer*) buffer)->vulkanBuffer; VkDescriptorSet descriptorSets[4]; uint32_t dynamicOffsets[2]; @@ -7344,6 +7309,7 @@ static Refresh_Buffer* VULKAN_CreateBuffer( Refresh_BufferUsageFlags usageFlags, uint32_t sizeInBytes ) { + VulkanBufferContainer* bufferContainer; VulkanBuffer* buffer; VulkanResourceAccessType resourceAccessType; VkBufferUsageFlags vulkanUsageFlags = @@ -7391,6 +7357,11 @@ static Refresh_Buffer* VULKAN_CreateBuffer( return NULL; } + bufferContainer = SDL_malloc(sizeof(VulkanBufferContainer*)); + bufferContainer->vulkanBuffer = buffer; + + buffer->container = bufferContainer; + return (Refresh_Buffer*) buffer; } @@ -7964,7 +7935,7 @@ static void VULKAN_SetBufferData( ) { VulkanRenderer* renderer = (VulkanRenderer*) driverData; VulkanCommandBuffer* vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer; - VulkanBuffer* vulkanBuffer = (VulkanBuffer*) buffer; + VulkanBuffer* vulkanBuffer = ((VulkanBufferContainer*) buffer)->vulkanBuffer; VulkanTransferBuffer* transferBuffer; uint8_t* transferBufferPointer; VkBufferCopy bufferCopy; @@ -7983,8 +7954,8 @@ static void VULKAN_SetBufferData( } transferBufferPointer = - transferBuffer->buffer->allocation->mapPointer + - transferBuffer->buffer->offset + + transferBuffer->buffer->usedRegion->allocation->mapPointer + + transferBuffer->buffer->usedRegion->resourceOffset + transferBuffer->offset; SDL_memcpy( @@ -8392,13 +8363,13 @@ static void VULKAN_GetBufferData( void *data, uint32_t dataLengthInBytes ) { - VulkanBuffer* vulkanBuffer = (VulkanBuffer*) buffer; + VulkanBuffer* vulkanBuffer = ((VulkanBufferContainer*) buffer)->vulkanBuffer; uint8_t *dataPtr = (uint8_t*) data; uint8_t *mapPointer; mapPointer = - vulkanBuffer->allocation->mapPointer + - vulkanBuffer->offset; + vulkanBuffer->usedRegion->allocation->mapPointer + + vulkanBuffer->usedRegion->resourceOffset; SDL_memcpy( dataPtr, @@ -8416,7 +8387,7 @@ static void VULKAN_CopyTextureToBuffer( VulkanRenderer *renderer = (VulkanRenderer*) driverData; VulkanCommandBuffer *vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer; VulkanTexture *vulkanTexture = (VulkanTexture*) textureSlice->texture; - VulkanBuffer* vulkanBuffer = (VulkanBuffer*) buffer; + VulkanBuffer* vulkanBuffer = ((VulkanBufferContainer*) buffer)->vulkanBuffer; VulkanResourceAccessType prevResourceAccess; VkBufferImageCopy imageCopy; @@ -8529,13 +8500,10 @@ static void VULKAN_QueueDestroySampler( SDL_UnlockMutex(renderer->disposeLock); } -static void VULKAN_QueueDestroyBuffer( - Refresh_Renderer *driverData, - Refresh_Buffer *buffer +static void VULKAN_INTERNAL_QueueDestroyBuffer( + VulkanRenderer *renderer, + VulkanBuffer *vulkanBuffer ) { - VulkanRenderer *renderer = (VulkanRenderer*) driverData; - VulkanBuffer *vulkanBuffer = (VulkanBuffer*) buffer; - SDL_LockMutex(renderer->disposeLock); EXPAND_ARRAY_IF_NEEDED( @@ -8554,6 +8522,34 @@ static void VULKAN_QueueDestroyBuffer( SDL_UnlockMutex(renderer->disposeLock); } +static void VULKAN_QueueDestroyBuffer( + Refresh_Renderer *driverData, + Refresh_Buffer *buffer +) { + VulkanRenderer *renderer = (VulkanRenderer*) driverData; + VulkanBufferContainer *vulkanBufferContainer = (VulkanBufferContainer*) buffer; + VulkanBuffer *vulkanBuffer = vulkanBufferContainer->vulkanBuffer; + + SDL_LockMutex(renderer->disposeLock); + + VULKAN_INTERNAL_QueueDestroyBuffer(renderer, vulkanBuffer); + + EXPAND_ARRAY_IF_NEEDED( + renderer->bufferContainersToDestroy, + VulkanBufferContainer *, + renderer->bufferContainersToDestroyCount + 1, + renderer->bufferContainersToDestroyCapacity, + renderer->bufferContainersToDestroyCapacity * 2 + ) + + renderer->bufferContainersToDestroy[ + renderer->bufferContainersToDestroyCount + ] = vulkanBufferContainer; + renderer->bufferContainersToDestroyCount += 1; + + SDL_UnlockMutex(renderer->disposeLock); +} + static void VULKAN_QueueDestroyShaderModule( Refresh_Renderer *driverData, Refresh_ShaderModule *shaderModule @@ -9392,7 +9388,7 @@ static void VULKAN_BindVertexBuffers( for (i = 0; i < bindingCount; i += 1) { - currentVulkanBuffer = (VulkanBuffer*) pBuffers[i]; + currentVulkanBuffer = ((VulkanBufferContainer*) pBuffers[i])->vulkanBuffer; buffers[i] = currentVulkanBuffer->buffer; VULKAN_INTERNAL_TrackBuffer(renderer, vulkanCommandBuffer, currentVulkanBuffer); } @@ -9417,7 +9413,7 @@ static void VULKAN_BindIndexBuffer( ) { VulkanRenderer* renderer = (VulkanRenderer*) driverData; VulkanCommandBuffer *vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer; - VulkanBuffer* vulkanBuffer = (VulkanBuffer*) buffer; + VulkanBuffer* vulkanBuffer = ((VulkanBufferContainer*) buffer)->vulkanBuffer; VULKAN_INTERNAL_TrackBuffer(renderer, vulkanCommandBuffer, vulkanBuffer); @@ -9501,7 +9497,7 @@ static void VULKAN_BindComputeBuffers( for (i = 0; i < computePipeline->pipelineLayout->bufferDescriptorSetCache->bindingCount; i += 1) { - currentVulkanBuffer = (VulkanBuffer*) pBuffers[i]; + currentVulkanBuffer = ((VulkanBufferContainer*) pBuffers[i])->vulkanBuffer; descriptorBufferInfos[i].buffer = currentVulkanBuffer->buffer; descriptorBufferInfos[i].offset = 0; @@ -10217,14 +10213,20 @@ static void VULKAN_INTERNAL_PerformPendingDestroys( } } + /* containers are not vulkan resources, so no need for ref counting */ + for (i = renderer->bufferContainersToDestroyCount - 1; i >= 0; i -= 1) + { + SDL_free(renderer->bufferContainersToDestroy[i]); + renderer->bufferContainersToDestroyCount -= 1; + } + for (i = renderer->buffersToDestroyCount - 1; i >= 0; i -= 1) { if (SDL_AtomicGet(&renderer->buffersToDestroy[i]->referenceCount) == 0) { VULKAN_INTERNAL_DestroyBuffer( renderer, - renderer->buffersToDestroy[i] - ); + renderer->buffersToDestroy[i]); renderer->buffersToDestroy[i] = renderer->buffersToDestroy[renderer->buffersToDestroyCount - 1]; renderer->buffersToDestroyCount -= 1; @@ -11009,9 +11011,6 @@ static uint8_t VULKAN_INTERNAL_IsDeviceSuitable( static void VULKAN_INTERNAL_GetPhysicalDeviceProperties( VulkanRenderer *renderer ) { - VkDeviceSize deviceLocalHeapSize; - int32_t i; - renderer->physicalDeviceDriverProperties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES_KHR; renderer->physicalDeviceDriverProperties.pNext = NULL; @@ -11030,21 +11029,6 @@ static void VULKAN_INTERNAL_GetPhysicalDeviceProperties( renderer->physicalDevice, &renderer->memoryProperties ); - - deviceLocalHeapSize = 0; - for (i = 0; i < renderer->memoryProperties.memoryHeapCount; i += 1) - { - if ( renderer->memoryProperties.memoryHeaps[i].flags & - VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ) - { - if (renderer->memoryProperties.memoryHeaps[i].size > deviceLocalHeapSize) - { - deviceLocalHeapSize = renderer->memoryProperties.memoryHeaps[i].size; - } - } - } - - renderer->maxDeviceLocalHeapUsage = deviceLocalHeapSize; } static uint8_t VULKAN_INTERNAL_DeterminePhysicalDevice( @@ -11861,6 +11845,14 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->buffersToDestroyCapacity ); + renderer->bufferContainersToDestroyCapacity = 16; + renderer->bufferContainersToDestroyCount = 0; + + renderer->bufferContainersToDestroy = SDL_malloc( + sizeof(VulkanBufferContainer *) * + renderer->bufferContainersToDestroyCapacity + ); + renderer->samplersToDestroyCapacity = 16; renderer->samplersToDestroyCount = 0; -- 2.25.1 From 74112396ec529bcbeb5e25ee39d3d45bd351fd40 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Mon, 15 May 2023 16:39:02 -0700 Subject: [PATCH 03/12] implement texture container and defrag --- src/Refresh_Driver_Vulkan.c | 823 +++++++++++++--------------- src/Refresh_Driver_Vulkan_vkfuncs.h | 1 + 2 files changed, 383 insertions(+), 441 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index cade7fa..f92b2a2 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -702,6 +702,9 @@ struct VulkanBuffer VulkanResourceAccessType resourceAccessType; VkBufferUsageFlags usage; + uint8_t preferDeviceLocal; + uint8_t isTransferBuffer; + SDL_atomic_t referenceCount; /* Tracks command buffer usage */ VulkanBufferContainer *container; @@ -772,6 +775,11 @@ typedef struct VulkanShaderModule SDL_atomic_t referenceCount; } VulkanShaderModule; +typedef struct VulkanTextureContainer /* Cast from Refresh_Texture */ +{ + VulkanTexture *vulkanTexture; +} VulkanTextureContainer; + struct VulkanTexture { VulkanMemoryUsedRegion *usedRegion; @@ -796,6 +804,8 @@ struct VulkanTexture struct VulkanTexture *msaaTex; SDL_atomic_t referenceCount; + + VulkanTextureContainer *container; }; typedef struct VulkanRenderTarget @@ -1759,10 +1769,6 @@ typedef struct VulkanRenderer uint32_t buffersToDestroyCount; uint32_t buffersToDestroyCapacity; - VulkanBufferContainer **bufferContainersToDestroy; - uint32_t bufferContainersToDestroyCount; - uint32_t bufferContainersToDestroyCapacity; - VulkanSampler **samplersToDestroy; uint32_t samplersToDestroyCount; uint32_t samplersToDestroyCapacity; @@ -1804,6 +1810,7 @@ typedef struct VulkanRenderer /* Forward declarations */ +static uint8_t VULKAN_INTERNAL_DefragmentMemory(VulkanRenderer *renderer); static void VULKAN_INTERNAL_BeginCommandBuffer(VulkanRenderer *renderer, VulkanCommandBuffer *commandBuffer); static void VULKAN_UnclaimWindow(Refresh_Renderer *driverData, void *windowHandle); static void VULKAN_Wait(Refresh_Renderer *driverData); @@ -1869,7 +1876,7 @@ static inline void LogVulkanResultAsWarn( #define VULKAN_ERROR_CHECK(res, fn, ret) \ if (res != VK_SUCCESS) \ { \ - FNA3D_LogError("%s %s", #fn, VkErrorMessages(res)); \ + Refresh_LogError("%s %s", #fn, VkErrorMessages(res)); \ return ret; \ } @@ -2346,6 +2353,35 @@ static void VULKAN_INTERNAL_RemoveMemoryUsedRegion( SDL_UnlockMutex(renderer->allocatorLock); } +static uint8_t VULKAN_INTERNAL_FindMemoryType( + VulkanRenderer *renderer, + uint32_t typeFilter, + VkMemoryPropertyFlags requiredProperties, + VkMemoryPropertyFlags ignoredProperties, + uint32_t *memoryTypeIndex +) { + uint32_t i; + + for (i = *memoryTypeIndex; i < renderer->memoryProperties.memoryTypeCount; i += 1) + { + if ( (typeFilter & (1 << i)) && + (renderer->memoryProperties.memoryTypes[i].propertyFlags & requiredProperties) == requiredProperties && + (renderer->memoryProperties.memoryTypes[i].propertyFlags & ignoredProperties) == 0 ) + { + *memoryTypeIndex = i; + return 1; + } + } + + Refresh_LogWarn( + "Failed to find memory type %X, required %X, ignored %X", + typeFilter, + requiredProperties, + ignoredProperties + ); + return 0; +} + static uint8_t VULKAN_INTERNAL_FindBufferMemoryRequirements( VulkanRenderer *renderer, VkBuffer buffer, @@ -2536,7 +2572,7 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( SDL_free(allocation); - FNA3D_LogWarn("vkAllocateMemory: %s", VkErrorMessages(result)); + Refresh_LogWarn("vkAllocateMemory: %s", VkErrorMessages(result)); return 0; } @@ -2656,7 +2692,7 @@ static uint8_t VULKAN_INTERNAL_BindResourceMemory( if ( (buffer == VK_NULL_HANDLE && image == VK_NULL_HANDLE) || (buffer != VK_NULL_HANDLE && image != VK_NULL_HANDLE) ) { - FNA3D_LogError("BindResourceMemory must be given either a VulkanBuffer or a VulkanTexture"); + Refresh_LogError("BindResourceMemory must be given either a VulkanBuffer or a VulkanTexture"); return 0; } @@ -2778,7 +2814,7 @@ static uint8_t VULKAN_INTERNAL_BindResourceMemory( SDL_UnlockMutex(renderer->allocatorLock); /* Responsibility of the caller to handle being out of memory */ - FNA3D_LogWarn("Failed to allocate memory!"); + Refresh_LogWarn("Failed to allocate memory!"); return 2; } @@ -2914,10 +2950,10 @@ static uint8_t VULKAN_INTERNAL_BindMemoryForImage( if (isRenderTarget) { - FNA3D_LogWarn("RenderTarget is allocated in host memory, pre-allocate your targets!"); + Refresh_LogWarn("RenderTarget is allocated in host memory, pre-allocate your targets!"); } - FNA3D_LogWarn("Out of device local memory, falling back to host memory"); + Refresh_LogWarn("Out of device local memory, falling back to host memory"); while (VULKAN_INTERNAL_FindImageMemoryRequirements( renderer, @@ -3028,7 +3064,7 @@ static uint8_t VULKAN_INTERNAL_BindMemoryForBuffer( /* Follow-up for the warning logged by FindMemoryType */ if (!renderer->unifiedMemoryWarning) { - FNA3D_LogWarn("No unified memory found, falling back to host memory"); + Refresh_LogWarn("No unified memory found, falling back to host memory"); renderer->unifiedMemoryWarning = 1; } @@ -3090,348 +3126,6 @@ static uint8_t VULKAN_INTERNAL_FindAllocationToDefragment( return 0; } -static uint8_t VULKAN_INTERNAL_DefragmentMemory( - VulkanRenderer *renderer -) { - VulkanMemorySubAllocator allocator; - VulkanMemoryAllocation *allocation; - uint32_t allocationIndexToDefrag; - VulkanMemoryUsedRegion *currentRegion; - VulkanBuffer* newBuffer; - VkBufferCopy bufferCopy; - VkImage copyImage; - VkImageCopy *imageCopyRegions; - VkImageAspectFlags aspectFlags; - VulkanCommandBuffer *commandBuffer; - VkCommandBufferBeginInfo beginInfo; - VkPipelineStageFlags waitFlags = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; - VkSubmitInfo submitInfo; - VkResult result; - uint32_t i, level; - VulkanResourceAccessType copyResourceAccessType = RESOURCE_ACCESS_NONE; - VulkanResourceAccessType originalResourceAccessType; - - SDL_LockMutex(renderer->allocatorLock); - - renderer->needDefrag = 0; - - beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; - beginInfo.pNext = NULL; - beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; - beginInfo.pInheritanceInfo = NULL; - - commandBuffer = VULKAN_AcquireCommandBuffer((Refresh_Renderer *) renderer); - - if (VULKAN_INTERNAL_FindAllocationToDefragment( - renderer, - &allocator, - &allocationIndexToDefrag - )) { - allocation = allocator.allocations[allocationIndexToDefrag]; - - VULKAN_INTERNAL_MakeMemoryUnavailable( - renderer, - allocation - ); - - /* For each used region in the allocation - * create a new resource, copy the data - * and re-point the resource containers - */ - for (i = 0; i < allocation->usedRegionCount; i += 1) - { - currentRegion = allocation->usedRegions[i]; - copyResourceAccessType = RESOURCE_ACCESS_NONE; - - if (currentRegion->isBuffer) - { - currentRegion->vulkanBuffer->usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; - - // TODO: modify this stuff to change the buffer pointers around - newBuffer = VULKAN_INTERNAL_CreateBuffer( - renderer, - currentRegion->vulkanBuffer->size, - RESOURCE_ACCESS_NONE, - currentRegion->vulkanBuffer->usage - ); - - if (newBuffer == NULL) - { - Refresh_LogError("Failed to create defrag buffer!"); - return 0; - } - - originalResourceAccessType = currentRegion->vulkanBuffer->resourceAccessType; - - VULKAN_INTERNAL_BufferMemoryBarrier( - renderer, - RESOURCE_ACCESS_TRANSFER_READ, - currentRegion->vulkanBuffer->buffer, - ¤tRegion->vulkanBuffer->resourceAccessType - ); - - VULKAN_INTERNAL_BufferMemoryBarrier( - renderer, - RESOURCE_ACCESS_TRANSFER_WRITE, - newBuffer, - ©ResourceAccessType - ); - - bufferCopy.srcOffset = 0; - bufferCopy.dstOffset = 0; - bufferCopy.size = currentRegion->resourceSize; - - renderer->vkCmdCopyBuffer( - commandBuffer->commandBuffer, - currentRegion->vulkanBuffer->buffer, - newBuffer->buffer, - 1, - &bufferCopy - ); - - VULKAN_INTERNAL_BufferMemoryBarrier( - renderer, - originalResourceAccessType, - newBuffer, - ©ResourceAccessType - ); - - SDL_AtomicIncRef(&newBuffer->referenceCount); - SDL_AtomicIncRef(¤tRegion->vulkanBuffer->referenceCount); - - /* re-point original container to new buffer */ - newBuffer->container = currentRegion->vulkanBuffer->container; - newBuffer->container->vulkanBuffer = newBuffer; - - VULKAN_INTERNAL_QueueDestroyBuffer(renderer, currentRegion->vulkanBuffer); - - renderer->needDefrag = 1; - } - else - { - result = renderer->vkCreateImage( - renderer->logicalDevice, - ¤tRegion->vulkanTexture->imageCreateInfo, - NULL, - ©Image - ); - - VULKAN_ERROR_CHECK(result, vkCreateImage, 0) - - if (VULKAN_INTERNAL_BindMemoryForImage( - renderer, - copyImage, - 0, - &newRegion - ) != 1) - { - /* Out of memory, abort */ - renderer->vkDestroyImage( - renderer->logicalDevice, - copyImage, - NULL - ); - - break; - } - - if (IsDepthFormat(currentRegion->vulkanTexture->surfaceFormat)) - { - aspectFlags = VK_IMAGE_ASPECT_DEPTH_BIT; - - if (DepthFormatContainsStencil(currentRegion->vulkanTexture->surfaceFormat)) - { - aspectFlags |= VK_IMAGE_ASPECT_STENCIL_BIT; - } - } - else - { - aspectFlags = VK_IMAGE_ASPECT_COLOR_BIT; - } - - originalResourceAccessType = currentRegion->vulkanTexture->resourceAccessType; - - VULKAN_INTERNAL_ImageMemoryBarrier( - renderer, - commandBuffer->commandBuffer, - RESOURCE_ACCESS_TRANSFER_READ, - aspectFlags, - 0, - currentRegion->vulkanTexture->layerCount, - 0, - currentRegion->vulkanTexture->levelCount, - 0, - currentRegion->vulkanTexture->image, - ¤tRegion->vulkanTexture->resourceAccessType - ); - - VULKAN_INTERNAL_ImageMemoryBarrier( - renderer, - commandBuffer->commandBuffer, - RESOURCE_ACCESS_TRANSFER_WRITE, - aspectFlags, - 0, - currentRegion->vulkanTexture->layerCount, - 0, - currentRegion->vulkanTexture->levelCount, - 0, - copyImage, - ©ResourceAccessType - ); - - imageCopyRegions = SDL_stack_alloc(VkImageCopy, currentRegion->vulkanTexture->levelCount); - - for (level = 0; level < currentRegion->vulkanTexture->levelCount; level += 1) - { - imageCopyRegions[level].srcOffset.x = 0; - imageCopyRegions[level].srcOffset.y = 0; - imageCopyRegions[level].srcOffset.z = 0; - imageCopyRegions[level].srcSubresource.aspectMask = aspectFlags; - imageCopyRegions[level].srcSubresource.baseArrayLayer = 0; - imageCopyRegions[level].srcSubresource.layerCount = currentRegion->vulkanTexture->layerCount; - imageCopyRegions[level].srcSubresource.mipLevel = level; - imageCopyRegions[level].extent.width = SDL_max(1, currentRegion->vulkanTexture->dimensions.width >> level); - imageCopyRegions[level].extent.height = SDL_max(1, currentRegion->vulkanTexture->dimensions.height >> level); - imageCopyRegions[level].extent.depth = currentRegion->vulkanTexture->depth; - imageCopyRegions[level].dstOffset.x = 0; - imageCopyRegions[level].dstOffset.y = 0; - imageCopyRegions[level].dstOffset.z = 0; - imageCopyRegions[level].dstSubresource.aspectMask = aspectFlags; - imageCopyRegions[level].dstSubresource.baseArrayLayer = 0; - imageCopyRegions[level].dstSubresource.layerCount = currentRegion->vulkanTexture->layerCount; - imageCopyRegions[level].dstSubresource.mipLevel = level; - } - - renderer->vkCmdCopyImage( - commandBuffer->commandBuffer, - currentRegion->vulkanTexture->image, - AccessMap[currentRegion->vulkanTexture->resourceAccessType].imageLayout, - copyImage, - AccessMap[copyResourceAccessType].imageLayout, - currentRegion->vulkanTexture->levelCount, - imageCopyRegions - ); - - VULKAN_INTERNAL_ImageMemoryBarrier( - renderer, - originalResourceAccessType, - aspectFlags, - 0, - currentRegion->vulkanTexture->layerCount, - 0, - currentRegion->vulkanTexture->levelCount, - 0, - copyImage, - ©ResourceAccessType - ); - - SDL_stack_free(imageCopyRegions); - - if (renderer->defragmentedImagesToDestroyCount >= renderer->defragmentedImagesToDestroyCapacity) - { - renderer->defragmentedImagesToDestroyCapacity *= 2; - renderer->defragmentedImagesToDestroy = SDL_realloc( - renderer->defragmentedImagesToDestroy, - sizeof(VkImage) * renderer->defragmentedImagesToDestroyCapacity - ); - } - - if (renderer->defragmentedImageViewsToDestroyCount >= renderer->defragmentedImageViewsToDestroyCapacity) - { - renderer->defragmentedImageViewsToDestroyCapacity *= 2; - renderer->defragmentedImageViewsToDestroy = SDL_realloc( - renderer->defragmentedImageViewsToDestroy, - sizeof(VkImageView) * renderer->defragmentedImageViewsToDestroyCapacity - ); - } - - if (renderer->usedRegionsToDestroyCount >= renderer->usedRegionsToDestroyCapacity) - { - renderer->usedRegionsToDestroyCapacity *= 2; - renderer->usedRegionsToDestroy = SDL_realloc( - renderer->usedRegionsToDestroy, - sizeof(VulkanMemoryUsedRegion*) * renderer->usedRegionsToDestroyCapacity - ); - } - - renderer->defragmentedImagesToDestroy[ - renderer->defragmentedImagesToDestroyCount - ] = currentRegion->vulkanTexture->image; - - renderer->defragmentedImagesToDestroyCount += 1; - - renderer->defragmentedImageViewsToDestroy[ - renderer->defragmentedImageViewsToDestroyCount - ] = currentRegion->vulkanTexture->view; - - renderer->defragmentedImageViewsToDestroyCount += 1; - - renderer->usedRegionsToDestroy[ - renderer->usedRegionsToDestroyCount - ] = currentRegion; - - renderer->usedRegionsToDestroyCount += 1; - - currentRegion->vulkanTexture->viewCreateInfo.image = copyImage; - - renderer->vkCreateImageView( - renderer->logicalDevice, - ¤tRegion->vulkanTexture->viewCreateInfo, - NULL, - ¤tRegion->vulkanTexture->view - ); - - newRegion->isBuffer = 0; - - newRegion->vulkanTexture = currentRegion->vulkanTexture; - newRegion->vulkanTexture->usedRegion = newRegion; /* lol */ - newRegion->vulkanTexture->image = copyImage; - newRegion->vulkanTexture->resourceAccessType = copyResourceAccessType; - - renderer->needDefrag = 1; - } - } - } - - renderer->vkEndCommandBuffer( - renderer->defragCommandBufferContainer->commandBuffer - ); - - submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; - submitInfo.pNext = NULL; - submitInfo.commandBufferCount = 1; - submitInfo.pCommandBuffers = &renderer->defragCommandBufferContainer->commandBuffer; - submitInfo.waitSemaphoreCount = 1; - submitInfo.pWaitSemaphores = &renderer->defragSemaphore; - submitInfo.pWaitDstStageMask = &waitFlags; - submitInfo.signalSemaphoreCount = 0; - submitInfo.pSignalSemaphores = NULL; - - result = renderer->vkResetFences( - renderer->logicalDevice, - 1, - &renderer->defragCommandBufferContainer->inFlightFence - ); - VULKAN_ERROR_CHECK(result, vkResetFences, 0) - - result = renderer->vkQueueSubmit( - renderer->unifiedQueue, - 1, - &submitInfo, - renderer->defragCommandBufferContainer->inFlightFence - ); - VULKAN_ERROR_CHECK(result, vkQueueSubmit, 0) - - renderer->currentCommandBufferContainer = NULL; - renderer->numActiveCommands = 0; - - renderer->defragTimer = 0; - - SDL_UnlockMutex(renderer->allocatorLock); - - return 1; -} - /* Memory Barriers */ static void VULKAN_INTERNAL_BufferMemoryBarrier( @@ -3782,7 +3476,7 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( { hash = &renderer->renderTargetHashArray.elements[i].key; - if ((VulkanTexture*) hash->texture == texture) + if (hash->texture == texture) { VULKAN_INTERNAL_RemoveFramebuffersContainingView( renderer, @@ -4407,7 +4101,9 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( VulkanRenderer *renderer, VkDeviceSize size, VulkanResourceAccessType resourceAccessType, - VkBufferUsageFlags usage + VkBufferUsageFlags usage, + uint8_t preferDeviceLocal, + uint8_t isTransferBuffer ) { VulkanBuffer* buffer; VkResult vulkanResult; @@ -4419,6 +4115,8 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( buffer->size = size; buffer->resourceAccessType = resourceAccessType; buffer->usage = usage; + buffer->preferDeviceLocal = preferDeviceLocal; + buffer->isTransferBuffer = isTransferBuffer; bufferCreateInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; bufferCreateInfo.pNext = NULL; @@ -4437,8 +4135,6 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( ); VULKAN_ERROR_CHECK(vulkanResult, vkCreateBuffer, 0) - buffer->bufferCreateInfo = bufferCreateInfo; - bindResult = VULKAN_INTERNAL_BindMemoryForBuffer( renderer, buffer->buffer, @@ -4569,7 +4265,9 @@ static uint8_t VULKAN_INTERNAL_CreateUniformBuffer( renderer, UBO_BUFFER_SIZE, resourceAccessType, - VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT + VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + 0, + 0 ); if (buffer->vulkanBuffer == NULL) @@ -4658,7 +4356,9 @@ static VulkanUniformBuffer* VULKAN_INTERNAL_CreateDummyUniformBuffer( renderer, UBO_BUFFER_SIZE, resourceAccessType, - VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT + VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT, + 0, + 0 ); buffer->offset = 0; @@ -5313,9 +5013,7 @@ static uint8_t VULKAN_INTERNAL_CreateSwapchain( swapchainData->textures[i].resourceAccessType = RESOURCE_ACCESS_NONE; /* Swapchain memory is managed by the driver */ - swapchainData->textures[i].allocation = NULL; - swapchainData->textures[i].offset = 0; - swapchainData->textures[i].memorySize = 0; + swapchainData->textures[i].usedRegion = NULL; swapchainData->textures[i].dimensions = swapchainData->extent; swapchainData->textures[i].format = swapchainData->swapchainFormat; @@ -5988,9 +5686,6 @@ static VulkanTexture* VULKAN_INTERNAL_CreateTexture( ); VULKAN_ERROR_CHECK(vulkanResult, vkCreateImage, 0) - texture->isRenderTarget = isRenderTarget; - texture->imageCreateInfo = imageCreateInfo; - bindResult = VULKAN_INTERNAL_BindMemoryForImage( renderer, texture->image, @@ -6005,7 +5700,8 @@ static VulkanTexture* VULKAN_INTERNAL_CreateTexture( texture->image, NULL); - return bindResult; + Refresh_LogError("Unable to bind memory for texture!"); + return NULL; } texture->usedRegion->vulkanTexture = texture; /* lol */ @@ -6046,7 +5742,7 @@ static VulkanTexture* VULKAN_INTERNAL_CreateTexture( { LogVulkanResultAsError("vkCreateImageView", vulkanResult); Refresh_LogError("Failed to create texture image view"); - return 0; + return NULL; } texture->dimensions.width = width; @@ -6205,7 +5901,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass( for (i = 0; i < colorAttachmentCount; i += 1) { - texture = (VulkanTexture*) colorAttachmentInfos[i].texture; + texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture; if (texture->msaaTex != NULL) { @@ -6328,7 +6024,7 @@ static VkRenderPass VULKAN_INTERNAL_CreateRenderPass( } else { - texture = (VulkanTexture*) depthStencilAttachmentInfo->texture; + texture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture; attachmentDescriptions[attachmentDescriptionCount].flags = 0; attachmentDescriptions[attachmentDescriptionCount].format = texture->format; @@ -7222,7 +6918,8 @@ static Refresh_Texture* VULKAN_CreateTexture( VkImageAspectFlags imageAspectFlags; uint8_t isDepthFormat = IsRefreshDepthFormat(textureCreateInfo->format); VkFormat format; - VulkanTexture *result; + VulkanTextureContainer *container; + VulkanTexture *vulkanTexture; if (isDepthFormat) { @@ -7267,7 +6964,7 @@ static Refresh_Texture* VULKAN_CreateTexture( imageAspectFlags = VK_IMAGE_ASPECT_COLOR_BIT; } - result = VULKAN_INTERNAL_CreateTexture( + vulkanTexture = VULKAN_INTERNAL_CreateTexture( renderer, textureCreateInfo->width, textureCreateInfo->height, @@ -7283,11 +6980,11 @@ static Refresh_Texture* VULKAN_CreateTexture( ); /* create the MSAA texture for color attachments, if needed */ - if ( result != NULL && + if ( vulkanTexture != NULL && !isDepthFormat && textureCreateInfo->sampleCount > REFRESH_SAMPLECOUNT_1 ) { - result->msaaTex = VULKAN_INTERNAL_CreateTexture( + vulkanTexture->msaaTex = VULKAN_INTERNAL_CreateTexture( renderer, textureCreateInfo->width, textureCreateInfo->height, @@ -7301,7 +6998,11 @@ static Refresh_Texture* VULKAN_CreateTexture( ); } - return (Refresh_Texture*) result; + container = SDL_malloc(sizeof(VulkanTextureContainer)); + container->vulkanTexture = vulkanTexture; + vulkanTexture->container = container; + + return (Refresh_Texture*) container; } static Refresh_Buffer* VULKAN_CreateBuffer( @@ -7348,7 +7049,9 @@ static Refresh_Buffer* VULKAN_CreateBuffer( (VulkanRenderer*)driverData, sizeInBytes, resourceAccessType, - vulkanUsageFlags + vulkanUsageFlags, + 1, + 0 ); if (buffer == NULL) @@ -7357,9 +7060,8 @@ static Refresh_Buffer* VULKAN_CreateBuffer( return NULL; } - bufferContainer = SDL_malloc(sizeof(VulkanBufferContainer*)); + bufferContainer = SDL_malloc(sizeof(VulkanBufferContainer)); bufferContainer->vulkanBuffer = buffer; - buffer->container = bufferContainer; return (Refresh_Buffer*) buffer; @@ -7407,7 +7109,9 @@ static VulkanTransferBuffer* VULKAN_INTERNAL_AcquireTransferBuffer( renderer, size, RESOURCE_ACCESS_TRANSFER_READ_WRITE, - VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + 1, + 1 ); if (transferBuffer == NULL) @@ -7439,7 +7143,7 @@ static void VULKAN_SetTextureData( uint32_t dataLengthInBytes ) { VulkanRenderer *renderer = (VulkanRenderer*) driverData; - VulkanTexture *vulkanTexture = (VulkanTexture*) textureSlice->texture; + VulkanTexture *vulkanTexture = ((VulkanTextureContainer*) textureSlice->texture)->vulkanTexture; VulkanCommandBuffer *vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer; VulkanTransferBuffer *transferBuffer; VkBufferImageCopy imageCopy; @@ -7465,8 +7169,8 @@ static void VULKAN_SetTextureData( } stagingBufferPointer = - transferBuffer->buffer->allocation->mapPointer + - transferBuffer->buffer->offset + + transferBuffer->buffer->usedRegion->allocation->mapPointer + + transferBuffer->buffer->usedRegion->resourceOffset + transferBuffer->offset; SDL_memcpy( @@ -7553,7 +7257,7 @@ static void VULKAN_SetTextureDataYUV( uint32_t dataLength ) { VulkanRenderer *renderer = (VulkanRenderer*) driverData; - VulkanTexture *tex = (VulkanTexture*) y; + VulkanTexture *tex = ((VulkanTextureContainer*) y)->vulkanTexture; VulkanCommandBuffer *vulkanCommandBuffer = (VulkanCommandBuffer*)commandBuffer; VulkanTransferBuffer *transferBuffer; @@ -7576,8 +7280,8 @@ static void VULKAN_SetTextureDataYUV( } stagingBufferPointer = - transferBuffer->buffer->allocation->mapPointer + - transferBuffer->buffer->offset + + transferBuffer->buffer->usedRegion->allocation->mapPointer + + transferBuffer->buffer->usedRegion->resourceOffset + transferBuffer->offset; /* Initialize values that are the same for Y, U, and V */ @@ -7660,7 +7364,7 @@ static void VULKAN_SetTextureDataYUV( imageCopy.bufferOffset = transferBuffer->offset + yDataLength; - tex = (VulkanTexture*) u; + tex = ((VulkanTextureContainer*) u)->vulkanTexture; SDL_memcpy( stagingBufferPointer + yDataLength, @@ -7715,7 +7419,7 @@ static void VULKAN_SetTextureDataYUV( imageCopy.bufferOffset = transferBuffer->offset + yDataLength + uvDataLength; - tex = (VulkanTexture*) v; + tex = ((VulkanTextureContainer*) v)->vulkanTexture; SDL_memcpy( stagingBufferPointer + yDataLength + uvDataLength, @@ -7778,8 +7482,8 @@ static void VULKAN_INTERNAL_BlitImage( VkFilter filter ) { VkImageBlit blit; - VulkanTexture *sourceTexture = (VulkanTexture*) sourceTextureSlice->texture; - VulkanTexture *destinationTexture = (VulkanTexture*) destinationTextureSlice->texture; + VulkanTexture *sourceTexture = ((VulkanTextureContainer*) sourceTextureSlice->texture)->vulkanTexture; + VulkanTexture *destinationTexture = ((VulkanTextureContainer*) destinationTextureSlice->texture)->vulkanTexture; VulkanResourceAccessType originalSourceAccessType = sourceTexture->resourceAccessType; @@ -7919,7 +7623,7 @@ static void VULKAN_INTERNAL_SetBufferData( uint32_t dataLength ) { SDL_memcpy( - vulkanBuffer->allocation->mapPointer + vulkanBuffer->offset + offsetInBytes, + vulkanBuffer->usedRegion->allocation->mapPointer + vulkanBuffer->usedRegion->resourceOffset + offsetInBytes, data, dataLength ); @@ -8295,7 +7999,7 @@ static void VULKAN_BindVertexSamplers( for (i = 0; i < samplerCount; i += 1) { - currentTexture = (VulkanTexture*) pTextures[i]; + currentTexture = ((VulkanTextureContainer*) pTextures[i])->vulkanTexture; currentSampler = (VulkanSampler*) pSamplers[i]; descriptorImageInfos[i].imageView = currentTexture->view; descriptorImageInfos[i].sampler = currentSampler->sampler; @@ -8338,7 +8042,7 @@ static void VULKAN_BindFragmentSamplers( for (i = 0; i < samplerCount; i += 1) { - currentTexture = (VulkanTexture*) pTextures[i]; + currentTexture = ((VulkanTextureContainer*) pTextures[i])->vulkanTexture; currentSampler = (VulkanSampler*) pSamplers[i]; descriptorImageInfos[i].imageView = currentTexture->view; descriptorImageInfos[i].sampler = currentSampler->sampler; @@ -8386,7 +8090,7 @@ static void VULKAN_CopyTextureToBuffer( ) { VulkanRenderer *renderer = (VulkanRenderer*) driverData; VulkanCommandBuffer *vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer; - VulkanTexture *vulkanTexture = (VulkanTexture*) textureSlice->texture; + VulkanTexture *vulkanTexture = ((VulkanTextureContainer*) textureSlice->texture)->vulkanTexture; VulkanBuffer* vulkanBuffer = ((VulkanBufferContainer*) buffer)->vulkanBuffer; VulkanResourceAccessType prevResourceAccess; @@ -8454,13 +8158,10 @@ static void VULKAN_CopyTextureToBuffer( VULKAN_INTERNAL_TrackTexture(renderer, vulkanCommandBuffer, vulkanTexture); } -static void VULKAN_QueueDestroyTexture( - Refresh_Renderer *driverData, - Refresh_Texture *texture +static void VULKAN_INTERNAL_QueueDestroyTexture( + VulkanRenderer *renderer, + VulkanTexture *vulkanTexture ) { - VulkanRenderer *renderer = (VulkanRenderer*) driverData; - VulkanTexture* vulkanTexture = (VulkanTexture*) texture; - SDL_LockMutex(renderer->disposeLock); EXPAND_ARRAY_IF_NEEDED( @@ -8471,12 +8172,32 @@ static void VULKAN_QueueDestroyTexture( renderer->texturesToDestroyCapacity * 2 ) - renderer->texturesToDestroy[renderer->texturesToDestroyCount] = vulkanTexture; + renderer->texturesToDestroy[ + renderer->texturesToDestroyCount + ] = vulkanTexture; renderer->texturesToDestroyCount += 1; SDL_UnlockMutex(renderer->disposeLock); } +static void VULKAN_QueueDestroyTexture( + Refresh_Renderer *driverData, + Refresh_Texture *texture +) { + VulkanRenderer *renderer = (VulkanRenderer*) driverData; + VulkanTextureContainer *vulkanTextureContainer = (VulkanTextureContainer *)texture; + VulkanTexture *vulkanTexture = vulkanTextureContainer->vulkanTexture; + + SDL_LockMutex(renderer->disposeLock); + + VULKAN_INTERNAL_QueueDestroyTexture(renderer, vulkanTexture); + + /* Containers are just client handles, so we can destroy immediately */ + SDL_free(vulkanTextureContainer); + + SDL_UnlockMutex(renderer->disposeLock); +} + static void VULKAN_QueueDestroySampler( Refresh_Renderer *driverData, Refresh_Sampler *sampler @@ -8534,18 +8255,8 @@ static void VULKAN_QueueDestroyBuffer( VULKAN_INTERNAL_QueueDestroyBuffer(renderer, vulkanBuffer); - EXPAND_ARRAY_IF_NEEDED( - renderer->bufferContainersToDestroy, - VulkanBufferContainer *, - renderer->bufferContainersToDestroyCount + 1, - renderer->bufferContainersToDestroyCapacity, - renderer->bufferContainersToDestroyCapacity * 2 - ) - - renderer->bufferContainersToDestroy[ - renderer->bufferContainersToDestroyCount - ] = vulkanBufferContainer; - renderer->bufferContainersToDestroyCount += 1; + /* Containers are just client handles, so we can destroy immediately */ + SDL_free(vulkanBufferContainer); SDL_UnlockMutex(renderer->disposeLock); } @@ -8637,7 +8348,7 @@ static VkRenderPass VULKAN_INTERNAL_FetchRenderPass( for (i = 0; i < colorAttachmentCount; i += 1) { - hash.colorTargetDescriptions[i].format = ((VulkanTexture*) colorAttachmentInfos[i].texture)->format; + hash.colorTargetDescriptions[i].format = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture->format; hash.colorTargetDescriptions[i].clearColor = colorAttachmentInfos[i].clearColor; hash.colorTargetDescriptions[i].loadOp = colorAttachmentInfos[i].loadOp; hash.colorTargetDescriptions[i].storeOp = colorAttachmentInfos[i].storeOp; @@ -8646,7 +8357,7 @@ static VkRenderPass VULKAN_INTERNAL_FetchRenderPass( hash.colorAttachmentSampleCount = REFRESH_SAMPLECOUNT_1; if (colorAttachmentCount > 0) { - texture = (VulkanTexture*) colorAttachmentInfos[0].texture; + texture = ((VulkanTextureContainer*) colorAttachmentInfos[0].texture)->vulkanTexture; if (texture->msaaTex != NULL) { hash.colorAttachmentSampleCount = texture->msaaTex->sampleCount; @@ -8665,7 +8376,7 @@ static VkRenderPass VULKAN_INTERNAL_FetchRenderPass( } else { - hash.depthStencilTargetDescription.format = ((VulkanTexture*) depthStencilAttachmentInfo->texture)->format; + hash.depthStencilTargetDescription.format = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture->format; hash.depthStencilTargetDescription.loadOp = depthStencilAttachmentInfo->loadOp; hash.depthStencilTargetDescription.storeOp = depthStencilAttachmentInfo->storeOp; hash.depthStencilTargetDescription.stencilLoadOp = depthStencilAttachmentInfo->stencilLoadOp; @@ -8735,7 +8446,7 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( for (i = 0; i < colorAttachmentCount; i += 1) { - texture = (VulkanTexture*) colorAttachmentInfos[i].texture; + texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture; renderTarget = VULKAN_INTERNAL_FetchRenderTarget( renderer, @@ -8771,7 +8482,7 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( } else { - texture = (VulkanTexture*) depthStencilAttachmentInfo->texture; + texture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture; renderTarget = VULKAN_INTERNAL_FetchRenderTarget( renderer, texture, @@ -8804,7 +8515,7 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( for (i = 0; i < colorAttachmentCount; i += 1) { - texture = (VulkanTexture*) colorAttachmentInfos[i].texture; + texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture; renderTarget = VULKAN_INTERNAL_FetchRenderTarget( renderer, @@ -8838,7 +8549,7 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( if (depthStencilAttachmentInfo != NULL) { - texture = (VulkanTexture*) depthStencilAttachmentInfo->texture; + texture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture; renderTarget = VULKAN_INTERNAL_FetchRenderTarget( renderer, texture, @@ -8983,7 +8694,7 @@ static void VULKAN_BeginRenderPass( for (i = 0; i < colorAttachmentCount; i += 1) { - texture = (VulkanTexture*) colorAttachmentInfos[i].texture; + texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture; w = texture->dimensions.width >> colorAttachmentInfos[i].level; h = texture->dimensions.height >> colorAttachmentInfos[i].level; @@ -9000,7 +8711,7 @@ static void VULKAN_BeginRenderPass( if (depthStencilAttachmentInfo != NULL) { - texture = (VulkanTexture*) depthStencilAttachmentInfo->texture; + texture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture; w = texture->dimensions.width >> depthStencilAttachmentInfo->level; h = texture->dimensions.height >> depthStencilAttachmentInfo->level; @@ -9041,7 +8752,7 @@ static void VULKAN_BeginRenderPass( for (i = 0; i < colorAttachmentCount; i += 1) { - texture = (VulkanTexture*) colorAttachmentInfos[i].texture; + texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture; VULKAN_INTERNAL_ImageMemoryBarrier( renderer, @@ -9068,7 +8779,7 @@ static void VULKAN_BeginRenderPass( if (depthStencilAttachmentInfo != NULL) { - texture = (VulkanTexture*) depthStencilAttachmentInfo->texture; + texture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture; depthAspectFlags = VK_IMAGE_ASPECT_DEPTH_BIT; if (IsStencilFormat(texture->format)) @@ -9108,7 +8819,7 @@ static void VULKAN_BeginRenderPass( clearValues[i].color.float32[2] = colorAttachmentInfos[i].clearColor.z; clearValues[i].color.float32[3] = colorAttachmentInfos[i].clearColor.w; - texture = (VulkanTexture*) colorAttachmentInfos[i].texture; + texture = ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture; if (texture->msaaTex != NULL) { clearValues[i+1].color.float32[0] = colorAttachmentInfos[i].clearColor.x; @@ -9150,13 +8861,13 @@ static void VULKAN_BeginRenderPass( for (i = 0; i < colorAttachmentCount; i += 1) { vulkanCommandBuffer->renderPassColorTargetTextures[i] = - (VulkanTexture*) colorAttachmentInfos[i].texture; + ((VulkanTextureContainer*) colorAttachmentInfos[i].texture)->vulkanTexture; } vulkanCommandBuffer->renderPassColorTargetCount = colorAttachmentCount; if (depthStencilAttachmentInfo != NULL) { - vulkanCommandBuffer->renderPassDepthTexture = (VulkanTexture*) depthStencilAttachmentInfo->texture; + vulkanCommandBuffer->renderPassDepthTexture = ((VulkanTextureContainer*) depthStencilAttachmentInfo->texture)->vulkanTexture; } /* Set sensible default viewport state */ @@ -9555,7 +9266,7 @@ static void VULKAN_BindComputeTextures( for (i = 0; i < computePipeline->pipelineLayout->imageDescriptorSetCache->bindingCount; i += 1) { - currentTexture = (VulkanTexture*) pTextures[i]; + currentTexture = ((VulkanTextureContainer*) pTextures[i])->vulkanTexture; descriptorImageInfos[i].imageView = currentTexture->view; descriptorImageInfos[i].sampler = VK_NULL_HANDLE; descriptorImageInfos[i].imageLayout = VK_IMAGE_LAYOUT_GENERAL; @@ -10213,13 +9924,6 @@ static void VULKAN_INTERNAL_PerformPendingDestroys( } } - /* containers are not vulkan resources, so no need for ref counting */ - for (i = renderer->bufferContainersToDestroyCount - 1; i >= 0; i -= 1) - { - SDL_free(renderer->bufferContainersToDestroy[i]); - renderer->bufferContainersToDestroyCount -= 1; - } - for (i = renderer->buffersToDestroyCount - 1; i >= 0; i -= 1) { if (SDL_AtomicGet(&renderer->buffersToDestroy[i]->referenceCount) == 0) @@ -10624,6 +10328,251 @@ static void VULKAN_Submit( SDL_UnlockMutex(renderer->submitLock); } +static uint8_t VULKAN_INTERNAL_DefragmentMemory( + VulkanRenderer *renderer +) { + VulkanMemorySubAllocator allocator; + VulkanMemoryAllocation *allocation; + uint32_t allocationIndexToDefrag; + VulkanMemoryUsedRegion *currentRegion; + VulkanBuffer* newBuffer; + VulkanTexture* newTexture; + VkBufferCopy bufferCopy; + VkImageCopy *imageCopyRegions; + VkImageAspectFlags aspectFlags; + VulkanCommandBuffer *commandBuffer; + VkCommandBufferBeginInfo beginInfo; + VkPipelineStageFlags waitFlags = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + uint32_t i, level; + VulkanResourceAccessType copyResourceAccessType = RESOURCE_ACCESS_NONE; + VulkanResourceAccessType originalResourceAccessType; + + SDL_LockMutex(renderer->allocatorLock); + + renderer->needDefrag = 0; + + beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + beginInfo.pNext = NULL; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + beginInfo.pInheritanceInfo = NULL; + + commandBuffer = (VulkanCommandBuffer*) VULKAN_AcquireCommandBuffer((Refresh_Renderer *) renderer); + + if (VULKAN_INTERNAL_FindAllocationToDefragment( + renderer, + &allocator, + &allocationIndexToDefrag + )) { + allocation = allocator.allocations[allocationIndexToDefrag]; + + VULKAN_INTERNAL_MakeMemoryUnavailable( + renderer, + allocation + ); + + /* For each used region in the allocation + * create a new resource, copy the data + * and re-point the resource containers + */ + for (i = 0; i < allocation->usedRegionCount; i += 1) + { + currentRegion = allocation->usedRegions[i]; + copyResourceAccessType = RESOURCE_ACCESS_NONE; + + if (currentRegion->isBuffer) + { + currentRegion->vulkanBuffer->usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + newBuffer = VULKAN_INTERNAL_CreateBuffer( + renderer, + currentRegion->vulkanBuffer->size, + RESOURCE_ACCESS_NONE, + currentRegion->vulkanBuffer->usage, + currentRegion->vulkanBuffer->preferDeviceLocal, + currentRegion->vulkanBuffer->isTransferBuffer + ); + + if (newBuffer == NULL) + { + Refresh_LogError("Failed to create defrag buffer!"); + return 0; + } + + originalResourceAccessType = currentRegion->vulkanBuffer->resourceAccessType; + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_READ, + currentRegion->vulkanBuffer + ); + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_WRITE, + newBuffer + ); + + bufferCopy.srcOffset = 0; + bufferCopy.dstOffset = 0; + bufferCopy.size = currentRegion->resourceSize; + + renderer->vkCmdCopyBuffer( + commandBuffer->commandBuffer, + currentRegion->vulkanBuffer->buffer, + newBuffer->buffer, + 1, + &bufferCopy + ); + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + originalResourceAccessType, + newBuffer + ); + + VULKAN_INTERNAL_TrackBuffer(renderer, commandBuffer, currentRegion->vulkanBuffer); + VULKAN_INTERNAL_TrackBuffer(renderer, commandBuffer, newBuffer); + + /* re-point original container to new buffer */ + newBuffer->container = currentRegion->vulkanBuffer->container; + newBuffer->container->vulkanBuffer = newBuffer; + currentRegion->vulkanBuffer->container = NULL; + + VULKAN_INTERNAL_QueueDestroyBuffer(renderer, currentRegion->vulkanBuffer); + + renderer->needDefrag = 1; + } + else + { + newTexture = VULKAN_INTERNAL_CreateTexture( + renderer, + currentRegion->vulkanTexture->dimensions.width, + currentRegion->vulkanTexture->dimensions.height, + currentRegion->vulkanTexture->depth, + currentRegion->vulkanTexture->isCube, + currentRegion->vulkanTexture->levelCount, + currentRegion->vulkanTexture->sampleCount, + currentRegion->vulkanTexture->format, + currentRegion->vulkanTexture->aspectFlags, + currentRegion->vulkanTexture->usageFlags + ); + + if (newTexture == NULL) + { + Refresh_LogError("Failed to create defrag texture!"); + return 0; + } + + originalResourceAccessType = currentRegion->vulkanTexture->resourceAccessType; + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_READ, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + currentRegion->vulkanTexture->image, + ¤tRegion->vulkanTexture->resourceAccessType + ); + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_WRITE, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + newTexture->image, + ©ResourceAccessType + ); + + imageCopyRegions = SDL_stack_alloc(VkImageCopy, currentRegion->vulkanTexture->levelCount); + + for (level = 0; level < currentRegion->vulkanTexture->levelCount; level += 1) + { + imageCopyRegions[level].srcOffset.x = 0; + imageCopyRegions[level].srcOffset.y = 0; + imageCopyRegions[level].srcOffset.z = 0; + imageCopyRegions[level].srcSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].srcSubresource.baseArrayLayer = 0; + imageCopyRegions[level].srcSubresource.layerCount = currentRegion->vulkanTexture->layerCount; + imageCopyRegions[level].srcSubresource.mipLevel = level; + imageCopyRegions[level].extent.width = SDL_max(1, currentRegion->vulkanTexture->dimensions.width >> level); + imageCopyRegions[level].extent.height = SDL_max(1, currentRegion->vulkanTexture->dimensions.height >> level); + imageCopyRegions[level].extent.depth = currentRegion->vulkanTexture->depth; + imageCopyRegions[level].dstOffset.x = 0; + imageCopyRegions[level].dstOffset.y = 0; + imageCopyRegions[level].dstOffset.z = 0; + imageCopyRegions[level].dstSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].dstSubresource.baseArrayLayer = 0; + imageCopyRegions[level].dstSubresource.layerCount = currentRegion->vulkanTexture->layerCount; + imageCopyRegions[level].dstSubresource.mipLevel = level; + } + + renderer->vkCmdCopyImage( + commandBuffer->commandBuffer, + currentRegion->vulkanTexture->image, + AccessMap[currentRegion->vulkanTexture->resourceAccessType].imageLayout, + newTexture->image, + AccessMap[copyResourceAccessType].imageLayout, + currentRegion->vulkanTexture->levelCount, + imageCopyRegions + ); + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + originalResourceAccessType, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + newTexture->image, + ©ResourceAccessType + ); + + SDL_stack_free(imageCopyRegions); + + VULKAN_INTERNAL_TrackTexture(renderer, commandBuffer, currentRegion->vulkanTexture); + VULKAN_INTERNAL_TrackTexture(renderer, commandBuffer, newTexture); + + /* re-point original container to new texture */ + newTexture->container = currentRegion->vulkanTexture->container; + newTexture->container->vulkanTexture = newTexture; + currentRegion->vulkanTexture->container = NULL; + + VULKAN_INTERNAL_QueueDestroyTexture(renderer, currentRegion->vulkanTexture); + + renderer->needDefrag = 1; + } + } + } + + VULKAN_Submit( + (Refresh_Renderer*) renderer, + 1, + (Refresh_CommandBuffer**) &commandBuffer + ); + + renderer->defragTimer = 0; + + SDL_UnlockMutex(renderer->allocatorLock); + + return 1; +} + /* Device instantiation */ static inline uint8_t VULKAN_INTERNAL_SupportsExtension( @@ -11845,14 +11794,6 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->buffersToDestroyCapacity ); - renderer->bufferContainersToDestroyCapacity = 16; - renderer->bufferContainersToDestroyCount = 0; - - renderer->bufferContainersToDestroy = SDL_malloc( - sizeof(VulkanBufferContainer *) * - renderer->bufferContainersToDestroyCapacity - ); - renderer->samplersToDestroyCapacity = 16; renderer->samplersToDestroyCount = 0; diff --git a/src/Refresh_Driver_Vulkan_vkfuncs.h b/src/Refresh_Driver_Vulkan_vkfuncs.h index 69f06ea..47749ff 100644 --- a/src/Refresh_Driver_Vulkan_vkfuncs.h +++ b/src/Refresh_Driver_Vulkan_vkfuncs.h @@ -89,6 +89,7 @@ VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdClearDepthStencilImage, (VkCommandBuff VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdCopyBuffer, (VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkBuffer dstBuffer, uint32_t regionCount, const VkBufferCopy* pRegions)) VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdCopyBufferToImage, (VkCommandBuffer commandBuffer, VkBuffer srcBuffer, VkImage dstImage, VkImageLayout dstImageLayout, uint32_t regionCount, const VkBufferImageCopy *pRegions)) VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdCopyImageToBuffer, (VkCommandBuffer commandBuffer, VkImage srcImage, VkImageLayout srcImageLayout, VkBuffer dstBuffer, uint32_t regionCount, const VkBufferImageCopy *pRegions)) +VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdCopyImage, (VkCommandBuffer commandBuffer, VkImage srcImage, VkImageLayout srcImageLayout, VkImage dstImage, VkImageLayout dstImageLayout, uint32_t regionCount, const VkImageCopy* pRegions)) VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdDispatch, (VkCommandBuffer commandBuffer, uint32_t groupCountX, uint32_t groupCountY, uint32_t groupCountZ)) VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdDraw, (VkCommandBuffer commandBuffer, uint32_t vertexCount, uint32_t instanceCount, uint32_t firstVertex, uint32_t firstInstance)) VULKAN_DEVICE_FUNCTION(BaseVK, void, vkCmdDrawIndexed, (VkCommandBuffer commandBuffer, uint32_t indexCount, uint32_t instanceCount, uint32_t firstIndex, int32_t vertexOffset, uint32_t firstInstance)) -- 2.25.1 From 2ef3c04e8e503caed308de76a5ae8afaf84436b1 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Mon, 15 May 2023 17:17:27 -0700 Subject: [PATCH 04/12] resource acquire fixes --- src/Refresh_Driver_Vulkan.c | 66 ++++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 31 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index f92b2a2..df1568c 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -833,7 +833,7 @@ typedef struct VulkanSwapchainData /* Swapchain images */ VkExtent2D extent; - VulkanTexture *textures; + VulkanTextureContainer *textureContainers; uint32_t imageCount; /* Synchronization primitives */ @@ -3695,17 +3695,19 @@ static void VULKAN_INTERNAL_DestroySwapchain( { VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( renderer, - &swapchainData->textures[i] + swapchainData->textureContainers[i].vulkanTexture ); renderer->vkDestroyImageView( renderer->logicalDevice, - swapchainData->textures[i].view, + swapchainData->textureContainers[i].vulkanTexture->view, NULL ); + + SDL_free(swapchainData->textureContainers[i].vulkanTexture); } - SDL_free(swapchainData->textures); + SDL_free(swapchainData->textureContainers); renderer->vkDestroySwapchainKHR( renderer->logicalDevice, @@ -4946,11 +4948,11 @@ static uint8_t VULKAN_INTERNAL_CreateSwapchain( NULL ); - swapchainData->textures = SDL_malloc( - sizeof(VulkanTexture) * swapchainData->imageCount + swapchainData->textureContainers = SDL_malloc( + sizeof(VulkanTextureContainer) * swapchainData->imageCount ); - if (!swapchainData->textures) + if (!swapchainData->textureContainers) { SDL_OutOfMemory(); renderer->vkDestroySurfaceKHR( @@ -4985,7 +4987,9 @@ static uint8_t VULKAN_INTERNAL_CreateSwapchain( for (i = 0; i < swapchainData->imageCount; i += 1) { - swapchainData->textures[i].image = swapchainImages[i]; + swapchainData->textureContainers[i].vulkanTexture = SDL_malloc(sizeof(VulkanTexture)); + + swapchainData->textureContainers[i].vulkanTexture->image = swapchainImages[i]; imageViewCreateInfo.image = swapchainImages[i]; @@ -4993,7 +4997,7 @@ static uint8_t VULKAN_INTERNAL_CreateSwapchain( renderer->logicalDevice, &imageViewCreateInfo, NULL, - &swapchainData->textures[i].view + &swapchainData->textureContainers[i].vulkanTexture->view ); if (vulkanResult != VK_SUCCESS) @@ -5004,30 +5008,30 @@ static uint8_t VULKAN_INTERNAL_CreateSwapchain( NULL ); SDL_stack_free(swapchainImages); - SDL_free(swapchainData->textures); + SDL_free(swapchainData->textureContainers); SDL_free(swapchainData); LogVulkanResultAsError("vkCreateImageView", vulkanResult); return 0; } - swapchainData->textures[i].resourceAccessType = RESOURCE_ACCESS_NONE; + swapchainData->textureContainers[i].vulkanTexture->resourceAccessType = RESOURCE_ACCESS_NONE; /* Swapchain memory is managed by the driver */ - swapchainData->textures[i].usedRegion = NULL; + swapchainData->textureContainers[i].vulkanTexture->usedRegion = NULL; - swapchainData->textures[i].dimensions = swapchainData->extent; - swapchainData->textures[i].format = swapchainData->swapchainFormat; - swapchainData->textures[i].is3D = 0; - swapchainData->textures[i].isCube = 0; - swapchainData->textures[i].layerCount = 1; - swapchainData->textures[i].levelCount = 1; - swapchainData->textures[i].sampleCount = REFRESH_SAMPLECOUNT_1; - swapchainData->textures[i].usageFlags = + swapchainData->textureContainers[i].vulkanTexture->dimensions = swapchainData->extent; + swapchainData->textureContainers[i].vulkanTexture->format = swapchainData->swapchainFormat; + swapchainData->textureContainers[i].vulkanTexture->is3D = 0; + swapchainData->textureContainers[i].vulkanTexture->isCube = 0; + swapchainData->textureContainers[i].vulkanTexture->layerCount = 1; + swapchainData->textureContainers[i].vulkanTexture->levelCount = 1; + swapchainData->textureContainers[i].vulkanTexture->sampleCount = REFRESH_SAMPLECOUNT_1; + swapchainData->textureContainers[i].vulkanTexture->usageFlags = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; - swapchainData->textures[i].aspectFlags = VK_IMAGE_ASPECT_COLOR_BIT; - swapchainData->textures[i].resourceAccessType = RESOURCE_ACCESS_NONE; - swapchainData->textures[i].msaaTex = NULL; + swapchainData->textureContainers[i].vulkanTexture->aspectFlags = VK_IMAGE_ASPECT_COLOR_BIT; + swapchainData->textureContainers[i].vulkanTexture->resourceAccessType = RESOURCE_ACCESS_NONE; + swapchainData->textureContainers[i].vulkanTexture->msaaTex = NULL; } SDL_stack_free(swapchainImages); @@ -7064,7 +7068,7 @@ static Refresh_Buffer* VULKAN_CreateBuffer( bufferContainer->vulkanBuffer = buffer; buffer->container = bufferContainer; - return (Refresh_Buffer*) buffer; + return (Refresh_Buffer*) bufferContainer; } /* Setters */ @@ -9721,7 +9725,7 @@ static Refresh_Texture* VULKAN_AcquireSwapchainTexture( WindowData *windowData; VulkanSwapchainData *swapchainData; VkResult acquireResult = VK_SUCCESS; - VulkanTexture *swapchainTexture = NULL; + VulkanTextureContainer *swapchainTextureContainer = NULL; VulkanPresentData *presentData; windowData = VULKAN_INTERNAL_FetchWindowData(windowHandle); @@ -9784,7 +9788,7 @@ static Refresh_Texture* VULKAN_AcquireSwapchainTexture( } } - swapchainTexture = &swapchainData->textures[swapchainImageIndex]; + swapchainTextureContainer = &swapchainData->textureContainers[swapchainImageIndex]; VULKAN_INTERNAL_ImageMemoryBarrier( renderer, @@ -9796,8 +9800,8 @@ static Refresh_Texture* VULKAN_AcquireSwapchainTexture( 0, 1, 0, - swapchainTexture->image, - &swapchainTexture->resourceAccessType + swapchainTextureContainer->vulkanTexture->image, + &swapchainTextureContainer->vulkanTexture->resourceAccessType ); /* Set up present struct */ @@ -9846,7 +9850,7 @@ static Refresh_Texture* VULKAN_AcquireSwapchainTexture( *pWidth = swapchainData->extent.width; *pHeight = swapchainData->extent.height; - return (Refresh_Texture*) swapchainTexture; + return (Refresh_Texture*) swapchainTextureContainer; } static Refresh_TextureFormat VULKAN_GetSwapchainFormat( @@ -10228,8 +10232,8 @@ static void VULKAN_Submit( 0, 1, 0, - currentCommandBuffer->presentDatas[j].windowData->swapchainData->textures[swapchainImageIndex].image, - ¤tCommandBuffer->presentDatas[j].windowData->swapchainData->textures[swapchainImageIndex].resourceAccessType + currentCommandBuffer->presentDatas[j].windowData->swapchainData->textureContainers[swapchainImageIndex].vulkanTexture->image, + ¤tCommandBuffer->presentDatas[j].windowData->swapchainData->textureContainers[swapchainImageIndex].vulkanTexture->resourceAccessType ); } -- 2.25.1 From c3d2b17c4dbe6a07a27564af9e38f1f0d12702e3 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Tue, 16 May 2023 12:18:47 -0700 Subject: [PATCH 05/12] defrag fixes --- src/Refresh_Driver_Vulkan.c | 102 +++++++++++++++++++++++++----------- 1 file changed, 72 insertions(+), 30 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index df1568c..fbc9f23 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -703,7 +703,6 @@ struct VulkanBuffer VkBufferUsageFlags usage; uint8_t preferDeviceLocal; - uint8_t isTransferBuffer; SDL_atomic_t referenceCount; /* Tracks command buffer usage */ @@ -3115,7 +3114,7 @@ static uint8_t VULKAN_INTERNAL_FindAllocationToDefragment( for (j = 0; j < allocator->allocationCount; j += 1) { - if (allocator->allocations[j]->freeRegionCount > 1) + if (allocator->allocations[j]->availableForAllocation == 1 && allocator->allocations[j]->freeRegionCount > 1) { *allocationIndexToDefrag = j; return 1; @@ -4118,7 +4117,6 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( buffer->resourceAccessType = resourceAccessType; buffer->usage = usage; buffer->preferDeviceLocal = preferDeviceLocal; - buffer->isTransferBuffer = isTransferBuffer; bufferCreateInfo.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; bufferCreateInfo.pNext = NULL; @@ -4142,7 +4140,7 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( buffer->buffer, buffer->size, buffer->preferDeviceLocal, - buffer->isTransferBuffer, + isTransferBuffer, &buffer->usedRegion ); @@ -4157,6 +4155,7 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( } buffer->usedRegion->vulkanBuffer = buffer; /* lol */ + buffer->container = NULL; buffer->resourceAccessType = resourceAccessType; @@ -5319,27 +5318,29 @@ static void VULKAN_DestroyDevice( { allocator = &renderer->memoryAllocator->subAllocators[i]; - for (j = 0; j < allocator->allocationCount; j += 1) + for (j = allocator->allocationCount - 1; j >= 0; j -= 1) { - for (k = 0; k < allocator->allocations[j]->freeRegionCount; k += 1) + for (k = allocator->allocations[j]->usedRegionCount - 1; k >= 0; k -= 1) { - SDL_free(allocator->allocations[j]->freeRegions[k]); + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + allocator->allocations[j]->usedRegions[k] + ); } - SDL_free(allocator->allocations[j]->freeRegions); - - renderer->vkFreeMemory( - renderer->logicalDevice, - allocator->allocations[j]->memory, - NULL + VULKAN_INTERNAL_DeallocateMemory( + renderer, + allocator, + j ); - - SDL_DestroyMutex(allocator->allocations[j]->memoryLock); - SDL_free(allocator->allocations[j]); } - SDL_free(allocator->allocations); - SDL_free(allocator->sortedFreeRegions); + if (renderer->memoryAllocator->subAllocators[i].allocations != NULL) + { + SDL_free(renderer->memoryAllocator->subAllocators[i].allocations); + } + + SDL_free(renderer->memoryAllocator->subAllocators[i].sortedFreeRegions); } SDL_free(renderer->memoryAllocator); @@ -7118,7 +7119,7 @@ static VulkanTransferBuffer* VULKAN_INTERNAL_AcquireTransferBuffer( 1 ); - if (transferBuffer == NULL) + if (transferBuffer->buffer == NULL) { Refresh_LogError("Failed to allocate transfer buffer!"); return NULL; @@ -10047,7 +10048,7 @@ static void VULKAN_INTERNAL_CleanCommandBuffer( for (i = 0; i < commandBuffer->transferBufferCount; i += 1) { - VULKAN_INTERNAL_DestroyBuffer(renderer, commandBuffer->transferBuffers[i]->buffer); + VULKAN_INTERNAL_QueueDestroyBuffer(renderer, commandBuffer->transferBuffers[i]->buffer); SDL_free(commandBuffer->transferBuffers[i]); commandBuffer->transferBuffers[i] = NULL; } @@ -10202,6 +10203,8 @@ static void VULKAN_Submit( VulkanCommandBuffer *currentCommandBuffer; VkPipelineStageFlags waitStages[MAX_PRESENT_COUNT]; uint32_t swapchainImageIndex; + uint8_t commandBufferCleaned = 0; + VulkanMemorySubAllocator *allocator; int32_t i, j; SDL_LockMutex(renderer->submitLock); @@ -10322,13 +10325,50 @@ static void VULKAN_Submit( renderer, renderer->submittedCommandBuffers[i] ); + + commandBufferCleaned = 1; } } + if (commandBufferCleaned) + { + SDL_LockMutex(renderer->allocatorLock); + + for (i = 0; i < VK_MAX_MEMORY_TYPES; i += 1) + { + allocator = &renderer->memoryAllocator->subAllocators[i]; + + for (j = allocator->allocationCount - 1; j >= 0; j -= 1) + { + if (allocator->allocations[j]->usedRegionCount == 0) + { + VULKAN_INTERNAL_DeallocateMemory( + renderer, + allocator, + j + ); + } + } + } + + SDL_UnlockMutex(renderer->allocatorLock); + } + /* Check pending destroys */ VULKAN_INTERNAL_PerformPendingDestroys(renderer); + /* Defrag! */ + if (renderer->needDefrag) + { + renderer->defragTimer += 1; + + if (renderer->defragTimer == 1) + { + VULKAN_INTERNAL_DefragmentMemory(renderer); + } + } + SDL_UnlockMutex(renderer->submitLock); } @@ -10343,7 +10383,6 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( VulkanTexture* newTexture; VkBufferCopy bufferCopy; VkImageCopy *imageCopyRegions; - VkImageAspectFlags aspectFlags; VulkanCommandBuffer *commandBuffer; VkCommandBufferBeginInfo beginInfo; VkPipelineStageFlags waitFlags = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; @@ -10393,7 +10432,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( RESOURCE_ACCESS_NONE, currentRegion->vulkanBuffer->usage, currentRegion->vulkanBuffer->preferDeviceLocal, - currentRegion->vulkanBuffer->isTransferBuffer + 0 ); if (newBuffer == NULL) @@ -10441,9 +10480,12 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( VULKAN_INTERNAL_TrackBuffer(renderer, commandBuffer, newBuffer); /* re-point original container to new buffer */ - newBuffer->container = currentRegion->vulkanBuffer->container; - newBuffer->container->vulkanBuffer = newBuffer; - currentRegion->vulkanBuffer->container = NULL; + if (currentRegion->vulkanBuffer->container != NULL) + { + newBuffer->container = currentRegion->vulkanBuffer->container; + newBuffer->container->vulkanBuffer = newBuffer; + currentRegion->vulkanBuffer->container = NULL; + } VULKAN_INTERNAL_QueueDestroyBuffer(renderer, currentRegion->vulkanBuffer); @@ -10476,7 +10518,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( renderer, commandBuffer->commandBuffer, RESOURCE_ACCESS_TRANSFER_READ, - aspectFlags, + currentRegion->vulkanTexture->aspectFlags, 0, currentRegion->vulkanTexture->layerCount, 0, @@ -10490,7 +10532,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( renderer, commandBuffer->commandBuffer, RESOURCE_ACCESS_TRANSFER_WRITE, - aspectFlags, + currentRegion->vulkanTexture->aspectFlags, 0, currentRegion->vulkanTexture->layerCount, 0, @@ -10507,7 +10549,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( imageCopyRegions[level].srcOffset.x = 0; imageCopyRegions[level].srcOffset.y = 0; imageCopyRegions[level].srcOffset.z = 0; - imageCopyRegions[level].srcSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].srcSubresource.aspectMask = currentRegion->vulkanTexture->aspectFlags; imageCopyRegions[level].srcSubresource.baseArrayLayer = 0; imageCopyRegions[level].srcSubresource.layerCount = currentRegion->vulkanTexture->layerCount; imageCopyRegions[level].srcSubresource.mipLevel = level; @@ -10517,7 +10559,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( imageCopyRegions[level].dstOffset.x = 0; imageCopyRegions[level].dstOffset.y = 0; imageCopyRegions[level].dstOffset.z = 0; - imageCopyRegions[level].dstSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].dstSubresource.aspectMask = currentRegion->vulkanTexture->aspectFlags; imageCopyRegions[level].dstSubresource.baseArrayLayer = 0; imageCopyRegions[level].dstSubresource.layerCount = currentRegion->vulkanTexture->layerCount; imageCopyRegions[level].dstSubresource.mipLevel = level; @@ -10537,7 +10579,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( renderer, commandBuffer->commandBuffer, originalResourceAccessType, - aspectFlags, + currentRegion->vulkanTexture->aspectFlags, 0, currentRegion->vulkanTexture->layerCount, 0, -- 2.25.1 From 45b71bd63d2986e3289ccecb2ad911d3be375c08 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Tue, 16 May 2023 16:05:43 -0700 Subject: [PATCH 06/12] fix CopyTextureToTexture --- src/Refresh_Driver_Vulkan.c | 42 +++++++++++++++---------------------- 1 file changed, 17 insertions(+), 25 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index fbc9f23..d1a0fee 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -7480,10 +7480,9 @@ static void VULKAN_SetTextureDataYUV( static void VULKAN_INTERNAL_BlitImage( VulkanRenderer *renderer, - VkCommandBuffer commandBuffer, + VulkanCommandBuffer *commandBuffer, Refresh_TextureSlice *sourceTextureSlice, Refresh_TextureSlice *destinationTextureSlice, - VulkanResourceAccessType newDestinationAccessType, VkFilter filter ) { VkImageBlit blit; @@ -7491,11 +7490,17 @@ static void VULKAN_INTERNAL_BlitImage( VulkanTexture *destinationTexture = ((VulkanTextureContainer*) destinationTextureSlice->texture)->vulkanTexture; VulkanResourceAccessType originalSourceAccessType = sourceTexture->resourceAccessType; + VulkanResourceAccessType originalDestinationAccessType = destinationTexture->resourceAccessType; + + if (originalDestinationAccessType == RESOURCE_ACCESS_NONE) + { + originalDestinationAccessType = RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE; + } /* TODO: is it worth it to only transition the specific subresource? */ VULKAN_INTERNAL_ImageMemoryBarrier( renderer, - commandBuffer, + commandBuffer->commandBuffer, RESOURCE_ACCESS_TRANSFER_READ, VK_IMAGE_ASPECT_COLOR_BIT, 0, @@ -7509,7 +7514,7 @@ static void VULKAN_INTERNAL_BlitImage( VULKAN_INTERNAL_ImageMemoryBarrier( renderer, - commandBuffer, + commandBuffer->commandBuffer, RESOURCE_ACCESS_TRANSFER_WRITE, VK_IMAGE_ASPECT_COLOR_BIT, 0, @@ -7546,7 +7551,7 @@ static void VULKAN_INTERNAL_BlitImage( blit.dstSubresource.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; renderer->vkCmdBlitImage( - commandBuffer, + commandBuffer->commandBuffer, sourceTexture->image, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, destinationTexture->image, @@ -7559,7 +7564,7 @@ static void VULKAN_INTERNAL_BlitImage( /* TODO: is it worth it to only transition the specific subresource? */ VULKAN_INTERNAL_ImageMemoryBarrier( renderer, - commandBuffer, + commandBuffer->commandBuffer, originalSourceAccessType, VK_IMAGE_ASPECT_COLOR_BIT, 0, @@ -7573,8 +7578,8 @@ static void VULKAN_INTERNAL_BlitImage( VULKAN_INTERNAL_ImageMemoryBarrier( renderer, - commandBuffer, - newDestinationAccessType, + commandBuffer->commandBuffer, + originalDestinationAccessType, VK_IMAGE_ASPECT_COLOR_BIT, 0, destinationTexture->layerCount, @@ -7584,6 +7589,9 @@ static void VULKAN_INTERNAL_BlitImage( destinationTexture->image, &destinationTexture->resourceAccessType ); + + VULKAN_INTERNAL_TrackTexture(renderer, commandBuffer, sourceTexture); + VULKAN_INTERNAL_TrackTexture(renderer, commandBuffer, destinationTexture); } REFRESHAPI void VULKAN_CopyTextureToTexture( @@ -7595,30 +7603,14 @@ REFRESHAPI void VULKAN_CopyTextureToTexture( ) { VulkanRenderer *renderer = (VulkanRenderer*)driverData; VulkanCommandBuffer *vulkanCommandBuffer = (VulkanCommandBuffer*) commandBuffer; - VulkanTexture *sourceTexture = (VulkanTexture*) sourceTextureSlice->texture; - VulkanTexture *destinationTexture = (VulkanTexture*) destinationTextureSlice->texture; - VulkanResourceAccessType destinationAccessType = destinationTexture->resourceAccessType; - - if (destinationTexture->usageFlags & VK_IMAGE_USAGE_SAMPLED_BIT) - { - destinationAccessType = RESOURCE_ACCESS_ANY_SHADER_READ_SAMPLED_IMAGE; - } - else if (destinationTexture->usageFlags & VK_IMAGE_USAGE_STORAGE_BIT) - { - destinationAccessType = RESOURCE_ACCESS_COMPUTE_SHADER_STORAGE_IMAGE_READ_WRITE; - } VULKAN_INTERNAL_BlitImage( renderer, - vulkanCommandBuffer->commandBuffer, + vulkanCommandBuffer, sourceTextureSlice, destinationTextureSlice, - destinationAccessType, RefreshToVK_Filter[filter] ); - - VULKAN_INTERNAL_TrackTexture(renderer, vulkanCommandBuffer, sourceTexture); - VULKAN_INTERNAL_TrackTexture(renderer, vulkanCommandBuffer, destinationTexture); } static void VULKAN_INTERNAL_SetBufferData( -- 2.25.1 From 994f5dc0b920f622470a96150ee8d9d8529f45fa Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Tue, 16 May 2023 18:20:43 -0700 Subject: [PATCH 07/12] fixed-size transfer pool --- src/Refresh_Driver_Vulkan.c | 114 +++++++++++++++++++++++++++++++++++- 1 file changed, 111 insertions(+), 3 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index d1a0fee..fb81d56 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -75,6 +75,7 @@ static uint32_t deviceExtensionCount = SDL_arraysize(deviceExtensionNames); #define MAX_ALLOCATION_SIZE 256000000 /* 256MB */ #define ALLOCATION_INCREMENT 16000000 /* 16MB */ #define TRANSFER_BUFFER_STARTING_SIZE 8000000 /* 8MB */ +#define POOLED_TRANSFER_BUFFER_SIZE 16000000 /* 16MB */ #define UBO_BUFFER_SIZE 16000 /* 16KB */ #define DESCRIPTOR_POOL_STARTING_SIZE 128 #define WINDOW_DATA "Refresh_VulkanWindowData" @@ -1516,8 +1517,18 @@ typedef struct VulkanTransferBuffer { VulkanBuffer* buffer; VkDeviceSize offset; + uint8_t fromPool; } VulkanTransferBuffer; +typedef struct VulkanTransferBufferPool +{ + SDL_mutex *lock; + + VulkanTransferBuffer **availableBuffers; + uint32_t availableBufferCount; + uint32_t availableBufferCapacity; +} VulkanTransferBufferPool; + typedef struct VulkanCommandPool VulkanCommandPool; typedef struct VulkanCommandBuffer @@ -1723,6 +1734,8 @@ typedef struct VulkanRenderer uint32_t submittedCommandBufferCount; uint32_t submittedCommandBufferCapacity; + VulkanTransferBufferPool transferBufferPool; + CommandPoolHashTable commandPoolHashTable; DescriptorSetLayoutHashTable descriptorSetLayoutHashTable; GraphicsPipelineLayoutHashTable graphicsPipelineLayoutHashTable; @@ -5151,6 +5164,15 @@ static void VULKAN_DestroyDevice( SDL_free(renderer->dummyFragmentUniformBuffer); SDL_free(renderer->dummyComputeUniformBuffer); + for (i = 0; i < renderer->transferBufferPool.availableBufferCount; i += 1) + { + VULKAN_INTERNAL_DestroyBuffer(renderer, renderer->transferBufferPool.availableBuffers[i]->buffer); + SDL_free(renderer->transferBufferPool.availableBuffers[i]); + } + + SDL_free(renderer->transferBufferPool.availableBuffers); + SDL_DestroyMutex(renderer->transferBufferPool.lock); + for (i = 0; i < NUM_COMMAND_POOL_BUCKETS; i += 1) { commandPoolHashArray = renderer->commandPoolHashTable.buckets[i]; @@ -7099,6 +7121,40 @@ static VulkanTransferBuffer* VULKAN_INTERNAL_AcquireTransferBuffer( } } + /* Nothing fits, can we get a transfer buffer from the pool? */ + + SDL_LockMutex(renderer->transferBufferPool.lock); + + for (i = 0; i < renderer->transferBufferPool.availableBufferCount; i += 1) + { + transferBuffer = renderer->transferBufferPool.availableBuffers[i]; + offset = transferBuffer->offset + alignment - (transferBuffer->offset % alignment); + + if (offset + requiredSize <= transferBuffer->buffer->size) + { + if (commandBuffer->transferBufferCount == commandBuffer->transferBufferCapacity) + { + commandBuffer->transferBufferCapacity *= 2; + commandBuffer->transferBuffers = SDL_realloc( + commandBuffer->transferBuffers, + commandBuffer->transferBufferCapacity * sizeof(VulkanTransferBuffer*) + ); + } + + commandBuffer->transferBuffers[commandBuffer->transferBufferCount] = transferBuffer; + commandBuffer->transferBufferCount += 1; + + renderer->transferBufferPool.availableBuffers[i] = renderer->transferBufferPool.availableBuffers[renderer->transferBufferPool.availableBufferCount - 1]; + renderer->transferBufferPool.availableBufferCount -= 1; + SDL_UnlockMutex(renderer->transferBufferPool.lock); + + transferBuffer->offset = offset; + return transferBuffer; + } + } + + SDL_UnlockMutex(renderer->transferBufferPool.lock); + /* Nothing fits, so let's create a new transfer buffer */ size = TRANSFER_BUFFER_STARTING_SIZE; @@ -7118,10 +7174,12 @@ static VulkanTransferBuffer* VULKAN_INTERNAL_AcquireTransferBuffer( 1, 1 ); + transferBuffer->fromPool = 0; if (transferBuffer->buffer == NULL) { Refresh_LogError("Failed to allocate transfer buffer!"); + SDL_free(transferBuffer); return NULL; } @@ -10040,9 +10098,22 @@ static void VULKAN_INTERNAL_CleanCommandBuffer( for (i = 0; i < commandBuffer->transferBufferCount; i += 1) { - VULKAN_INTERNAL_QueueDestroyBuffer(renderer, commandBuffer->transferBuffers[i]->buffer); - SDL_free(commandBuffer->transferBuffers[i]); - commandBuffer->transferBuffers[i] = NULL; + if (commandBuffer->transferBuffers[i]->fromPool) + { + SDL_LockMutex(renderer->transferBufferPool.lock); + + commandBuffer->transferBuffers[i]->offset = 0; + renderer->transferBufferPool.availableBuffers[renderer->transferBufferPool.availableBufferCount] = commandBuffer->transferBuffers[i]; + renderer->transferBufferPool.availableBufferCount += 1; + + SDL_UnlockMutex(renderer->transferBufferPool.lock); + } + else + { + VULKAN_INTERNAL_QueueDestroyBuffer(renderer, commandBuffer->transferBuffers[i]->buffer); + SDL_free(commandBuffer->transferBuffers[i]); + commandBuffer->transferBuffers[i] = NULL; + } } commandBuffer->transferBufferCount = 0; @@ -11392,6 +11463,9 @@ static Refresh_Device* VULKAN_CreateDevice( /* Variables: Image Format Detection */ VkImageFormatProperties imageFormatProperties; + /* Variables: Transfer buffer init */ + VulkanTransferBuffer *transferBuffer; + SDL_memset(renderer, '\0', sizeof(VulkanRenderer)); renderer->debugMode = debugMode; @@ -11774,6 +11848,40 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->renderTargetHashArray.count = 0; renderer->renderTargetHashArray.capacity = 0; + /* Initialize transfer buffer pool */ + + renderer->transferBufferPool.lock = SDL_CreateMutex(); + + renderer->transferBufferPool.availableBufferCapacity = 4; + renderer->transferBufferPool.availableBufferCount = 0; + renderer->transferBufferPool.availableBuffers = SDL_malloc( + renderer->transferBufferPool.availableBufferCapacity * sizeof(VulkanTransferBuffer*) + ); + + for (i = 0; i < renderer->transferBufferPool.availableBufferCapacity; i += 1) + { + transferBuffer = SDL_malloc(sizeof(VulkanTransferBuffer)); + transferBuffer->offset = 0; + transferBuffer->fromPool = 1; + transferBuffer->buffer = VULKAN_INTERNAL_CreateBuffer( + renderer, + POOLED_TRANSFER_BUFFER_SIZE, + RESOURCE_ACCESS_TRANSFER_READ_WRITE, + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + 1, + 1 + ); + + if (transferBuffer->buffer == NULL) + { + Refresh_LogError("Failed to allocate transfer buffer!"); + SDL_free(transferBuffer); + } + + renderer->transferBufferPool.availableBuffers[i] = transferBuffer; + renderer->transferBufferPool.availableBufferCount += 1; + } + /* Some drivers don't support D16, so we have to fall back to D32. */ vulkanResult = renderer->vkGetPhysicalDeviceImageFormatProperties( -- 2.25.1 From 48761bba56f93d9c77f6c6e791f2b28e06c83ece Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Tue, 16 May 2023 18:21:30 -0700 Subject: [PATCH 08/12] add comment --- src/Refresh_Driver_Vulkan.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index fb81d56..b4b6a57 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -10096,6 +10096,8 @@ static void VULKAN_INTERNAL_CleanCommandBuffer( commandBuffer->boundUniformBufferCount = 0; + /* Clean up transfer buffers */ + for (i = 0; i < commandBuffer->transferBufferCount; i += 1) { if (commandBuffer->transferBuffers[i]->fromPool) -- 2.25.1 From 0ef91c7f42309a3a484f129277492e379294f474 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Tue, 16 May 2023 18:52:53 -0700 Subject: [PATCH 09/12] release allocator lock before submitting --- src/Refresh_Driver_Vulkan.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index b4b6a57..1b3c43b 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -10671,6 +10671,8 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( } } + SDL_UnlockMutex(renderer->allocatorLock); + VULKAN_Submit( (Refresh_Renderer*) renderer, 1, @@ -10679,8 +10681,6 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( renderer->defragTimer = 0; - SDL_UnlockMutex(renderer->allocatorLock); - return 1; } -- 2.25.1 From 313b4b81cca008057513510d847ac53ec7d03efd Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Tue, 16 May 2023 20:02:39 -0700 Subject: [PATCH 10/12] rework nested mutex lock on FetchFramebuffer --- src/Refresh_Driver_Vulkan.c | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index 1b3c43b..50ebd0e 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -1811,7 +1811,6 @@ typedef struct VulkanRenderer uint8_t needDefrag; uint32_t defragTimer; - uint8_t resourceFreed; #define VULKAN_INSTANCE_FUNCTION(ext, ret, func, params) \ vkfntype_##func func; @@ -2361,7 +2360,6 @@ static void VULKAN_INTERNAL_RemoveMemoryUsedRegion( SDL_free(usedRegion); - renderer->resourceFreed = 1; SDL_UnlockMutex(renderer->allocatorLock); } @@ -8489,8 +8487,6 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( uint32_t attachmentCount = 0; uint32_t i; - SDL_LockMutex(renderer->framebufferFetchLock); - for (i = 0; i < MAX_COLOR_TARGET_BINDINGS; i += 1) { hash.colorAttachmentViews[i] = VK_NULL_HANDLE; @@ -8551,14 +8547,17 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( hash.width = width; hash.height = height; + SDL_LockMutex(renderer->framebufferFetchLock); + vulkanFramebuffer = FramebufferHashArray_Fetch( &renderer->framebufferHashArray, &hash ); + SDL_UnlockMutex(renderer->framebufferFetchLock); + if (vulkanFramebuffer != NULL) { - SDL_UnlockMutex(renderer->framebufferFetchLock); return vulkanFramebuffer; } @@ -8637,11 +8636,15 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( if (result == VK_SUCCESS) { + SDL_LockMutex(renderer->framebufferFetchLock); + FramebufferHashArray_Insert( &renderer->framebufferHashArray, hash, vulkanFramebuffer ); + + SDL_UnlockMutex(renderer->framebufferFetchLock); } else { @@ -8650,7 +8653,6 @@ static VulkanFramebuffer* VULKAN_INTERNAL_FetchFramebuffer( vulkanFramebuffer = NULL; } - SDL_UnlockMutex(renderer->framebufferFetchLock); return vulkanFramebuffer; } @@ -10246,7 +10248,11 @@ static void VULKAN_Wait( { LogVulkanResultAsError("vkWaitForFences", result); } + } + for (i = renderer->submittedCommandBufferCount - 1; i >= 0; i -= 1) + { + commandBuffer = renderer->submittedCommandBuffers[i]; VULKAN_INTERNAL_CleanCommandBuffer(renderer, commandBuffer); } @@ -11983,7 +11989,6 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->needDefrag = 0; renderer->defragTimer = 0; - renderer->resourceFreed = 0; return result; } -- 2.25.1 From e482783c423dc90f247a5a8f842ec0c078c0ce12 Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Wed, 17 May 2023 12:22:52 -0700 Subject: [PATCH 11/12] deadlock fix + defrag timing tweak --- src/Refresh_Driver_Vulkan.c | 63 ++++++++++++++++++++++++++----------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index 50ebd0e..2130818 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -78,6 +78,7 @@ static uint32_t deviceExtensionCount = SDL_arraysize(deviceExtensionNames); #define POOLED_TRANSFER_BUFFER_SIZE 16000000 /* 16MB */ #define UBO_BUFFER_SIZE 16000 /* 16KB */ #define DESCRIPTOR_POOL_STARTING_SIZE 128 +#define DEFRAG_TIME 200 #define WINDOW_DATA "Refresh_VulkanWindowData" #define IDENTITY_SWIZZLE \ @@ -1810,7 +1811,8 @@ typedef struct VulkanRenderer SDL_mutex *renderTargetFetchLock; uint8_t needDefrag; - uint32_t defragTimer; + uint64_t defragTimestamp; + uint8_t defragInProgress; #define VULKAN_INSTANCE_FUNCTION(ext, ret, func, params) \ vkfntype_##func func; @@ -2356,6 +2358,7 @@ static void VULKAN_INTERNAL_RemoveMemoryUsedRegion( if (!usedRegion->allocation->dedicated) { renderer->needDefrag = 1; + renderer->defragTimestamp = SDL_GetTicks64() + DEFRAG_TIME; /* reset timer so we batch defrags */ } SDL_free(usedRegion); @@ -3445,8 +3448,6 @@ static void VULKAN_INTERNAL_RemoveFramebuffersContainingView( FramebufferHash *hash; int32_t i, j; - SDL_LockMutex(renderer->framebufferFetchLock); - for (i = renderer->framebufferHashArray.count - 1; i >= 0; i -= 1) { hash = &renderer->framebufferHashArray.elements[i].key; @@ -3469,8 +3470,6 @@ static void VULKAN_INTERNAL_RemoveFramebuffersContainingView( } } } - - SDL_UnlockMutex(renderer->framebufferFetchLock); } static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( @@ -3478,8 +3477,15 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( VulkanTexture *texture ) { RenderTargetHash *hash; + VkImageView *viewsToCheck; + int32_t viewsToCheckCount; + int32_t viewsToCheckCapacity; int32_t i; + viewsToCheckCapacity = 16; + viewsToCheckCount = 0; + viewsToCheck = SDL_malloc(sizeof(VkImageView) * viewsToCheckCapacity); + SDL_LockMutex(renderer->renderTargetFetchLock); for (i = renderer->renderTargetHashArray.count - 1; i >= 0; i -= 1) @@ -3488,11 +3494,17 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( if (hash->texture == texture) { - VULKAN_INTERNAL_RemoveFramebuffersContainingView( - renderer, - renderer->renderTargetHashArray.elements[i].value->view + EXPAND_ARRAY_IF_NEEDED( + viewsToCheck, + VkImageView, + viewsToCheckCount + 1, + viewsToCheckCapacity, + viewsToCheckCapacity * 2 ); + viewsToCheck[viewsToCheckCount] = renderer->renderTargetHashArray.elements[i].value->view; + viewsToCheckCount += 1; + VULKAN_INTERNAL_DestroyRenderTarget( renderer, renderer->renderTargetHashArray.elements[i].value @@ -3502,10 +3514,26 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( &renderer->renderTargetHashArray, i ); + + Refresh_LogInfo("destroying render target!"); } } SDL_UnlockMutex(renderer->renderTargetFetchLock); + + SDL_LockMutex(renderer->framebufferFetchLock); + + for (i = 0; i < viewsToCheckCount; i += 1) + { + VULKAN_INTERNAL_RemoveFramebuffersContainingView( + renderer, + viewsToCheck[i] + ); + } + + SDL_UnlockMutex(renderer->framebufferFetchLock); + + SDL_free(viewsToCheck); } static void VULKAN_INTERNAL_DestroyTexture( @@ -3550,11 +3578,6 @@ static void VULKAN_INTERNAL_DestroyRenderTarget( VulkanRenderer *renderer, VulkanRenderTarget *renderTarget ) { - VULKAN_INTERNAL_RemoveFramebuffersContainingView( - renderer, - renderTarget->view - ); - renderer->vkDestroyImageView( renderer->logicalDevice, renderTarget->view, @@ -10430,11 +10453,9 @@ static void VULKAN_Submit( VULKAN_INTERNAL_PerformPendingDestroys(renderer); /* Defrag! */ - if (renderer->needDefrag) + if (renderer->needDefrag && !renderer->defragInProgress) { - renderer->defragTimer += 1; - - if (renderer->defragTimer == 1) + if (SDL_GetTicks64() >= renderer->defragTimestamp) { VULKAN_INTERNAL_DefragmentMemory(renderer); } @@ -10464,6 +10485,7 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( SDL_LockMutex(renderer->allocatorLock); renderer->needDefrag = 0; + renderer->defragInProgress = 1; beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; beginInfo.pNext = NULL; @@ -10679,13 +10701,15 @@ static uint8_t VULKAN_INTERNAL_DefragmentMemory( SDL_UnlockMutex(renderer->allocatorLock); + renderer->defragTimestamp = SDL_GetTicks64() + DEFRAG_TIME; + VULKAN_Submit( (Refresh_Renderer*) renderer, 1, (Refresh_CommandBuffer**) &commandBuffer ); - renderer->defragTimer = 0; + renderer->defragInProgress = 0; return 1; } @@ -11988,7 +12012,8 @@ static Refresh_Device* VULKAN_CreateDevice( ); renderer->needDefrag = 0; - renderer->defragTimer = 0; + renderer->defragTimestamp = 0; + renderer->defragInProgress = 0; return result; } -- 2.25.1 From 5838db73b76a64e6f7683da077272a0b44c332af Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Wed, 17 May 2023 12:23:48 -0700 Subject: [PATCH 12/12] remove destroy render target log --- src/Refresh_Driver_Vulkan.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index 2130818..8e98ce2 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -3514,8 +3514,6 @@ static void VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( &renderer->renderTargetHashArray, i ); - - Refresh_LogInfo("destroying render target!"); } } -- 2.25.1