From 746ad10c7baa4599055812eba2d35ff183c043fa Mon Sep 17 00:00:00 2001 From: cosmonaut Date: Fri, 12 May 2023 15:11:18 -0700 Subject: [PATCH] start on memory management rewrite --- src/Refresh_Driver_Vulkan.c | 1515 +++++++++++++++++++++++++---------- 1 file changed, 1095 insertions(+), 420 deletions(-) diff --git a/src/Refresh_Driver_Vulkan.c b/src/Refresh_Driver_Vulkan.c index 2deae02..95913ff 100644 --- a/src/Refresh_Driver_Vulkan.c +++ b/src/Refresh_Driver_Vulkan.c @@ -71,12 +71,12 @@ static uint32_t deviceExtensionCount = SDL_arraysize(deviceExtensionNames); /* Defines */ -#define STARTING_ALLOCATION_SIZE 64000000 /* 64MB */ -#define MAX_ALLOCATION_SIZE 256000000 /* 256MB */ +#define STARTING_ALLOCATION_SIZE 64000000 /* 64MB */ +#define MAX_ALLOCATION_SIZE 256000000 /* 256MB */ +#define ALLOCATION_INCREMENT 16000000 /* 16MB */ #define TRANSFER_BUFFER_STARTING_SIZE 8000000 /* 8MB */ -#define UBO_BUFFER_SIZE 16000 /* 16KB */ +#define UBO_BUFFER_SIZE 16000 /* 16KB */ #define DESCRIPTOR_POOL_STARTING_SIZE 128 -#define DESCRIPTOR_SET_DEACTIVATE_FRAMES 10 #define WINDOW_DATA "Refresh_VulkanWindowData" #define IDENTITY_SWIZZLE \ @@ -389,6 +389,8 @@ static VkBorderColor RefreshToVK_BorderColor[] = /* Memory Allocation */ typedef struct VulkanMemoryAllocation VulkanMemoryAllocation; +typedef struct VulkanBuffer VulkanBuffer; +typedef struct VulkanTexture VulkanTexture; typedef struct VulkanMemoryFreeRegion { @@ -399,8 +401,25 @@ typedef struct VulkanMemoryFreeRegion uint32_t sortedIndex; } VulkanMemoryFreeRegion; +typedef struct VulkanMemoryUsedRegion +{ + VulkanMemoryAllocation *allocation; + VkDeviceSize offset; + VkDeviceSize size; + VkDeviceSize resourceOffset; /* differs from offset based on alignment*/ + VkDeviceSize resourceSize; /* differs from size based on alignment */ + VkDeviceSize alignment; + uint8_t isBuffer; + REFRESHNAMELESS union + { + VulkanBuffer *vulkanBuffer; + VulkanTexture *vulkanTexture; + }; +} VulkanMemoryUsedRegion; + typedef struct VulkanMemorySubAllocator { + uint32_t memoryTypeIndex; VkDeviceSize nextAllocationSize; VulkanMemoryAllocation **allocations; uint32_t allocationCount; @@ -414,10 +433,16 @@ struct VulkanMemoryAllocation VulkanMemorySubAllocator *allocator; VkDeviceMemory memory; VkDeviceSize size; + VulkanMemoryUsedRegion **usedRegions; + uint32_t usedRegionCount; + uint32_t usedRegionCapacity; VulkanMemoryFreeRegion **freeRegions; uint32_t freeRegionCount; uint32_t freeRegionCapacity; uint8_t dedicated; + uint8_t availableForAllocation; + VkDeviceSize freeSpace; + VkDeviceSize usedSpace; uint8_t *mapPointer; SDL_mutex *memoryLock; }; @@ -664,18 +689,16 @@ static const VulkanResourceAccessInfo AccessMap[RESOURCE_ACCESS_TYPES_COUNT] = /* Memory structures */ -typedef struct VulkanBuffer /* cast from Refresh_Buffer */ +struct VulkanBuffer /* cast from Refresh_Buffer */ { VkBuffer buffer; VkDeviceSize size; - VkDeviceSize offset; /* move this to UsedMemoryRegion system */ - VkDeviceSize memorySize; /* move this to UsedMemoryRegion system */ - VulkanMemoryAllocation *allocation; /* see above */ + VulkanMemoryUsedRegion *usedRegion; VulkanResourceAccessType resourceAccessType; VkBufferUsageFlags usage; SDL_atomic_t referenceCount; /* Tracks command buffer usage */ -} VulkanBuffer; +}; typedef struct VulkanUniformBufferPool VulkanUniformBufferPool; @@ -742,11 +765,9 @@ typedef struct VulkanShaderModule SDL_atomic_t referenceCount; } VulkanShaderModule; -typedef struct VulkanTexture +struct VulkanTexture { - VulkanMemoryAllocation *allocation; - VkDeviceSize offset; - VkDeviceSize memorySize; + VulkanMemoryUsedRegion *usedRegion; VkImage image; VkImageView view; @@ -768,7 +789,7 @@ typedef struct VulkanTexture struct VulkanTexture *msaaTex; SDL_atomic_t referenceCount; -} VulkanTexture; +}; typedef struct VulkanRenderTarget { @@ -1481,15 +1502,6 @@ typedef struct VulkanTransferBuffer VkDeviceSize offset; } VulkanTransferBuffer; -typedef struct VulkanTransferBufferPool -{ - SDL_mutex *lock; - - VulkanTransferBuffer **availableBuffers; - uint32_t availableBufferCount; - uint32_t availableBufferCapacity; -} VulkanTransferBufferPool; - typedef struct VulkanCommandPool VulkanCommandPool; typedef struct VulkanCommandBuffer @@ -1673,6 +1685,7 @@ typedef struct VulkanRenderer VkPhysicalDeviceProperties2 physicalDeviceProperties; VkPhysicalDeviceDriverPropertiesKHR physicalDeviceDriverProperties; VkDevice logicalDevice; + uint8_t unifiedMemoryWarning; uint8_t supportsDebugUtils; uint8_t debugMode; @@ -1694,8 +1707,6 @@ typedef struct VulkanRenderer uint32_t submittedCommandBufferCount; uint32_t submittedCommandBufferCapacity; - VulkanTransferBufferPool transferBufferPool; - CommandPoolHashTable commandPoolHashTable; DescriptorSetLayoutHashTable descriptorSetLayoutHashTable; GraphicsPipelineLayoutHashTable graphicsPipelineLayoutHashTable; @@ -1769,7 +1780,11 @@ typedef struct VulkanRenderer SDL_mutex *framebufferFetchLock; SDL_mutex *renderTargetFetchLock; - #define VULKAN_INSTANCE_FUNCTION(ext, ret, func, params) \ + uint8_t needDefrag; + uint32_t defragTimer; + uint8_t resourceFreed; + +#define VULKAN_INSTANCE_FUNCTION(ext, ret, func, params) \ vkfntype_##func func; #define VULKAN_DEVICE_FUNCTION(ext, ret, func, params) \ vkfntype_##func func; @@ -1840,6 +1855,13 @@ static inline void LogVulkanResultAsWarn( } } +#define VULKAN_ERROR_CHECK(res, fn, ret) \ + if (res != VK_SUCCESS) \ + { \ + FNA3D_LogError("%s %s", #fn, VkErrorMessages(res)); \ + return ret; \ + } + /* Utility */ static inline VkFormat RefreshToVK_DepthFormat( @@ -2043,6 +2065,8 @@ static inline Refresh_SampleCount VULKAN_INTERNAL_GetMaxMultiSampleCount( /* Memory Management */ +/* Vulkan: Memory Allocation */ + static inline VkDeviceSize VULKAN_INTERNAL_NextHighestAlignment( VkDeviceSize n, VkDeviceSize align @@ -2050,24 +2074,59 @@ static inline VkDeviceSize VULKAN_INTERNAL_NextHighestAlignment( return align * ((n + align - 1) / align); } +static void VULKAN_INTERNAL_MakeMemoryUnavailable( + VulkanRenderer* renderer, + VulkanMemoryAllocation *allocation +) { + uint32_t i, j; + VulkanMemoryFreeRegion *freeRegion; + + allocation->availableForAllocation = 0; + + for (i = 0; i < allocation->freeRegionCount; i += 1) + { + freeRegion = allocation->freeRegions[i]; + + /* close the gap in the sorted list */ + if (allocation->allocator->sortedFreeRegionCount > 1) + { + for (j = freeRegion->sortedIndex; j < allocation->allocator->sortedFreeRegionCount - 1; j += 1) + { + allocation->allocator->sortedFreeRegions[j] = + allocation->allocator->sortedFreeRegions[j + 1]; + + allocation->allocator->sortedFreeRegions[j]->sortedIndex = j; + } + } + + allocation->allocator->sortedFreeRegionCount -= 1; + } +} + static void VULKAN_INTERNAL_RemoveMemoryFreeRegion( + VulkanRenderer *renderer, VulkanMemoryFreeRegion *freeRegion ) { uint32_t i; - /* close the gap in the sorted list */ - if (freeRegion->allocation->allocator->sortedFreeRegionCount > 1) + SDL_LockMutex(renderer->allocatorLock); + + if (freeRegion->allocation->availableForAllocation) { - for (i = freeRegion->sortedIndex; i < freeRegion->allocation->allocator->sortedFreeRegionCount - 1; i += 1) + /* close the gap in the sorted list */ + if (freeRegion->allocation->allocator->sortedFreeRegionCount > 1) { - freeRegion->allocation->allocator->sortedFreeRegions[i] = - freeRegion->allocation->allocator->sortedFreeRegions[i + 1]; + for (i = freeRegion->sortedIndex; i < freeRegion->allocation->allocator->sortedFreeRegionCount - 1; i += 1) + { + freeRegion->allocation->allocator->sortedFreeRegions[i] = + freeRegion->allocation->allocator->sortedFreeRegions[i + 1]; - freeRegion->allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; + freeRegion->allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; + } } - } - freeRegion->allocation->allocator->sortedFreeRegionCount -= 1; + freeRegion->allocation->allocator->sortedFreeRegionCount -= 1; + } /* close the gap in the buffer list */ if (freeRegion->allocation->freeRegionCount > 1 && freeRegion->allocationIndex != freeRegion->allocation->freeRegionCount - 1) @@ -2081,10 +2140,15 @@ static void VULKAN_INTERNAL_RemoveMemoryFreeRegion( freeRegion->allocation->freeRegionCount -= 1; + freeRegion->allocation->freeSpace -= freeRegion->size; + SDL_free(freeRegion); + + SDL_UnlockMutex(renderer->allocatorLock); } static void VULKAN_INTERNAL_NewMemoryFreeRegion( + VulkanRenderer *renderer, VulkanMemoryAllocation *allocation, VkDeviceSize offset, VkDeviceSize size @@ -2094,6 +2158,8 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( int32_t insertionIndex = 0; int32_t i; + SDL_LockMutex(renderer->allocatorLock); + /* look for an adjacent region to merge */ for (i = allocation->freeRegionCount - 1; i >= 0; i -= 1) { @@ -2103,8 +2169,10 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( newOffset = allocation->freeRegions[i]->offset; newSize = allocation->freeRegions[i]->size + size; - VULKAN_INTERNAL_RemoveMemoryFreeRegion(allocation->freeRegions[i]); - VULKAN_INTERNAL_NewMemoryFreeRegion(allocation, newOffset, newSize); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, allocation->freeRegions[i]); + VULKAN_INTERNAL_NewMemoryFreeRegion(renderer, allocation, newOffset, newSize); + + SDL_UnlockMutex(renderer->allocatorLock); return; } @@ -2114,8 +2182,10 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( newOffset = offset; newSize = allocation->freeRegions[i]->size + size; - VULKAN_INTERNAL_RemoveMemoryFreeRegion(allocation->freeRegions[i]); - VULKAN_INTERNAL_NewMemoryFreeRegion(allocation, newOffset, newSize); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, allocation->freeRegions[i]); + VULKAN_INTERNAL_NewMemoryFreeRegion(renderer, allocation, newOffset, newSize); + + SDL_UnlockMutex(renderer->allocatorLock); return; } } @@ -2136,71 +2206,140 @@ static void VULKAN_INTERNAL_NewMemoryFreeRegion( newFreeRegion->size = size; newFreeRegion->allocation = allocation; + allocation->freeSpace += size; + allocation->freeRegions[allocation->freeRegionCount - 1] = newFreeRegion; newFreeRegion->allocationIndex = allocation->freeRegionCount - 1; - for (i = 0; i < allocation->allocator->sortedFreeRegionCount; i += 1) + if (allocation->availableForAllocation) { - if (allocation->allocator->sortedFreeRegions[i]->size < size) + for (i = 0; i < allocation->allocator->sortedFreeRegionCount; i += 1) { - /* this is where the new region should go */ - break; + if (allocation->allocator->sortedFreeRegions[i]->size < size) + { + /* this is where the new region should go */ + break; + } + + insertionIndex += 1; } - insertionIndex += 1; + if (allocation->allocator->sortedFreeRegionCount + 1 > allocation->allocator->sortedFreeRegionCapacity) + { + allocation->allocator->sortedFreeRegionCapacity *= 2; + allocation->allocator->sortedFreeRegions = SDL_realloc( + allocation->allocator->sortedFreeRegions, + sizeof(VulkanMemoryFreeRegion*) * allocation->allocator->sortedFreeRegionCapacity + ); + } + + /* perform insertion sort */ + if (allocation->allocator->sortedFreeRegionCount > 0 && insertionIndex != allocation->allocator->sortedFreeRegionCount) + { + for (i = allocation->allocator->sortedFreeRegionCount; i > insertionIndex && i > 0; i -= 1) + { + allocation->allocator->sortedFreeRegions[i] = allocation->allocator->sortedFreeRegions[i - 1]; + allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; + } + } + + allocation->allocator->sortedFreeRegionCount += 1; + allocation->allocator->sortedFreeRegions[insertionIndex] = newFreeRegion; + newFreeRegion->sortedIndex = insertionIndex; } - if (allocation->allocator->sortedFreeRegionCount + 1 > allocation->allocator->sortedFreeRegionCapacity) + SDL_UnlockMutex(renderer->allocatorLock); +} + +static VulkanMemoryUsedRegion* VULKAN_INTERNAL_NewMemoryUsedRegion( + VulkanRenderer *renderer, + VulkanMemoryAllocation *allocation, + VkDeviceSize offset, + VkDeviceSize size, + VkDeviceSize resourceOffset, + VkDeviceSize resourceSize, + VkDeviceSize alignment +) { + VulkanMemoryUsedRegion *memoryUsedRegion; + + SDL_LockMutex(renderer->allocatorLock); + + if (allocation->usedRegionCount == allocation->usedRegionCapacity) { - allocation->allocator->sortedFreeRegionCapacity *= 2; - allocation->allocator->sortedFreeRegions = SDL_realloc( - allocation->allocator->sortedFreeRegions, - sizeof(VulkanMemoryFreeRegion*) * allocation->allocator->sortedFreeRegionCapacity + allocation->usedRegionCapacity *= 2; + allocation->usedRegions = SDL_realloc( + allocation->usedRegions, + allocation->usedRegionCapacity * sizeof(VulkanMemoryUsedRegion*) ); } - /* perform insertion sort */ - if (allocation->allocator->sortedFreeRegionCount > 0 && insertionIndex != allocation->allocator->sortedFreeRegionCount) - { - for (i = allocation->allocator->sortedFreeRegionCount; i > insertionIndex && i > 0; i -= 1) - { - allocation->allocator->sortedFreeRegions[i] = allocation->allocator->sortedFreeRegions[i - 1]; - allocation->allocator->sortedFreeRegions[i]->sortedIndex = i; - } - } + memoryUsedRegion = SDL_malloc(sizeof(VulkanMemoryUsedRegion)); + memoryUsedRegion->allocation = allocation; + memoryUsedRegion->offset = offset; + memoryUsedRegion->size = size; + memoryUsedRegion->resourceOffset = resourceOffset; + memoryUsedRegion->resourceSize = resourceSize; + memoryUsedRegion->alignment = alignment; - allocation->allocator->sortedFreeRegionCount += 1; - allocation->allocator->sortedFreeRegions[insertionIndex] = newFreeRegion; - newFreeRegion->sortedIndex = insertionIndex; + allocation->usedSpace += size; + + allocation->usedRegions[allocation->usedRegionCount] = memoryUsedRegion; + allocation->usedRegionCount += 1; + + SDL_UnlockMutex(renderer->allocatorLock); + + return memoryUsedRegion; } -static uint8_t VULKAN_INTERNAL_FindMemoryType( +static void VULKAN_INTERNAL_RemoveMemoryUsedRegion( VulkanRenderer *renderer, - uint32_t typeFilter, - VkMemoryPropertyFlags requiredProperties, - VkMemoryPropertyFlags ignoredProperties, - uint32_t *memoryTypeIndex + VulkanMemoryUsedRegion *usedRegion ) { uint32_t i; - for (i = *memoryTypeIndex; i < renderer->memoryProperties.memoryTypeCount; i += 1) + SDL_LockMutex(renderer->allocatorLock); + + for (i = 0; i < usedRegion->allocation->usedRegionCount; i += 1) { - if ( (typeFilter & (1 << i)) && - (renderer->memoryProperties.memoryTypes[i].propertyFlags & requiredProperties) == requiredProperties && - (renderer->memoryProperties.memoryTypes[i].propertyFlags & ignoredProperties) == 0 ) + if (usedRegion->allocation->usedRegions[i] == usedRegion) { - *memoryTypeIndex = i; - return 1; + /* plug the hole */ + if (i != usedRegion->allocation->usedRegionCount - 1) + { + usedRegion->allocation->usedRegions[i] = usedRegion->allocation->usedRegions[usedRegion->allocation->usedRegionCount - 1]; + } + + break; } } - Refresh_LogError("Failed to find memory properties %X, required %X, ignored %X", typeFilter, requiredProperties, ignoredProperties); - return 0; + usedRegion->allocation->usedSpace -= usedRegion->size; + + usedRegion->allocation->usedRegionCount -= 1; + + VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, + usedRegion->allocation, + usedRegion->offset, + usedRegion->size + ); + + if (!usedRegion->allocation->dedicated) + { + renderer->needDefrag = 1; + } + + SDL_free(usedRegion); + + renderer->resourceFreed = 1; + SDL_UnlockMutex(renderer->allocatorLock); } static uint8_t VULKAN_INTERNAL_FindBufferMemoryRequirements( VulkanRenderer *renderer, VkBuffer buffer, + VkMemoryPropertyFlags requiredMemoryProperties, + VkMemoryPropertyFlags ignoredMemoryProperties, VkMemoryRequirements2KHR *pMemoryRequirements, uint32_t *pMemoryTypeIndex ) { @@ -2216,20 +2355,13 @@ static uint8_t VULKAN_INTERNAL_FindBufferMemoryRequirements( pMemoryRequirements ); - if (!VULKAN_INTERNAL_FindMemoryType( + return VULKAN_INTERNAL_FindMemoryType( renderer, pMemoryRequirements->memoryRequirements.memoryTypeBits, - VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT, - 0, + requiredMemoryProperties, + ignoredMemoryProperties, pMemoryTypeIndex - )) { - Refresh_LogError( - "Could not find valid memory type for buffer creation" - ); - return 0; - } - - return 1; + ); } static uint8_t VULKAN_INTERNAL_FindImageMemoryRequirements( @@ -2252,20 +2384,60 @@ static uint8_t VULKAN_INTERNAL_FindImageMemoryRequirements( pMemoryRequirements ); - if (!VULKAN_INTERNAL_FindMemoryType( + return VULKAN_INTERNAL_FindMemoryType( renderer, pMemoryRequirements->memoryRequirements.memoryTypeBits, requiredMemoryPropertyFlags, ignoredMemoryPropertyFlags, pMemoryTypeIndex - )) { - Refresh_LogError( - "Could not find valid memory type for image creation" + ); +} + +static void VULKAN_INTERNAL_DeallocateMemory( + VulkanRenderer *renderer, + VulkanMemorySubAllocator *allocator, + uint32_t allocationIndex +) { + uint32_t i; + uint8_t isDeviceLocal = + (renderer->memoryProperties.memoryTypes[allocator->memoryTypeIndex].propertyFlags & + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0; + + VulkanMemoryAllocation *allocation = allocator->allocations[allocationIndex]; + + SDL_LockMutex(renderer->allocatorLock); + + for (i = 0; i < allocation->freeRegionCount; i += 1) + { + VULKAN_INTERNAL_RemoveMemoryFreeRegion( + renderer, + allocation->freeRegions[i] ); - return 0; + } + SDL_free(allocation->freeRegions); + + /* no need to iterate used regions because deallocate + * only happens when there are 0 used regions + */ + SDL_free(allocation->usedRegions); + + renderer->vkFreeMemory( + renderer->logicalDevice, + allocation->memory, + NULL + ); + + SDL_DestroyMutex(allocation->memoryLock); + SDL_free(allocation); + + if (allocationIndex != allocator->allocationCount - 1) + { + allocator->allocations[allocationIndex] = allocator->allocations[allocator->allocationCount - 1]; } - return 1; + allocator->allocationCount -= 1; + + SDL_UnlockMutex(renderer->allocatorLock); } static uint8_t VULKAN_INTERNAL_AllocateMemory( @@ -2290,8 +2462,20 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( allocation = SDL_malloc(sizeof(VulkanMemoryAllocation)); allocation->size = allocationSize; + allocation->freeSpace = 0; /* added by FreeRegions */ + allocation->usedSpace = 0; /* added by UsedRegions */ allocation->memoryLock = SDL_CreateMutex(); + allocator->allocationCount += 1; + allocator->allocations = SDL_realloc( + allocator->allocations, + sizeof(VulkanMemoryAllocation*) * allocator->allocationCount + ); + + allocator->allocations[ + allocator->allocationCount - 1 + ] = allocation; + if (dedicated) { dedicatedInfo.sType = @@ -2301,29 +2485,24 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( dedicatedInfo.image = image; allocInfo.pNext = &dedicatedInfo; - allocation->dedicated = 1; + allocation->availableForAllocation = 0; } else { allocInfo.pNext = NULL; - - allocator->allocationCount += 1; - allocator->allocations = SDL_realloc( - allocator->allocations, - sizeof(VulkanMemoryAllocation*) * allocator->allocationCount - ); - - allocator->allocations[ - allocator->allocationCount - 1 - ] = allocation; - allocation->dedicated = 0; + allocation->availableForAllocation = 1; } + allocation->usedRegions = SDL_malloc(sizeof(VulkanMemoryUsedRegion*)); + allocation->usedRegionCount = 0; + allocation->usedRegionCapacity = 1; + allocation->freeRegions = SDL_malloc(sizeof(VulkanMemoryFreeRegion*)); allocation->freeRegionCount = 0; allocation->freeRegionCapacity = 1; + allocation->allocator = allocator; result = renderer->vkAllocateMemory( @@ -2346,7 +2525,7 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( SDL_free(allocation); - LogVulkanResultAsWarn("vkAllocateMemory", result); + FNA3D_LogWarn("vkAllocateMemory: %s", VkErrorMessages(result)); return 0; } @@ -2357,16 +2536,11 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( renderer->logicalDevice, allocation->memory, 0, - allocation->size, + VK_WHOLE_SIZE, 0, (void**) &allocation->mapPointer ); - - if (result != VK_SUCCESS) - { - LogVulkanResultAsError("vkMapMemory", result); - return 0; - } + VULKAN_ERROR_CHECK(result, vkMapMemory, 0) } else { @@ -2374,6 +2548,7 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( } VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, allocation, 0, allocation->size @@ -2383,40 +2558,101 @@ static uint8_t VULKAN_INTERNAL_AllocateMemory( return 1; } -static uint8_t VULKAN_INTERNAL_FindAvailableMemory( +static uint8_t VULKAN_INTERNAL_BindBufferMemory( VulkanRenderer *renderer, + VulkanMemoryUsedRegion *usedRegion, + VkDeviceSize alignedOffset, + VkBuffer buffer +) { + VkResult vulkanResult; + + SDL_LockMutex(usedRegion->allocation->memoryLock); + + vulkanResult = renderer->vkBindBufferMemory( + renderer->logicalDevice, + buffer, + usedRegion->allocation->memory, + alignedOffset + ); + + SDL_UnlockMutex(usedRegion->allocation->memoryLock); + + VULKAN_ERROR_CHECK(vulkanResult, vkBindBufferMemory, 0) + + return 1; +} + +static uint8_t VULKAN_INTERNAL_BindImageMemory( + VulkanRenderer *renderer, + VulkanMemoryUsedRegion *usedRegion, + VkDeviceSize alignedOffset, + VkImage image +) { + VkResult vulkanResult; + + SDL_LockMutex(usedRegion->allocation->memoryLock); + + vulkanResult = renderer->vkBindImageMemory( + renderer->logicalDevice, + image, + usedRegion->allocation->memory, + alignedOffset + ); + + SDL_UnlockMutex(usedRegion->allocation->memoryLock); + + VULKAN_ERROR_CHECK(vulkanResult, vkBindBufferMemory, 0) + + return 1; +} + +static uint8_t VULKAN_INTERNAL_BindResourceMemory( + VulkanRenderer* renderer, uint32_t memoryTypeIndex, - VkMemoryRequirements2KHR *memoryRequirements, - VkMemoryDedicatedRequirementsKHR *dedicatedRequirements, + VkMemoryRequirements2KHR* memoryRequirements, + VkMemoryDedicatedRequirementsKHR* dedicatedRequirements, + uint8_t forceDedicated, + VkDeviceSize resourceSize, /* may be different from requirements size! */ VkBuffer buffer, /* may be VK_NULL_HANDLE */ VkImage image, /* may be VK_NULL_HANDLE */ - VulkanMemoryAllocation **pMemoryAllocation, - VkDeviceSize *pOffset, - VkDeviceSize *pSize + VulkanMemoryUsedRegion** pMemoryUsedRegion ) { VulkanMemoryAllocation *allocation; VulkanMemorySubAllocator *allocator; VulkanMemoryFreeRegion *region; + VulkanMemoryUsedRegion *usedRegion; VkDeviceSize requiredSize, allocationSize; VkDeviceSize alignedOffset; uint32_t newRegionSize, newRegionOffset; uint8_t shouldAllocDedicated = + forceDedicated || dedicatedRequirements->prefersDedicatedAllocation || dedicatedRequirements->requiresDedicatedAllocation; - uint8_t isHostVisible, allocationResult; + uint8_t isDeviceLocal, isHostVisible, allocationResult; isHostVisible = (renderer->memoryProperties.memoryTypes[memoryTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0; + isDeviceLocal = + (renderer->memoryProperties.memoryTypes[memoryTypeIndex].propertyFlags & + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0; + allocator = &renderer->memoryAllocator->subAllocators[memoryTypeIndex]; requiredSize = memoryRequirements->memoryRequirements.size; + if ( (buffer == VK_NULL_HANDLE && image == VK_NULL_HANDLE) || + (buffer != VK_NULL_HANDLE && image != VK_NULL_HANDLE) ) + { + FNA3D_LogError("BindResourceMemory must be given either a VulkanBuffer or a VulkanTexture"); + return 0; + } + SDL_LockMutex(renderer->allocatorLock); /* find the largest free region and use it */ - if (allocator->sortedFreeRegionCount > 0) + if (!shouldAllocDedicated && allocator->sortedFreeRegionCount > 0) { region = allocator->sortedFreeRegions[0]; allocation = region->allocation; @@ -2428,31 +2664,29 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( if (alignedOffset + requiredSize <= region->offset + region->size) { - *pMemoryAllocation = allocation; + usedRegion = VULKAN_INTERNAL_NewMemoryUsedRegion( + renderer, + allocation, + region->offset, + requiredSize + (alignedOffset - region->offset), + alignedOffset, + resourceSize, + memoryRequirements->memoryRequirements.alignment + ); - /* not aligned - create a new free region */ - if (region->offset != alignedOffset) - { - VULKAN_INTERNAL_NewMemoryFreeRegion( - allocation, - region->offset, - alignedOffset - region->offset - ); - } - - *pOffset = alignedOffset; - *pSize = requiredSize; + usedRegion->isBuffer = buffer != VK_NULL_HANDLE; newRegionSize = region->size - ((alignedOffset - region->offset) + requiredSize); newRegionOffset = alignedOffset + requiredSize; /* remove and add modified region to re-sort */ - VULKAN_INTERNAL_RemoveMemoryFreeRegion(region); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, region); /* if size is 0, no need to re-insert */ if (newRegionSize != 0) { VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, allocation, newRegionOffset, newRegionSize @@ -2461,6 +2695,40 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( SDL_UnlockMutex(renderer->allocatorLock); + if (buffer != VK_NULL_HANDLE) + { + if (!VULKAN_INTERNAL_BindBufferMemory( + renderer, + usedRegion, + alignedOffset, + buffer + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); + + return 0; + } + } + else if (image != VK_NULL_HANDLE) + { + if (!VULKAN_INTERNAL_BindImageMemory( + renderer, + usedRegion, + alignedOffset, + image + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); + + return 0; + } + } + + *pMemoryUsedRegion = usedRegion; return 1; } } @@ -2473,14 +2741,13 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( } else if (requiredSize > allocator->nextAllocationSize) { - /* allocate a page of required size aligned to STARTING_ALLOCATION_SIZE increments */ + /* allocate a page of required size aligned to ALLOCATION_INCREMENT increments */ allocationSize = - VULKAN_INTERNAL_NextHighestAlignment(requiredSize, STARTING_ALLOCATION_SIZE); + VULKAN_INTERNAL_NextHighestAlignment(requiredSize, ALLOCATION_INCREMENT); } else { allocationSize = allocator->nextAllocationSize; - allocator->nextAllocationSize = SDL_min(allocator->nextAllocationSize * 2, MAX_ALLOCATION_SIZE); } allocationResult = VULKAN_INTERNAL_AllocateMemory( @@ -2500,24 +2767,33 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( SDL_UnlockMutex(renderer->allocatorLock); /* Responsibility of the caller to handle being out of memory */ - Refresh_LogWarn("Failed to allocate memory!"); + FNA3D_LogWarn("Failed to allocate memory!"); return 2; } - *pMemoryAllocation = allocation; - *pOffset = 0; - *pSize = requiredSize; + usedRegion = VULKAN_INTERNAL_NewMemoryUsedRegion( + renderer, + allocation, + 0, + requiredSize, + 0, + resourceSize, + memoryRequirements->memoryRequirements.alignment + ); + + usedRegion->isBuffer = buffer != VK_NULL_HANDLE; region = allocation->freeRegions[0]; newRegionOffset = region->offset + requiredSize; newRegionSize = region->size - requiredSize; - VULKAN_INTERNAL_RemoveMemoryFreeRegion(region); + VULKAN_INTERNAL_RemoveMemoryFreeRegion(renderer, region); if (newRegionSize != 0) { VULKAN_INTERNAL_NewMemoryFreeRegion( + renderer, allocation, newRegionOffset, newRegionSize @@ -2526,68 +2802,50 @@ static uint8_t VULKAN_INTERNAL_FindAvailableMemory( SDL_UnlockMutex(renderer->allocatorLock); - return 1; -} - -static uint8_t VULKAN_INTERNAL_FindAvailableBufferMemory( - VulkanRenderer *renderer, - VkBuffer buffer, - VulkanMemoryAllocation **pMemoryAllocation, - VkDeviceSize *pOffset, - VkDeviceSize *pSize -) { - uint32_t memoryTypeIndex = 0; - VkMemoryDedicatedRequirementsKHR dedicatedRequirements = + if (buffer != VK_NULL_HANDLE) { - VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR, - NULL - }; - VkMemoryRequirements2KHR memoryRequirements = - { - VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR, - &dedicatedRequirements - }; - uint8_t findResult = 0; - - while (VULKAN_INTERNAL_FindBufferMemoryRequirements( - renderer, - buffer, - &memoryRequirements, - &memoryTypeIndex - )) { - findResult = VULKAN_INTERNAL_FindAvailableMemory( + if (!VULKAN_INTERNAL_BindBufferMemory( renderer, - memoryTypeIndex, - &memoryRequirements, - &dedicatedRequirements, - buffer, - VK_NULL_HANDLE, - pMemoryAllocation, - pOffset, - pSize - ); + usedRegion, + 0, + buffer + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); - if (findResult == 1) - { - break; + return 0; } - else - { - memoryTypeIndex += 1; + } + else if (image != VK_NULL_HANDLE) + { + if (!VULKAN_INTERNAL_BindImageMemory( + renderer, + usedRegion, + 0, + image + )) { + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + usedRegion + ); + + return 0; } } - return findResult; + *pMemoryUsedRegion = usedRegion; + return 1; } -static uint8_t VULKAN_INTERNAL_FindAvailableTextureMemory( - VulkanRenderer *renderer, +static uint8_t VULKAN_INTERNAL_BindMemoryForImage( + VulkanRenderer* renderer, VkImage image, - uint8_t cpuAllocation, - VulkanMemoryAllocation **pMemoryAllocation, - VkDeviceSize *pOffset, - VkDeviceSize *pSize + uint8_t isRenderTarget, + VulkanMemoryUsedRegion** usedRegion ) { + uint8_t bindResult = 0; uint32_t memoryTypeIndex = 0; VkMemoryPropertyFlags requiredMemoryPropertyFlags; VkMemoryPropertyFlags ignoredMemoryPropertyFlags; @@ -2601,18 +2859,10 @@ static uint8_t VULKAN_INTERNAL_FindAvailableTextureMemory( VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR, &dedicatedRequirements }; - uint8_t findResult = 0; - if (cpuAllocation) - { - requiredMemoryPropertyFlags = 0; - ignoredMemoryPropertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - } - else - { - requiredMemoryPropertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; - ignoredMemoryPropertyFlags = 0; - } + /* Prefer GPU allocation */ + requiredMemoryPropertyFlags = VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + ignoredMemoryPropertyFlags = 0; while (VULKAN_INTERNAL_FindImageMemoryRequirements( renderer, @@ -2622,29 +2872,599 @@ static uint8_t VULKAN_INTERNAL_FindAvailableTextureMemory( &memoryRequirements, &memoryTypeIndex )) { - findResult = VULKAN_INTERNAL_FindAvailableMemory( + bindResult = VULKAN_INTERNAL_BindResourceMemory( renderer, memoryTypeIndex, &memoryRequirements, &dedicatedRequirements, + isRenderTarget, + memoryRequirements.memoryRequirements.size, VK_NULL_HANDLE, image, - pMemoryAllocation, - pOffset, - pSize + usedRegion ); - if (findResult == 1) + if (bindResult == 1) { break; } - else + else /* Bind failed, try the next device-local heap */ { memoryTypeIndex += 1; } } - return findResult; + /* Bind _still_ failed, try again without device local */ + if (bindResult != 1) + { + memoryTypeIndex = 0; + requiredMemoryPropertyFlags = 0; + ignoredMemoryPropertyFlags = VK_MEMORY_HEAP_DEVICE_LOCAL_BIT; + + if (isRenderTarget) + { + FNA3D_LogWarn("RenderTarget is allocated in host memory, pre-allocate your targets!"); + } + + FNA3D_LogWarn("Out of device local memory, falling back to host memory"); + + while (VULKAN_INTERNAL_FindImageMemoryRequirements( + renderer, + image, + requiredMemoryPropertyFlags, + ignoredMemoryPropertyFlags, + &memoryRequirements, + &memoryTypeIndex + )) { + bindResult = VULKAN_INTERNAL_BindResourceMemory( + renderer, + memoryTypeIndex, + &memoryRequirements, + &dedicatedRequirements, + isRenderTarget, + memoryRequirements.memoryRequirements.size, + VK_NULL_HANDLE, + image, + usedRegion + ); + + if (bindResult == 1) + { + break; + } + else /* Bind failed, try the next heap */ + { + memoryTypeIndex += 1; + } + } + } + + return bindResult; +} + +static uint8_t VULKAN_INTERNAL_BindMemoryForBuffer( + VulkanRenderer* renderer, + VkBuffer buffer, + VkDeviceSize size, + uint8_t preferDeviceLocal, + uint8_t isTransferBuffer, + VulkanMemoryUsedRegion** usedRegion +) { + uint8_t bindResult = 0; + uint32_t memoryTypeIndex = 0; + VkMemoryPropertyFlags requiredMemoryPropertyFlags; + VkMemoryPropertyFlags ignoredMemoryPropertyFlags; + VkMemoryDedicatedRequirementsKHR dedicatedRequirements = + { + VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR, + NULL + }; + VkMemoryRequirements2KHR memoryRequirements = + { + VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR, + &dedicatedRequirements + }; + + requiredMemoryPropertyFlags = + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + + if (preferDeviceLocal) + { + requiredMemoryPropertyFlags |= + VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT; + } + ignoredMemoryPropertyFlags = 0; + + while (VULKAN_INTERNAL_FindBufferMemoryRequirements( + renderer, + buffer, + requiredMemoryPropertyFlags, + ignoredMemoryPropertyFlags, + &memoryRequirements, + &memoryTypeIndex + )) { + bindResult = VULKAN_INTERNAL_BindResourceMemory( + renderer, + memoryTypeIndex, + &memoryRequirements, + &dedicatedRequirements, + isTransferBuffer, + size, + buffer, + VK_NULL_HANDLE, + usedRegion + ); + + if (bindResult == 1) + { + break; + } + else /* Bind failed, try the next device-local heap */ + { + memoryTypeIndex += 1; + } + } + + /* Bind failed, try again if originally preferred device local */ + if (bindResult != 1 && preferDeviceLocal) + { + memoryTypeIndex = 0; + requiredMemoryPropertyFlags = + VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | + VK_MEMORY_PROPERTY_HOST_COHERENT_BIT; + + /* Follow-up for the warning logged by FindMemoryType */ + if (!renderer->unifiedMemoryWarning) + { + FNA3D_LogWarn("No unified memory found, falling back to host memory"); + renderer->unifiedMemoryWarning = 1; + } + + while (VULKAN_INTERNAL_FindBufferMemoryRequirements( + renderer, + buffer, + requiredMemoryPropertyFlags, + ignoredMemoryPropertyFlags, + &memoryRequirements, + &memoryTypeIndex + )) { + bindResult = VULKAN_INTERNAL_BindResourceMemory( + renderer, + memoryTypeIndex, + &memoryRequirements, + &dedicatedRequirements, + isTransferBuffer, + size, + buffer, + VK_NULL_HANDLE, + usedRegion + ); + + if (bindResult == 1) + { + break; + } + else /* Bind failed, try the next heap */ + { + memoryTypeIndex += 1; + } + } + } + + return bindResult; +} + +static uint8_t VULKAN_INTERNAL_FindAllocationToDefragment( + VulkanRenderer *renderer, + VulkanMemorySubAllocator *allocator, + uint32_t *allocationIndexToDefrag +) { + uint32_t i, j; + + for (i = 0; i < VK_MAX_MEMORY_TYPES; i += 1) + { + *allocator = renderer->memoryAllocator->subAllocators[i]; + + for (j = 0; j < allocator->allocationCount; j += 1) + { + if (allocator->allocations[j]->freeRegionCount > 1) + { + *allocationIndexToDefrag = j; + return 1; + } + } + } + + return 0; +} + +static uint8_t VULKAN_INTERNAL_DefragmentMemory( + VulkanRenderer *renderer +) { + VulkanMemorySubAllocator allocator; + VulkanMemoryAllocation *allocation; + uint32_t allocationIndexToDefrag; + VulkanMemoryUsedRegion *currentRegion; + VulkanMemoryUsedRegion *newRegion; + VkBuffer copyBuffer; + VkBufferCopy bufferCopy; + VkImage copyImage; + VkImageCopy *imageCopyRegions; + VkImageAspectFlags aspectFlags; + VulkanCommandBuffer *commandBuffer; + VkCommandBufferBeginInfo beginInfo; + VkPipelineStageFlags waitFlags = VK_PIPELINE_STAGE_ALL_COMMANDS_BIT; + VkSubmitInfo submitInfo; + VkResult result; + uint32_t i, level; + VulkanResourceAccessType copyResourceAccessType = RESOURCE_ACCESS_NONE; + VulkanResourceAccessType originalResourceAccessType; + + SDL_LockMutex(renderer->commandLock); + SDL_LockMutex(renderer->allocatorLock); + + renderer->needDefrag = 0; + + beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + beginInfo.pNext = NULL; + beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT; + beginInfo.pInheritanceInfo = NULL; + + commandBuffer = VULKAN_AcquireCommandBuffer((Refresh_Renderer *) renderer); + + if (VULKAN_INTERNAL_FindAllocationToDefragment( + renderer, + &allocator, + &allocationIndexToDefrag + )) { + allocation = allocator.allocations[allocationIndexToDefrag]; + + VULKAN_INTERNAL_MakeMemoryUnavailable( + renderer, + allocation + ); + + for (i = 0; i < allocation->usedRegionCount; i += 1) + { + currentRegion = allocation->usedRegions[i]; + copyResourceAccessType = RESOURCE_ACCESS_NONE; + + if (currentRegion->isBuffer) + { + currentRegion->vulkanBuffer->bufferCreateInfo.usage |= VK_BUFFER_USAGE_TRANSFER_DST_BIT; + + copyBuffer = VULKAN_INTERNAL_CreateBuffer( + renderer, + currentRegion->vulkanBuffer->bufferCreateInfo.size, + RESOURCE_ACCESS_NONE, + currentRegion->vulkanBuffer->bufferCreateInfo.usage | VK_BUFFER_USAGE_TRANSFER_DST_BIT + ); + + result = renderer->vkCreateBuffer( + renderer->logicalDevice, + ¤tRegion->vulkanBuffer->bufferCreateInfo, + NULL, + ©Buffer + ); + VULKAN_ERROR_CHECK(result, vkCreateBuffer, 0) + + if ( + VULKAN_INTERNAL_BindMemoryForBuffer( + renderer, + copyBuffer, + currentRegion->resourceSize, + currentRegion->vulkanBuffer->preferDeviceLocal, + 0, + &newRegion + ) != 1) + { + /* Out of memory, abort */ + renderer->vkDestroyBuffer( + renderer->logicalDevice, + copyBuffer, + NULL + ); + break; + } + + originalResourceAccessType = currentRegion->vulkanBuffer->resourceAccessType; + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + RESOURCE_ACCESS_TRANSFER_READ, + currentRegion->vulkanBuffer->buffer, + ¤tRegion->vulkanBuffer->resourceAccessType + ); + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + RESOURCE_ACCESS_TRANSFER_WRITE, + copyBuffer, + ©ResourceAccessType + ); + + bufferCopy.srcOffset = 0; + bufferCopy.dstOffset = 0; + bufferCopy.size = currentRegion->resourceSize; + + renderer->vkCmdCopyBuffer( + commandBuffer->commandBuffer, + currentRegion->vulkanBuffer->buffer, + copyBuffer, + 1, + &bufferCopy + ); + + VULKAN_INTERNAL_BufferMemoryBarrier( + renderer, + originalResourceAccessType, + copyBuffer, + ©ResourceAccessType + ); + + if (renderer->defragmentedBuffersToDestroyCount >= renderer->defragmentedBuffersToDestroyCapacity) + { + renderer->defragmentedBuffersToDestroyCapacity *= 2; + renderer->defragmentedBuffersToDestroy = SDL_realloc( + renderer->defragmentedBuffersToDestroy, + sizeof(VkBuffer) * renderer->defragmentedBuffersToDestroyCapacity + ); + } + + if (renderer->usedRegionsToDestroyCount >= renderer->usedRegionsToDestroyCapacity) + { + renderer->usedRegionsToDestroyCapacity *= 2; + renderer->usedRegionsToDestroy = SDL_realloc( + renderer->usedRegionsToDestroy, + sizeof(VulkanMemoryUsedRegion*) * renderer->usedRegionsToDestroyCapacity + ); + } + + renderer->defragmentedBuffersToDestroy[ + renderer->defragmentedBuffersToDestroyCount + ] = currentRegion->vulkanBuffer->buffer; + + renderer->defragmentedBuffersToDestroyCount += 1; + + renderer->usedRegionsToDestroy[ + renderer->usedRegionsToDestroyCount + ] = currentRegion; + + renderer->usedRegionsToDestroyCount += 1; + + newRegion->isBuffer = 1; + newRegion->vulkanBuffer = currentRegion->vulkanBuffer; + newRegion->vulkanBuffer->usedRegion = newRegion; /* lol */ + newRegion->vulkanBuffer->buffer = copyBuffer; + newRegion->vulkanBuffer->resourceAccessType = copyResourceAccessType; + + renderer->needDefrag = 1; + } + else + { + result = renderer->vkCreateImage( + renderer->logicalDevice, + ¤tRegion->vulkanTexture->imageCreateInfo, + NULL, + ©Image + ); + + VULKAN_ERROR_CHECK(result, vkCreateImage, 0) + + if (VULKAN_INTERNAL_BindMemoryForImage( + renderer, + copyImage, + 0, + &newRegion + ) != 1) + { + /* Out of memory, abort */ + renderer->vkDestroyImage( + renderer->logicalDevice, + copyImage, + NULL + ); + + break; + } + + if (IsDepthFormat(currentRegion->vulkanTexture->surfaceFormat)) + { + aspectFlags = VK_IMAGE_ASPECT_DEPTH_BIT; + + if (DepthFormatContainsStencil(currentRegion->vulkanTexture->surfaceFormat)) + { + aspectFlags |= VK_IMAGE_ASPECT_STENCIL_BIT; + } + } + else + { + aspectFlags = VK_IMAGE_ASPECT_COLOR_BIT; + } + + originalResourceAccessType = currentRegion->vulkanTexture->resourceAccessType; + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_READ, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + currentRegion->vulkanTexture->image, + ¤tRegion->vulkanTexture->resourceAccessType + ); + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + commandBuffer->commandBuffer, + RESOURCE_ACCESS_TRANSFER_WRITE, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + copyImage, + ©ResourceAccessType + ); + + imageCopyRegions = SDL_stack_alloc(VkImageCopy, currentRegion->vulkanTexture->levelCount); + + for (level = 0; level < currentRegion->vulkanTexture->levelCount; level += 1) + { + imageCopyRegions[level].srcOffset.x = 0; + imageCopyRegions[level].srcOffset.y = 0; + imageCopyRegions[level].srcOffset.z = 0; + imageCopyRegions[level].srcSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].srcSubresource.baseArrayLayer = 0; + imageCopyRegions[level].srcSubresource.layerCount = currentRegion->vulkanTexture->layerCount; + imageCopyRegions[level].srcSubresource.mipLevel = level; + imageCopyRegions[level].extent.width = SDL_max(1, currentRegion->vulkanTexture->dimensions.width >> level); + imageCopyRegions[level].extent.height = SDL_max(1, currentRegion->vulkanTexture->dimensions.height >> level); + imageCopyRegions[level].extent.depth = currentRegion->vulkanTexture->depth; + imageCopyRegions[level].dstOffset.x = 0; + imageCopyRegions[level].dstOffset.y = 0; + imageCopyRegions[level].dstOffset.z = 0; + imageCopyRegions[level].dstSubresource.aspectMask = aspectFlags; + imageCopyRegions[level].dstSubresource.baseArrayLayer = 0; + imageCopyRegions[level].dstSubresource.layerCount = currentRegion->vulkanTexture->layerCount; + imageCopyRegions[level].dstSubresource.mipLevel = level; + } + + renderer->vkCmdCopyImage( + commandBuffer->commandBuffer, + currentRegion->vulkanTexture->image, + AccessMap[currentRegion->vulkanTexture->resourceAccessType].imageLayout, + copyImage, + AccessMap[copyResourceAccessType].imageLayout, + currentRegion->vulkanTexture->levelCount, + imageCopyRegions + ); + + VULKAN_INTERNAL_ImageMemoryBarrier( + renderer, + originalResourceAccessType, + aspectFlags, + 0, + currentRegion->vulkanTexture->layerCount, + 0, + currentRegion->vulkanTexture->levelCount, + 0, + copyImage, + ©ResourceAccessType + ); + + SDL_stack_free(imageCopyRegions); + + if (renderer->defragmentedImagesToDestroyCount >= renderer->defragmentedImagesToDestroyCapacity) + { + renderer->defragmentedImagesToDestroyCapacity *= 2; + renderer->defragmentedImagesToDestroy = SDL_realloc( + renderer->defragmentedImagesToDestroy, + sizeof(VkImage) * renderer->defragmentedImagesToDestroyCapacity + ); + } + + if (renderer->defragmentedImageViewsToDestroyCount >= renderer->defragmentedImageViewsToDestroyCapacity) + { + renderer->defragmentedImageViewsToDestroyCapacity *= 2; + renderer->defragmentedImageViewsToDestroy = SDL_realloc( + renderer->defragmentedImageViewsToDestroy, + sizeof(VkImageView) * renderer->defragmentedImageViewsToDestroyCapacity + ); + } + + if (renderer->usedRegionsToDestroyCount >= renderer->usedRegionsToDestroyCapacity) + { + renderer->usedRegionsToDestroyCapacity *= 2; + renderer->usedRegionsToDestroy = SDL_realloc( + renderer->usedRegionsToDestroy, + sizeof(VulkanMemoryUsedRegion*) * renderer->usedRegionsToDestroyCapacity + ); + } + + renderer->defragmentedImagesToDestroy[ + renderer->defragmentedImagesToDestroyCount + ] = currentRegion->vulkanTexture->image; + + renderer->defragmentedImagesToDestroyCount += 1; + + renderer->defragmentedImageViewsToDestroy[ + renderer->defragmentedImageViewsToDestroyCount + ] = currentRegion->vulkanTexture->view; + + renderer->defragmentedImageViewsToDestroyCount += 1; + + renderer->usedRegionsToDestroy[ + renderer->usedRegionsToDestroyCount + ] = currentRegion; + + renderer->usedRegionsToDestroyCount += 1; + + currentRegion->vulkanTexture->viewCreateInfo.image = copyImage; + + renderer->vkCreateImageView( + renderer->logicalDevice, + ¤tRegion->vulkanTexture->viewCreateInfo, + NULL, + ¤tRegion->vulkanTexture->view + ); + + newRegion->isBuffer = 0; + + newRegion->vulkanTexture = currentRegion->vulkanTexture; + newRegion->vulkanTexture->usedRegion = newRegion; /* lol */ + newRegion->vulkanTexture->image = copyImage; + newRegion->vulkanTexture->resourceAccessType = copyResourceAccessType; + + renderer->needDefrag = 1; + } + } + } + + renderer->vkEndCommandBuffer( + renderer->defragCommandBufferContainer->commandBuffer + ); + + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + submitInfo.pNext = NULL; + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &renderer->defragCommandBufferContainer->commandBuffer; + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = &renderer->defragSemaphore; + submitInfo.pWaitDstStageMask = &waitFlags; + submitInfo.signalSemaphoreCount = 0; + submitInfo.pSignalSemaphores = NULL; + + result = renderer->vkResetFences( + renderer->logicalDevice, + 1, + &renderer->defragCommandBufferContainer->inFlightFence + ); + VULKAN_ERROR_CHECK(result, vkResetFences, 0) + + result = renderer->vkQueueSubmit( + renderer->unifiedQueue, + 1, + &submitInfo, + renderer->defragCommandBufferContainer->inFlightFence + ); + VULKAN_ERROR_CHECK(result, vkQueueSubmit, 0) + + renderer->currentCommandBufferContainer = NULL; + renderer->numActiveCommands = 0; + + renderer->defragTimer = 0; + + SDL_UnlockMutex(renderer->commandLock); + SDL_UnlockMutex(renderer->allocatorLock); + + return 1; } /* Memory Barriers */ @@ -3023,31 +3843,6 @@ static void VULKAN_INTERNAL_DestroyTexture( VulkanRenderer* renderer, VulkanTexture* texture ) { - if (texture->allocation->dedicated) - { - renderer->vkFreeMemory( - renderer->logicalDevice, - texture->allocation->memory, - NULL - ); - - SDL_DestroyMutex(texture->allocation->memoryLock); - SDL_free(texture->allocation->freeRegions); - SDL_free(texture->allocation); - } - else - { - SDL_LockMutex(renderer->allocatorLock); - - VULKAN_INTERNAL_NewMemoryFreeRegion( - texture->allocation, - texture->offset, - texture->memorySize - ); - - SDL_UnlockMutex(renderer->allocatorLock); - } - VULKAN_INTERNAL_RemoveRenderTargetsContainingTexture( renderer, texture @@ -3065,6 +3860,11 @@ static void VULKAN_INTERNAL_DestroyTexture( NULL ); + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + texture->usedRegion + ); + /* destroy the msaa texture, if there is one */ if (texture->msaaTex != NULL) { @@ -3099,37 +3899,17 @@ static void VULKAN_INTERNAL_DestroyBuffer( VulkanRenderer* renderer, VulkanBuffer* buffer ) { - if (buffer->allocation->dedicated) - { - renderer->vkFreeMemory( - renderer->logicalDevice, - buffer->allocation->memory, - NULL - ); - - SDL_DestroyMutex(buffer->allocation->memoryLock); - SDL_free(buffer->allocation->freeRegions); - SDL_free(buffer->allocation); - } - else - { - SDL_LockMutex(renderer->allocatorLock); - - VULKAN_INTERNAL_NewMemoryFreeRegion( - buffer->allocation, - buffer->offset, - buffer->memorySize - ); - - SDL_UnlockMutex(renderer->allocatorLock); - } - renderer->vkDestroyBuffer( renderer->logicalDevice, buffer->buffer, NULL ); + VULKAN_INTERNAL_RemoveMemoryUsedRegion( + renderer, + buffer->usedRegion + ); + SDL_free(buffer); } @@ -3667,7 +4447,7 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( VulkanBuffer* buffer; VkResult vulkanResult; VkBufferCreateInfo bufferCreateInfo; - uint8_t findMemoryResult; + uint8_t bindResult; buffer = SDL_malloc(sizeof(VulkanBuffer)); @@ -3690,51 +4470,30 @@ static VulkanBuffer* VULKAN_INTERNAL_CreateBuffer( NULL, &buffer->buffer ); + VULKAN_ERROR_CHECK(vulkanResult, vkCreateBuffer, 0) - if (vulkanResult != VK_SUCCESS) - { - SDL_free(buffer); - LogVulkanResultAsError("vkCreateBuffer", vulkanResult); - Refresh_LogError("Failed to create VkBuffer"); - return NULL; - } + buffer->bufferCreateInfo = bufferCreateInfo; - findMemoryResult = VULKAN_INTERNAL_FindAvailableBufferMemory( + bindResult = VULKAN_INTERNAL_BindMemoryForBuffer( renderer, buffer->buffer, - &buffer->allocation, - &buffer->offset, - &buffer->memorySize + buffer->size, + buffer->preferDeviceLocal, + buffer->isTransferBuffer, + &buffer->usedRegion ); - /* We're out of available memory */ - if (findMemoryResult == 2) + if (bindResult != 1) { - Refresh_LogWarn("Out of buffer memory!"); - return NULL; - } - else if (findMemoryResult == 0) - { - Refresh_LogError("Failed to find buffer memory!"); + renderer->vkDestroyBuffer( + renderer->logicalDevice, + buffer->buffer, + NULL); + return NULL; } - SDL_LockMutex(buffer->allocation->memoryLock); - - vulkanResult = renderer->vkBindBufferMemory( - renderer->logicalDevice, - buffer->buffer, - buffer->allocation->memory, - buffer->offset - ); - - SDL_UnlockMutex(buffer->allocation->memoryLock); - - if (vulkanResult != VK_SUCCESS) - { - Refresh_LogError("Failed to bind buffer memory!"); - return NULL; - } + buffer->usedRegion->vulkanBuffer = buffer; /* lol */ buffer->resourceAccessType = resourceAccessType; @@ -4726,15 +5485,6 @@ static void VULKAN_DestroyDevice( SDL_free(renderer->dummyFragmentUniformBuffer); SDL_free(renderer->dummyComputeUniformBuffer); - for (i = 0; i < renderer->transferBufferPool.availableBufferCount; i += 1) - { - VULKAN_INTERNAL_DestroyBuffer(renderer, renderer->transferBufferPool.availableBuffers[i]->buffer); - SDL_free(renderer->transferBufferPool.availableBuffers[i]); - } - - SDL_free(renderer->transferBufferPool.availableBuffers); - SDL_DestroyMutex(renderer->transferBufferPool.lock); - for (i = 0; i < NUM_COMMAND_POOL_BUCKETS; i += 1) { commandPoolHashArray = renderer->commandPoolHashTable.buckets[i]; @@ -5223,7 +5973,7 @@ static VulkanTexture* VULKAN_INTERNAL_CreateTexture( VkImageCreateInfo imageCreateInfo; VkImageCreateFlags imageCreateFlags = 0; VkImageViewCreateInfo imageViewCreateInfo; - uint8_t findMemoryResult; + uint8_t bindResult; uint8_t is3D = depth > 1 ? 1 : 0; uint8_t layerCount = isCube ? 6 : 1; uint8_t isRenderTarget = @@ -5271,73 +6021,29 @@ static VulkanTexture* VULKAN_INTERNAL_CreateTexture( NULL, &texture->image ); + VULKAN_ERROR_CHECK(vulkanResult, vkCreateImage, 0) - if (vulkanResult != VK_SUCCESS) - { - LogVulkanResultAsError("vkCreateImage", vulkanResult); - Refresh_LogError("Failed to create texture!"); - } + texture->isRenderTarget = isRenderTarget; + texture->imageCreateInfo = imageCreateInfo; - /* Prefer GPU allocation */ - findMemoryResult = VULKAN_INTERNAL_FindAvailableTextureMemory( + bindResult = VULKAN_INTERNAL_BindMemoryForImage( renderer, texture->image, - 0, - &texture->allocation, - &texture->offset, - &texture->memorySize + isRenderTarget, + &texture->usedRegion ); - /* No device local memory available */ - if (findMemoryResult == 2) + if (bindResult != 1) { - if (isRenderTarget) - { - Refresh_LogWarn("RenderTarget is allocated in host memory, pre-allocate your targets!"); - } - - Refresh_LogWarn("Out of device local memory, falling back to host memory"); - - /* Attempt CPU allocation */ - findMemoryResult = VULKAN_INTERNAL_FindAvailableTextureMemory( - renderer, + renderer->vkDestroyImage( + renderer->logicalDevice, texture->image, - 1, - &texture->allocation, - &texture->offset, - &texture->memorySize - ); + NULL); - /* Memory alloc completely failed, time to die */ - if (findMemoryResult == 0) - { - Refresh_LogError("Something went very wrong allocating memory!"); - return 0; - } - else if (findMemoryResult == 2) - { - Refresh_LogError("Out of memory!"); - return 0; - } + return bindResult; } - SDL_LockMutex(texture->allocation->memoryLock); - - vulkanResult = renderer->vkBindImageMemory( - renderer->logicalDevice, - texture->image, - texture->allocation->memory, - texture->offset - ); - - SDL_UnlockMutex(texture->allocation->memoryLock); - - if (vulkanResult != VK_SUCCESS) - { - LogVulkanResultAsError("vkBindImageMemory", vulkanResult); - Refresh_LogError("Failed to bind texture memory!"); - return 0; - } + texture->usedRegion->vulkanTexture = texture; /* lol */ imageViewCreateInfo.sType = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO; imageViewCreateInfo.pNext = NULL; @@ -6715,41 +7421,7 @@ static VulkanTransferBuffer* VULKAN_INTERNAL_AcquireTransferBuffer( } } - /* Nothing fits, so let's get a transfer buffer from the pool */ - - SDL_LockMutex(renderer->transferBufferPool.lock); - - for (i = 0; i < renderer->transferBufferPool.availableBufferCount; i += 1) - { - transferBuffer = renderer->transferBufferPool.availableBuffers[i]; - offset = transferBuffer->offset + alignment - (transferBuffer->offset % alignment); - - if (offset + requiredSize <= transferBuffer->buffer->size) - { - if (commandBuffer->transferBufferCount == commandBuffer->transferBufferCapacity) - { - commandBuffer->transferBufferCapacity *= 2; - commandBuffer->transferBuffers = SDL_realloc( - commandBuffer->transferBuffers, - commandBuffer->transferBufferCapacity * sizeof(VulkanTransferBuffer*) - ); - } - - commandBuffer->transferBuffers[commandBuffer->transferBufferCount] = transferBuffer; - commandBuffer->transferBufferCount += 1; - - renderer->transferBufferPool.availableBuffers[i] = renderer->transferBufferPool.availableBuffers[renderer->transferBufferPool.availableBufferCount - 1]; - renderer->transferBufferPool.availableBufferCount -= 1; - SDL_UnlockMutex(renderer->transferBufferPool.lock); - - transferBuffer->offset = offset; - return transferBuffer; - } - } - - SDL_UnlockMutex(renderer->transferBufferPool.lock); - - /* Nothing fits still, so let's create a new transfer buffer */ + /* Nothing fits, so let's create a new transfer buffer */ size = TRANSFER_BUFFER_STARTING_SIZE; @@ -9663,26 +10335,13 @@ static void VULKAN_INTERNAL_CleanCommandBuffer( commandBuffer->boundUniformBufferCount = 0; - SDL_LockMutex(renderer->transferBufferPool.lock); - - if (renderer->transferBufferPool.availableBufferCount + commandBuffer->transferBufferCount >= renderer->transferBufferPool.availableBufferCapacity) - { - renderer->transferBufferPool.availableBufferCapacity = renderer->transferBufferPool.availableBufferCount + commandBuffer->transferBufferCount; - renderer->transferBufferPool.availableBuffers = SDL_realloc( - renderer->transferBufferPool.availableBuffers, - renderer->transferBufferPool.availableBufferCapacity * sizeof(VulkanTransferBuffer*) - ); - } - for (i = 0; i < commandBuffer->transferBufferCount; i += 1) { - commandBuffer->transferBuffers[i]->offset = 0; - renderer->transferBufferPool.availableBuffers[renderer->transferBufferPool.availableBufferCount] = commandBuffer->transferBuffers[i]; - renderer->transferBufferPool.availableBufferCount += 1; + VULKAN_INTERNAL_DestroyBuffer(renderer, commandBuffer->transferBuffers[i]->buffer); + SDL_free(commandBuffer->transferBuffers[i]); + commandBuffer->transferBuffers[i] = NULL; } - SDL_UnlockMutex(renderer->transferBufferPool.lock); - commandBuffer->transferBufferCount = 0; /* Bound descriptor sets are now available */ @@ -10350,6 +11009,9 @@ static uint8_t VULKAN_INTERNAL_IsDeviceSuitable( static void VULKAN_INTERNAL_GetPhysicalDeviceProperties( VulkanRenderer *renderer ) { + VkDeviceSize deviceLocalHeapSize; + int32_t i; + renderer->physicalDeviceDriverProperties.sType = VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_DRIVER_PROPERTIES_KHR; renderer->physicalDeviceDriverProperties.pNext = NULL; @@ -10368,6 +11030,21 @@ static void VULKAN_INTERNAL_GetPhysicalDeviceProperties( renderer->physicalDevice, &renderer->memoryProperties ); + + deviceLocalHeapSize = 0; + for (i = 0; i < renderer->memoryProperties.memoryHeapCount; i += 1) + { + if ( renderer->memoryProperties.memoryHeaps[i].flags & + VK_MEMORY_HEAP_DEVICE_LOCAL_BIT ) + { + if (renderer->memoryProperties.memoryHeaps[i].size > deviceLocalHeapSize) + { + deviceLocalHeapSize = renderer->memoryProperties.memoryHeaps[i].size; + } + } + } + + renderer->maxDeviceLocalHeapUsage = deviceLocalHeapSize; } static uint8_t VULKAN_INTERNAL_DeterminePhysicalDevice( @@ -10744,6 +11421,7 @@ static Refresh_Device* VULKAN_CreateDevice( /* Variables: Image Format Detection */ VkImageFormatProperties imageFormatProperties; + SDL_memset(renderer, '\0', sizeof(VulkanRenderer)); renderer->debugMode = debugMode; if (!VULKAN_INTERNAL_PrepareVulkan(renderer)) @@ -10828,6 +11506,7 @@ static Refresh_Device* VULKAN_CreateDevice( for (i = 0; i < VK_MAX_MEMORY_TYPES; i += 1) { + renderer->memoryAllocator->subAllocators[i].memoryTypeIndex = i; renderer->memoryAllocator->subAllocators[i].nextAllocationSize = STARTING_ALLOCATION_SIZE; renderer->memoryAllocator->subAllocators[i].allocations = NULL; renderer->memoryAllocator->subAllocators[i].allocationCount = 0; @@ -11124,14 +11803,6 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->renderTargetHashArray.count = 0; renderer->renderTargetHashArray.capacity = 0; - /* Initialize transfer buffer pool */ - - renderer->transferBufferPool.lock = SDL_CreateMutex(); - - renderer->transferBufferPool.availableBufferCapacity = 4; - renderer->transferBufferPool.availableBufferCount = 0; - renderer->transferBufferPool.availableBuffers = SDL_malloc(renderer->transferBufferPool.availableBufferCapacity * sizeof(VulkanTransferBuffer*)); - /* Some drivers don't support D16, so we have to fall back to D32. */ vulkanResult = renderer->vkGetPhysicalDeviceImageFormatProperties( @@ -11229,6 +11900,10 @@ static Refresh_Device* VULKAN_CreateDevice( renderer->framebuffersToDestroyCapacity ); + renderer->needDefrag = 0; + renderer->defragTimer = 0; + renderer->resourceFreed = 0; + return result; }