Vulkan: Isolate oversized memory pages from shared allocations

Graphics/GraphicsEngine/interface/GraphicsTypes.h

@@ -4109,12 +4109,17 @@ struct EngineVkCreateInfo DILIGENT_DERIVE(EngineCreateInfo)
     /// Device-local memory is used for USAGE_DEFAULT and USAGE_IMMUTABLE
     /// GPU resources, such as buffers and textures.
     ///
+    /// Allocations larger than this size use separate pages that are not shared
+    /// with other resources and are released independently of the memory reserve.
+    ///
     /// If there is no available GPU memory, the resource will fail to be created.
     Uint32 DeviceLocalMemoryPageSize        DEFAULT_INITIALIZER(16 << 20);
 
     /// Allocation granularity for host-visible memory.
 
     /// Host-visible memory is primarily used to upload data to GPU resources.
+    /// Allocations larger than this size use separate pages that are not shared
+    /// with other resources and are released independently of the memory reserve.
     Uint32 HostVisibleMemoryPageSize        DEFAULT_INITIALIZER(16 << 20);
 
     /// Amount of device-local memory reserved by the engine.

Graphics/GraphicsEngineVulkan/include/VulkanUtilities/MemoryManager.hpp

@@ -145,7 +145,7 @@ class MemoryPage
     friend struct MemoryAllocation;
 
     // MemoryManager holds m_PagesMtx while allocating so ShrinkMemory() cannot erase this page.
-    MemoryAllocation Allocate(VkDeviceSize size, VkDeviceSize alignment);
+    MemoryAllocation Allocate(VkDeviceSize size, VkDeviceSize alignment, bool RequireEmpty = false);
 
     // Memory is reclaimed immediately. The application is responsible to ensure it is not in use by the GPU
     void Free(MemoryAllocation&& Allocation);

Graphics/GraphicsEngineVulkan/src/VulkanUtilities/MemoryManager.cpp

@@ -96,9 +96,11 @@ MemoryPage::~MemoryPage()
     VERIFY(IsEmpty(), "Destroying a page with not all allocations released");
 }
 
-MemoryAllocation MemoryPage::Allocate(VkDeviceSize size, VkDeviceSize alignment)
+MemoryAllocation MemoryPage::Allocate(VkDeviceSize size, VkDeviceSize alignment, bool RequireEmpty)
 {
     std::lock_guard<std::mutex> Lock{m_Mutex};
+    if (RequireEmpty && !m_AllocationMgr.IsEmpty())
+        return MemoryAllocation{};
     VERIFY(size <= std::numeric_limits<AllocationsMgrOffsetType>::max(),
            "Allocation size (", size, ") exceeds maximum allowed value ",
            std::numeric_limits<AllocationsMgrOffsetType>::max());
@@ -161,7 +163,15 @@ MemoryAllocation MemoryManager::Allocate(const VkMemoryRequirements& MemReqs, Vk
 
 MemoryAllocation MemoryManager::Allocate(VkDeviceSize Size, VkDeviceSize Alignment, uint32_t MemoryTypeIndex, bool HostVisible, VkMemoryAllocateFlags AllocateFlags)
 {
-    MemoryAllocation Allocation;
+    // The suballocator doubles its alignment while initializing a page. Keep sizes below
+    // half its offset range to avoid overflow there as well as in alignment arithmetic.
+    constexpr VkDeviceSize MaxSize = std::numeric_limits<Diligent::VariableSizeAllocationsManager::OffsetType>::max() / 2;
+    if (Size == 0 || Alignment == 0 || (Alignment & (Alignment - 1)) != 0 ||
+        Alignment > MaxSize || Size > MaxSize - (Alignment - 1))
+        LOG_ERROR_AND_THROW("Invalid Vulkan memory allocation size or alignment");
+
+    const VkDeviceSize AlignedSize = Diligent::AlignUp(Size, Alignment);
+    MemoryAllocation   Allocation;
 
     // On integrated GPUs, there is no difference between host-visible and GPU-only
     // memory, so MemoryTypeIndex is the same. As GPU-only pages do not have CPU address,
@@ -173,25 +183,54 @@ MemoryAllocation MemoryManager::Allocate(VkDeviceSize Size, VkDeviceSize Alignme
     MemoryPageIndex             PageIdx{MemoryTypeIndex, HostVisible, AllocateFlags};
     std::lock_guard<std::mutex> Lock{m_PagesMtx};
 
-    auto range = m_Pages.equal_range(PageIdx);
+    const VkDeviceSize BasePageSize = HostVisible ? m_HostVisiblePageSize : m_DeviceLocalPageSize;
+
+    auto range             = m_Pages.equal_range(PageIdx);
+    auto BestOversizedPage = range.second;
     for (auto page_it = range.first; page_it != range.second; ++page_it)
     {
-        Allocation = page_it->second.Allocate(Size, Alignment);
+        MemoryPage& Page = page_it->second;
+        if (Page.GetPageSize() > BasePageSize)
+        {
+            // Small requests must not pin large pages or bypass available shared space.
+            if (AlignedSize > BasePageSize && Page.GetPageSize() >= AlignedSize && Page.IsEmpty() &&
+                (BestOversizedPage == range.second || Page.GetPageSize() < BestOversizedPage->second.GetPageSize()))
+                BestOversizedPage = page_it;
+            continue;
+        }
+        Allocation = Page.Allocate(Size, Alignment);
         if (Allocation.Page != nullptr)
             break;
     }
+    if (Allocation.Page == nullptr && BestOversizedPage != range.second)
+        Allocation = BestOversizedPage->second.Allocate(Size, Alignment, true);
 
     size_t stat_ind = HostVisible ? 1 : 0;
     if (Allocation.Page == nullptr)
     {
-        VkDeviceSize PageSize = HostVisible ? m_HostVisiblePageSize : m_DeviceLocalPageSize;
-        while (PageSize < Size)
-            PageSize *= 2;
+        // Collection may precede deferred releases. Reclaim unusable isolated pages
+        // before asking Vulkan for more memory, even if the next collection has not run.
+        for (auto page_it = range.first; page_it != range.second;)
+        {
+            MemoryPage& Page = page_it->second;
+            if (Page.GetPageSize() > BasePageSize && Page.IsEmpty())
+            {
+                m_CurrAllocatedSize[stat_ind] -= Page.GetPageSize();
+                OnPageDestroy(Page);
+                page_it = m_Pages.erase(page_it);
+            }
+            else
+                ++page_it;
+        }
+
+        // The suballocator rounds the requested size up to Alignment, even at offset zero.
+        const VkDeviceSize PageSize = std::max(BasePageSize, AlignedSize);
+        auto               it       = m_Pages.emplace(PageIdx, MemoryPage{*this, PageSize, MemoryTypeIndex, HostVisible, AllocateFlags});
 
+        // Page construction, mapping and map insertion may throw. Count only resident pages.
         m_CurrAllocatedSize[stat_ind] += PageSize;
         m_PeakAllocatedSize[stat_ind] = std::max(m_PeakAllocatedSize[stat_ind], m_CurrAllocatedSize[stat_ind]);
 
-        auto it = m_Pages.emplace(PageIdx, MemoryPage{*this, PageSize, MemoryTypeIndex, HostVisible, AllocateFlags});
         LOG_INFO_MESSAGE("MemoryManager '", m_MgrName, "': created new ", (HostVisible ? "host-visible" : "device-local"),
                          " page. (", Diligent::FormatMemorySize(PageSize, 2), ", type idx: ", MemoryTypeIndex,
                          "). Current allocated size: ", Diligent::FormatMemorySize(m_CurrAllocatedSize[stat_ind], 2));
@@ -214,8 +253,16 @@ MemoryAllocation MemoryManager::Allocate(VkDeviceSize Size, VkDeviceSize Alignme
 void MemoryManager::ShrinkMemory()
 {
     std::lock_guard<std::mutex> Lock{m_PagesMtx};
-    if (m_CurrAllocatedSize[0] <= m_DeviceLocalReserveSize && m_CurrAllocatedSize[1] <= m_HostVisibleReserveSize)
-        return;
+
+    // Isolated pages must not consume the reserve for shared pages.
+    std::array<VkDeviceSize, 2> SharedAllocatedSize = {};
+    for (const auto& Entry : m_Pages)
+    {
+        const bool         HostVisible  = Entry.first.IsHostVisible;
+        const VkDeviceSize BasePageSize = HostVisible ? m_HostVisiblePageSize : m_DeviceLocalPageSize;
+        if (Entry.second.GetPageSize() <= BasePageSize)
+            SharedAllocatedSize[HostVisible ? 1 : 0] += Entry.second.GetPageSize();
+    }
 
     auto it = m_Pages.begin();
     while (it != m_Pages.end())
@@ -225,9 +272,12 @@ void MemoryManager::ShrinkMemory()
         MemoryPage&  Page          = curr_it->second;
         bool         IsHostVisible = Page.GetCPUMemory() != nullptr;
         VkDeviceSize ReserveSize   = IsHostVisible ? m_HostVisibleReserveSize : m_DeviceLocalReserveSize;
-        if (m_CurrAllocatedSize[IsHostVisible ? 1 : 0] > ReserveSize && Page.IsEmpty())
+        VkDeviceSize BasePageSize  = IsHostVisible ? m_HostVisiblePageSize : m_DeviceLocalPageSize;
+        if ((Page.GetPageSize() > BasePageSize || SharedAllocatedSize[IsHostVisible ? 1 : 0] > ReserveSize) && Page.IsEmpty())
         {
             VkDeviceSize PageSize = Page.GetPageSize();
+            if (PageSize <= BasePageSize)
+                SharedAllocatedSize[IsHostVisible ? 1 : 0] -= PageSize;
             m_CurrAllocatedSize[IsHostVisible ? 1 : 0] -= PageSize;
             LOG_INFO_MESSAGE("MemoryManager '", m_MgrName, "': destroying ", (IsHostVisible ? "host-visible" : "device-local"),
                              " page (", Diligent::FormatMemorySize(PageSize, 2),

Tests/DiligentCoreAPITest/CMakeLists.txt

@@ -135,6 +135,9 @@ if(TARGET Diligent-HLSL2GLSLConverterLib)
 endif()
 
 if(VULKAN_SUPPORTED)
+    target_include_directories(DiligentCoreAPITest PRIVATE ../../Graphics/GraphicsEngineVulkan/include)
+    target_link_libraries(DiligentCoreAPITest PRIVATE Diligent-GraphicsEngineNextGenBase Diligent-GraphicsEngineVk-static)
+    target_compile_definitions(DiligentCoreAPITest PRIVATE DILIGENT_USE_VOLK=1)
     if(PLATFORM_MACOS)
         if(VULKAN_LIB_PATH)
             # Configure rpath so that the executable can find vulkan library