Implement support for multi-range buffers using Vulkan sparse mappings (#5427)1.1.1098

* Pass MultiRange to BufferManager * Implement support for multi-range buffers using Vulkan sparse mappings * Use multi-range for remaining buffers, delete old methods * Assume that more buffers are contiguous * Dispose multi-range buffers after they are removed from the list * Properly init BufferBounds for constant and storage buffers * Do not try reading zero bytes data from an unmapped address on the shader cache + PR feedback * Fix misaligned sparse buffer offsets * Null check can be simplified * PR feedback
author: gdkchan <gab.dark.100@gmail.com> 2023-12-04 16:30:19 -0300
committer: GitHub <noreply@github.com> 2023-12-04 20:30:19 +0100
commit: 1df6c07f78c4c3b8c7fc679d7466f79a10c2d496 (patch)
tree: b80d247e199503274054259cb2707f44cc072993 /src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
parent: 0531c16326c8215bff1c0a98f3ed217f01065446 (diff)
1 files changed, 496 insertions, 61 deletions
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
index 05cc312c..bd9aa39c 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
@@ -11,12 +11,24 @@ namespace Ryujinx.Graphics.Gpu.Memory
     /// </summary>
     class BufferCache : IDisposable
     {
-        private const int OverlapsBufferInitialCapacity = 10;
-        private const int OverlapsBufferMaxCapacity = 10000;
+        /// <summary>
+        /// Initial size for the array holding overlaps.
+        /// </summary>
+        public const int OverlapsBufferInitialCapacity = 10;
+
+        /// <summary>
+        /// Maximum size that an array holding overlaps may have after trimming.
+        /// </summary>
+        public const int OverlapsBufferMaxCapacity = 10000;
 
         private const ulong BufferAlignmentSize = 0x1000;
         private const ulong BufferAlignmentMask = BufferAlignmentSize - 1;
 
+        /// <summary>
+        /// Alignment required for sparse buffer mappings.
+        /// </summary>
+        public const ulong SparseBufferAlignmentSize = 0x10000;
+
         private const ulong MaxDynamicGrowthSize = 0x100000;
 
         private readonly GpuContext _context;
@@ -27,6 +39,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// Must lock for any access from other threads.
         /// </remarks>
         private readonly RangeList<Buffer> _buffers;
+        private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers;
 
         private Buffer[] _bufferOverlaps;
 
@@ -47,6 +60,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
             _physicalMemory = physicalMemory;
 
             _buffers = new RangeList<Buffer>();
+            _multiRangeBuffers = new MultiRangeList<MultiRangeBuffer>();
 
             _bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity];
 
@@ -66,45 +80,100 @@ namespace Ryujinx.Graphics.Gpu.Memory
             Buffer[] overlaps = new Buffer[10];
             int overlapCount;
 
-            ulong address = ((MemoryManager)sender).Translate(e.Address);
-            ulong size = e.Size;
+            MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
 
-            lock (_buffers)
+            for (int index = 0; index < range.Count; index++)
             {
-                overlapCount = _buffers.FindOverlaps(address, size, ref overlaps);
-            }
+                MemoryRange subRange = range.GetSubRange(index);
 
-            for (int i = 0; i < overlapCount; i++)
-            {
-                overlaps[i].Unmapped(address, size);
+                lock (_buffers)
+                {
+                    overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps);
+                }
+
+                for (int i = 0; i < overlapCount; i++)
+                {
+                    overlaps[i].Unmapped(subRange.Address, subRange.Size);
+                }
             }
         }
 
         /// <summary>
         /// Performs address translation of the GPU virtual address, and creates a
-        /// new buffer, if needed, for the specified range.
+        /// new buffer, if needed, for the specified contiguous range.
         /// </summary>
         /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
         /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
         /// <param name="size">Size in bytes of the buffer</param>
-        /// <returns>CPU virtual address of the buffer, after address translation</returns>
-        public ulong TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
+        /// <returns>Contiguous physical range of the buffer, after address translation</returns>
+        public MultiRange TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
         {
             if (gpuVa == 0)
             {
-                return 0;
+                return new MultiRange(MemoryManager.PteUnmapped, size);
             }
 
             ulong address = memoryManager.Translate(gpuVa);
 
-            if (address == MemoryManager.PteUnmapped)
+            if (address != MemoryManager.PteUnmapped)
             {
-                return 0;
+                CreateBuffer(address, size);
             }
 
-            CreateBuffer(address, size);
+            return new MultiRange(address, size);
+        }
 
-            return address;
+        /// <summary>
+        /// Performs address translation of the GPU virtual address, and creates
+        /// new buffers, if needed, for the specified range.
+        /// </summary>
+        /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
+        /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
+        /// <param name="size">Size in bytes of the buffer</param>
+        /// <returns>Physical ranges of the buffer, after address translation</returns>
+        public MultiRange TranslateAndCreateMultiBuffers(MemoryManager memoryManager, ulong gpuVa, ulong size)
+        {
+            if (gpuVa == 0)
+            {
+                return new MultiRange(MemoryManager.PteUnmapped, size);
+            }
+
+            bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
+
+            // Fast path not taken for non-contiguous ranges,
+            // since multi-range buffers are not coalesced, so a buffer that covers
+            // the entire cached range might not actually exist.
+            if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
+                range.Count == 1)
+            {
+                return range;
+            }
+
+            CreateBuffer(range);
+
+            return range;
+        }
+
+        /// <summary>
+        /// Creates a new buffer for the specified range, if it does not yet exist.
+        /// This can be used to ensure the existance of a buffer.
+        /// </summary>
+        /// <param name="range">Physical ranges of memory where the buffer data is located</param>
+        public void CreateBuffer(MultiRange range)
+        {
+            if (range.Count > 1)
+            {
+                CreateMultiRangeBuffer(range);
+            }
+            else
+            {
+                MemoryRange subRange = range.GetSubRange(0);
+
+                if (subRange.Address != MemoryManager.PteUnmapped)
+                {
+                    CreateBuffer(subRange.Address, subRange.Size);
+                }
+            }
         }
 
         /// <summary>
@@ -118,7 +187,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
             ulong endAddress = address + size;
 
             ulong alignedAddress = address & ~BufferAlignmentMask;
-
             ulong alignedEndAddress = (endAddress + BufferAlignmentMask) & ~BufferAlignmentMask;
 
             // The buffer must have the size of at least one page.
@@ -131,6 +199,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
+        /// Creates a new buffer for the specified range, if it does not yet exist.
+        /// This can be used to ensure the existance of a buffer.
+        /// </summary>
+        /// <param name="address">Address of the buffer in memory</param>
+        /// <param name="size">Size of the buffer in bytes</param>
+        /// <param name="alignment">Alignment of the start address of the buffer in bytes</param>
+        public void CreateBuffer(ulong address, ulong size, ulong alignment)
+        {
+            ulong alignmentMask = alignment - 1;
+            ulong pageAlignmentMask = BufferAlignmentMask;
+            ulong endAddress = address + size;
+
+            ulong alignedAddress = address & ~alignmentMask;
+            ulong alignedEndAddress = (endAddress + pageAlignmentMask) & ~pageAlignmentMask;
+
+            // The buffer must have the size of at least one page.
+            if (alignedEndAddress == alignedAddress)
+            {
+                alignedEndAddress += pageAlignmentMask;
+            }
+
+            CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress, alignment);
+        }
+
+        /// <summary>
+        /// Creates a buffer for a memory region composed of multiple physical ranges,
+        /// if it does not exist yet.
+        /// </summary>
+        /// <param name="range">Physical ranges of memory</param>
+        private void CreateMultiRangeBuffer(MultiRange range)
+        {
+            // Ensure all non-contiguous buffer we might use are sparse aligned.
+            for (int i = 0; i < range.Count; i++)
+            {
+                MemoryRange subRange = range.GetSubRange(i);
+
+                if (subRange.Address != MemoryManager.PteUnmapped)
+                {
+                    CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
+                }
+            }
+
+            // Create sparse buffer.
+            MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
+
+            int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
+
+            for (int index = 0; index < overlapCount; index++)
+            {
+                if (overlaps[index].Range.Contains(range))
+                {
+                    return;
+                }
+            }
+
+            for (int index = 0; index < overlapCount; index++)
+            {
+                if (range.Contains(overlaps[index].Range))
+                {
+                    _multiRangeBuffers.Remove(overlaps[index]);
+                    overlaps[index].Dispose();
+                }
+            }
+
+            BufferRange[] storages = new BufferRange[range.Count];
+            MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
+
+            ulong alignmentMask = SparseBufferAlignmentSize - 1;
+
+            for (int i = 0; i < range.Count; i++)
+            {
+                MemoryRange subRange = range.GetSubRange(i);
+
+                if (subRange.Address != MemoryManager.PteUnmapped)
+                {
+                    ulong endAddress = subRange.Address + subRange.Size;
+
+                    ulong alignedAddress = subRange.Address & ~alignmentMask;
+                    ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
+                    ulong alignedSize = alignedEndAddress - alignedAddress;
+
+                    Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
+                    BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
+
+                    storages[i] = bufferRange;
+                    alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+                }
+                else
+                {
+                    ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
+
+                    storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
+                    alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
+                }
+            }
+
+            MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
+
+            _multiRangeBuffers.Add(multiRangeBuffer);
+        }
+
+        /// <summary>
         /// Performs address translation of the GPU virtual address, and attempts to force
         /// the buffer in the region as dirty.
         /// The buffer lookup for this function is cached in a dictionary for quick access, which
@@ -150,7 +320,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 result.EndGpuAddress < gpuVa + size ||
                 result.UnmappedSequence != result.Buffer.UnmappedSequence)
             {
-                ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
+                MultiRange range = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
+                ulong address = range.GetSubRange(0).Address;
                 result = new BufferCacheEntry(address, gpuVa, GetBuffer(address, size));
 
                 _dirtyCache[gpuVa] = result;
@@ -184,7 +355,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 result.EndGpuAddress < alignedEndGpuVa ||
                 result.UnmappedSequence != result.Buffer.UnmappedSequence)
             {
-                ulong address = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
+                MultiRange range = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
+                ulong address = range.GetSubRange(0).Address;
                 result = new BufferCacheEntry(address, alignedGpuVa, GetBuffer(address, size));
 
                 _modifiedCache[alignedGpuVa] = result;
@@ -204,7 +376,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="size">Size in bytes of the buffer</param>
         private void CreateBufferAligned(ulong address, ulong size)
         {
-            int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
+            Buffer[] overlaps = _bufferOverlaps;
+            int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
 
             if (overlapsCount != 0)
             {
@@ -215,9 +388,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 // old buffer(s) to the new buffer.
 
                 ulong endAddress = address + size;
+                Buffer overlap0 = overlaps[0];
 
-                if (_bufferOverlaps[0].Address > address || _bufferOverlaps[0].EndAddress < endAddress)
+                if (overlap0.Address > address || overlap0.EndAddress < endAddress)
                 {
+                    bool anySparseCompatible = false;
+
                     // Check if the following conditions are met:
                     // - We have a single overlap.
                     // - The overlap starts at or before the requested range. That is, the overlap happens at the end.
@@ -228,23 +404,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
                     // Allowing for 2 pages (rather than just one) is necessary to catch cases where the
                     // range crosses a page, and after alignment, ends having a size of 2 pages.
                     if (overlapsCount == 1 &&
-                        address >= _bufferOverlaps[0].Address &&
-                        endAddress - _bufferOverlaps[0].EndAddress <= BufferAlignmentSize * 2)
+                        address >= overlap0.Address &&
+                        endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2)
                     {
                         // Try to grow the buffer by 1.5x of its current size.
                         // This improves performance in the cases where the buffer is resized often by small amounts.
-                        ulong existingSize = _bufferOverlaps[0].Size;
+                        ulong existingSize = overlap0.Size;
                         ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
 
                         size = Math.Max(size, growthSize);
                         endAddress = address + size;
 
-                        overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
+                        overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
                     }
 
                     for (int index = 0; index < overlapsCount; index++)
                     {
-                        Buffer buffer = _bufferOverlaps[index];
+                        Buffer buffer = overlaps[index];
+
+                        anySparseCompatible |= buffer.SparseCompatible;
 
                         address = Math.Min(address, buffer.Address);
                         endAddress = Math.Max(endAddress, buffer.EndAddress);
@@ -257,35 +435,91 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
                     ulong newSize = endAddress - address;
 
-                    Buffer newBuffer = new(_context, _physicalMemory, address, newSize, _bufferOverlaps.Take(overlapsCount));
+                    CreateBufferAligned(address, newSize, anySparseCompatible, overlaps, overlapsCount);
+                }
+            }
+            else
+            {
+                // No overlap, just create a new buffer.
+                Buffer buffer = new(_context, _physicalMemory, address, size, sparseCompatible: false);
 
-                    lock (_buffers)
-                    {
-                        _buffers.Add(newBuffer);
-                    }
+                lock (_buffers)
+                {
+                    _buffers.Add(buffer);
+                }
+            }
 
-                    for (int index = 0; index < overlapsCount; index++)
+            ShrinkOverlapsBufferIfNeeded();
+        }
+
+        /// <summary>
+        /// Creates a new buffer for the specified range, if needed.
+        /// If a buffer where this range can be fully contained already exists,
+        /// then the creation of a new buffer is not necessary.
+        /// </summary>
+        /// <param name="address">Address of the buffer in guest memory</param>
+        /// <param name="size">Size in bytes of the buffer</param>
+        /// <param name="alignment">Alignment of the start address of the buffer</param>
+        private void CreateBufferAligned(ulong address, ulong size, ulong alignment)
+        {
+            Buffer[] overlaps = _bufferOverlaps;
+            int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
+            bool sparseAligned = alignment >= SparseBufferAlignmentSize;
+
+            if (overlapsCount != 0)
+            {
+                // If the buffer already exists, make sure if covers the entire range,
+                // and make sure it is properly aligned, otherwise sparse mapping may fail.
+
+                ulong endAddress = address + size;
+                Buffer overlap0 = overlaps[0];
+
+                if (overlap0.Address > address ||
+                    overlap0.EndAddress < endAddress ||
+                    (overlap0.Address & (alignment - 1)) != 0 ||
+                    (!overlap0.SparseCompatible && sparseAligned))
+                {
+                    // We need to make sure the new buffer is properly aligned.
+                    // However, after the range is aligned, it is possible that it
+                    // overlaps more buffers, so try again after each extension
+                    // and ensure we cover all overlaps.
+
+                    int oldOverlapsCount;
+
+                    do
                     {
-                        Buffer buffer = _bufferOverlaps[index];
+                        for (int index = 0; index < overlapsCount; index++)
+                        {
+                            Buffer buffer = overlaps[index];
 
-                        int dstOffset = (int)(buffer.Address - newBuffer.Address);
+                            address = Math.Min(address, buffer.Address);
+                            endAddress = Math.Max(endAddress, buffer.EndAddress);
+                        }
 
-                        buffer.CopyTo(newBuffer, dstOffset);
-                        newBuffer.InheritModifiedRanges(buffer);
+                        address &= ~(alignment - 1);
 
-                        buffer.DecrementReferenceCount();
+                        oldOverlapsCount = overlapsCount;
+                        overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps);
                     }
+                    while (oldOverlapsCount != overlapsCount);
 
-                    newBuffer.SynchronizeMemory(address, newSize);
+                    lock (_buffers)
+                    {
+                        for (int index = 0; index < overlapsCount; index++)
+                        {
+                            _buffers.Remove(overlaps[index]);
+                        }
+                    }
+
+                    ulong newSize = endAddress - address;
 
-                    // Existing buffers were modified, we need to rebind everything.
-                    NotifyBuffersModified?.Invoke();
+                    CreateBufferAligned(address, newSize, sparseAligned, overlaps, overlapsCount);
                 }
             }
             else
             {
                 // No overlap, just create a new buffer.
-                Buffer buffer = new(_context, _physicalMemory, address, size);
+                Buffer buffer = new(_context, _physicalMemory, address, size, sparseAligned);
 
                 lock (_buffers)
                 {
@@ -297,6 +531,73 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
+        /// Creates a new buffer for the specified range, if needed.
+        /// If a buffer where this range can be fully contained already exists,
+        /// then the creation of a new buffer is not necessary.
+        /// </summary>
+        /// <param name="address">Address of the buffer in guest memory</param>
+        /// <param name="size">Size in bytes of the buffer</param>
+        /// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
+        /// <param name="overlaps">Buffers overlapping the range</param>
+        /// <param name="overlapsCount">Total of overlaps</param>
+        private void CreateBufferAligned(ulong address, ulong size, bool sparseCompatible, Buffer[] overlaps, int overlapsCount)
+        {
+            Buffer newBuffer = new Buffer(_context, _physicalMemory, address, size, sparseCompatible, overlaps.Take(overlapsCount));
+
+            lock (_buffers)
+            {
+                _buffers.Add(newBuffer);
+            }
+
+            for (int index = 0; index < overlapsCount; index++)
+            {
+                Buffer buffer = overlaps[index];
+
+                int dstOffset = (int)(buffer.Address - newBuffer.Address);
+
+                buffer.CopyTo(newBuffer, dstOffset);
+                newBuffer.InheritModifiedRanges(buffer);
+
+                buffer.DecrementReferenceCount();
+            }
+
+            newBuffer.SynchronizeMemory(address, size);
+
+            // Existing buffers were modified, we need to rebind everything.
+            NotifyBuffersModified?.Invoke();
+
+            RecreateMultiRangeBuffers(address, size);
+        }
+
+        /// <summary>
+        /// Recreates all the multi-range buffers that overlaps a given physical memory range.
+        /// </summary>
+        /// <param name="address">Start address of the range</param>
+        /// <param name="size">Size of the range in bytes</param>
+        private void RecreateMultiRangeBuffers(ulong address, ulong size)
+        {
+            if ((address & (SparseBufferAlignmentSize - 1)) != 0 || (size & (SparseBufferAlignmentSize - 1)) != 0)
+            {
+                return;
+            }
+
+            MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
+
+            int overlapCount = _multiRangeBuffers.FindOverlaps(address, size, ref overlaps);
+
+            for (int index = 0; index < overlapCount; index++)
+            {
+                _multiRangeBuffers.Remove(overlaps[index]);
+                overlaps[index].Dispose();
+            }
+
+            for (int index = 0; index < overlapCount; index++)
+            {
+                CreateMultiRangeBuffer(overlaps[index].Range);
+            }
+        }
+
+        /// <summary>
         /// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
         /// </summary>
         private void ShrinkOverlapsBufferIfNeeded()
@@ -319,9 +620,63 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="size">Size in bytes of the copy</param>
         public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
         {
-            ulong srcAddress = TranslateAndCreateBuffer(memoryManager, srcVa, size);
-            ulong dstAddress = TranslateAndCreateBuffer(memoryManager, dstVa, size);
+            MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
+            MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
+
+            if (srcRange.Count == 1 && dstRange.Count == 1)
+            {
+                CopyBufferSingleRange(memoryManager, srcRange.GetSubRange(0).Address, dstRange.GetSubRange(0).Address, size);
+            }
+            else
+            {
+                ulong copiedSize = 0;
+                ulong srcOffset = 0;
+                ulong dstOffset = 0;
+                int srcRangeIndex = 0;
+                int dstRangeIndex = 0;
+
+                while (copiedSize < size)
+                {
+                    if (srcRange.GetSubRange(srcRangeIndex).Size == srcOffset)
+                    {
+                        srcRangeIndex++;
+                        srcOffset = 0;
+                    }
+
+                    if (dstRange.GetSubRange(dstRangeIndex).Size == dstOffset)
+                    {
+                        dstRangeIndex++;
+                        dstOffset = 0;
+                    }
+
+                    MemoryRange srcSubRange = srcRange.GetSubRange(srcRangeIndex);
+                    MemoryRange dstSubRange = dstRange.GetSubRange(dstRangeIndex);
+
+                    ulong srcSize = srcSubRange.Size - srcOffset;
+                    ulong dstSize = dstSubRange.Size - dstOffset;
+                    ulong copySize = Math.Min(srcSize, dstSize);
+
+                    CopyBufferSingleRange(memoryManager, srcSubRange.Address + srcOffset, dstSubRange.Address + dstOffset, copySize);
+
+                    srcOffset += copySize;
+                    dstOffset += copySize;
+                    copiedSize += copySize;
+                }
+            }
+        }
 
+        /// <summary>
+        /// Copy a buffer data from a given address to another.
+        /// </summary>
+        /// <remarks>
+        /// This does a GPU side copy.
+        /// </remarks>
+        /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
+        /// <param name="srcAddress">Physical address of the copy source</param>
+        /// <param name="dstAddress">Physical address of the copy destination</param>
+        /// <param name="size">Size in bytes of the copy</param>
+        private void CopyBufferSingleRange(MemoryManager memoryManager, ulong srcAddress, ulong dstAddress, ulong size)
+        {
             Buffer srcBuffer = GetBuffer(srcAddress, size);
             Buffer dstBuffer = GetBuffer(dstAddress, size);
 
@@ -360,39 +715,98 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="value">Value to be written into the buffer</param>
         public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
         {
-            ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
+            MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
 
-            Buffer buffer = GetBuffer(address, size);
+            for (int index = 0; index < range.Count; index++)
+            {
+                MemoryRange subRange = range.GetSubRange(index);
+                Buffer buffer = GetBuffer(subRange.Address, subRange.Size);
 
-            int offset = (int)(address - buffer.Address);
+                int offset = (int)(subRange.Address - buffer.Address);
 
-            _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)size, value);
+                _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
 
-            memoryManager.Physical.FillTrackedResource(address, size, value, ResourceKind.Buffer);
+                memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
+            }
         }
 
         /// <summary>
-        /// Gets a buffer sub-range from a start address til a page boundary after the given size.
+        /// Gets a buffer sub-range starting at a given memory address, aligned to the next page boundary.
         /// </summary>
-        /// <param name="address">Start address of the memory range</param>
-        /// <param name="size">Size in bytes of the memory range</param>
+        /// <param name="range">Physical regions of memory where the buffer is mapped</param>
         /// <param name="write">Whether the buffer will be written to by this use</param>
         /// <returns>The buffer sub-range starting at the given memory address</returns>
-        public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false)
+        public BufferRange GetBufferRangeAligned(MultiRange range, bool write = false)
         {
-            return GetBuffer(address, size, write).GetRangeAligned(address, size, write);
+            if (range.Count > 1)
+            {
+                return GetBuffer(range, write).GetRange(range);
+            }
+            else
+            {
+                MemoryRange subRange = range.GetSubRange(0);
+                return GetBuffer(subRange.Address, subRange.Size, write).GetRangeAligned(subRange.Address, subRange.Size, write);
+            }
         }
 
         /// <summary>
         /// Gets a buffer sub-range for a given memory range.
         /// </summary>
-        /// <param name="address">Start address of the memory range</param>
-        /// <param name="size">Size in bytes of the memory range</param>
+        /// <param name="range">Physical regions of memory where the buffer is mapped</param>
         /// <param name="write">Whether the buffer will be written to by this use</param>
         /// <returns>The buffer sub-range for the given range</returns>
-        public BufferRange GetBufferRange(ulong address, ulong size, bool write = false)
+        public BufferRange GetBufferRange(MultiRange range, bool write = false)
+        {
+            if (range.Count > 1)
+            {
+                return GetBuffer(range, write).GetRange(range);
+            }
+            else
+            {
+                MemoryRange subRange = range.GetSubRange(0);
+                return GetBuffer(subRange.Address, subRange.Size, write).GetRange(subRange.Address, subRange.Size, write);
+            }
+        }
+
+        /// <summary>
+        /// Gets a buffer for a given memory range.
+        /// A buffer overlapping with the specified range is assumed to already exist on the cache.
+        /// </summary>
+        /// <param name="range">Physical regions of memory where the buffer is mapped</param>
+        /// <param name="write">Whether the buffer will be written to by this use</param>
+        /// <returns>The buffer where the range is fully contained</returns>
+        private MultiRangeBuffer GetBuffer(MultiRange range, bool write = false)
         {
-            return GetBuffer(address, size, write).GetRange(address, size, write);
+            for (int i = 0; i < range.Count; i++)
+            {
+                MemoryRange subRange = range.GetSubRange(i);
+
+                Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
+
+                subBuffer.SynchronizeMemory(subRange.Address, subRange.Size);
+
+                if (write)
+                {
+                    subBuffer.SignalModified(subRange.Address, subRange.Size);
+                }
+            }
+
+            MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
+
+            int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
+
+            MultiRangeBuffer buffer = null;
+
+            for (int i = 0; i < overlapCount; i++)
+            {
+                if (overlaps[i].Range.Contains(range))
+                {
+                    buffer = overlaps[i];
+                    break;
+                }
+            }
+
+            return buffer;
         }
 
         /// <summary>
@@ -429,9 +843,30 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <summary>
         /// Performs guest to host memory synchronization of a given memory range.
         /// </summary>
+        /// <param name="range">Physical regions of memory where the buffer is mapped</param>
+        public void SynchronizeBufferRange(MultiRange range)
+        {
+            if (range.Count == 1)
+            {
+                MemoryRange subRange = range.GetSubRange(0);
+                SynchronizeBufferRange(subRange.Address, subRange.Size);
+            }
+            else
+            {
+                for (int index = 0; index < range.Count; index++)
+                {
+                    MemoryRange subRange = range.GetSubRange(index);
+                    SynchronizeBufferRange(subRange.Address, subRange.Size);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Performs guest to host memory synchronization of a given memory range.
+        /// </summary>
         /// <param name="address">Start address of the memory range</param>
         /// <param name="size">Size in bytes of the memory range</param>
-        public void SynchronizeBufferRange(ulong address, ulong size)
+        private void SynchronizeBufferRange(ulong address, ulong size)
         {
             if (size != 0)
             {
@@ -491,7 +926,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
         /// <summary>
         /// Disposes all buffers in the cache.
-        /// It's an error to use the buffer manager after disposal.
+        /// It's an error to use the buffer cache after disposal.
         /// </summary>
         public void Dispose()
         {
author	gdkchan <gab.dark.100@gmail.com>	2023-12-04 16:30:19 -0300
committer	GitHub <noreply@github.com>	2023-12-04 20:30:19 +0100
commit	1df6c07f78c4c3b8c7fc679d7466f79a10c2d496 (patch)
tree	b80d247e199503274054259cb2707f44cc072993 /src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
parent	0531c16326c8215bff1c0a98f3ed217f01065446 (diff)