Implement virtual buffer dependencies (#6190)1.1.1208

* Implement virtual buffer copies * Introduce TranslateAndCreateMultiBuffersPhysicalOnly, use it for copy and clear * Rename VirtualBufferCache to VirtualRangeCache * Fix potential issue where virtual range could exist in the cache, without a physical buffer * Fix bug that could cause copy with negative size on CopyToDependantVirtualBuffer * Remove virtual copy back for SyncAction * GetData XML docs * Make field readonly * Fix virtual buffer modification tracking * Remove CopyFromDependantVirtualBuffers from ExternalFlush * Move things around a little to avoid perf impact - Inline null check for CopyFromDependantVirtualBuffers - Remove extra method call for SynchronizeMemoryWithVirtualCopyBack, prefer calling CopyFromDependantVirtualBuffers separately * Fix up XML doc --------- Co-authored-by: riperiperi <rhy3756547@hotmail.com>
author: gdkchan <gab.dark.100@gmail.com> 2024-02-22 11:03:07 -0300
committer: GitHub <noreply@github.com> 2024-02-22 11:03:07 -0300
commit: 167f50bbcd5b2378a038e540877be4d9b71a12f6 (patch)
tree: b425e6529ade9eb52dca3f9ff92ea767a78224b4 /src
parent: ba91f5d401bf5f9fc39aaa30feac2600e82f9c42 (diff)
5 files changed, 569 insertions, 45 deletions
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
index 12461e96..e01e5142 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
@@ -5,6 +5,8 @@ using Ryujinx.Memory.Tracking;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Threading;
 
 namespace Ryujinx.Graphics.Gpu.Memory
 {
@@ -65,6 +67,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
         private readonly Action<ulong, ulong> _loadDelegate;
         private readonly Action<ulong, ulong> _modifiedDelegate;
 
+        private HashSet<MultiRangeBuffer> _virtualDependencies;
+        private readonly ReaderWriterLockSlim _virtualDependenciesLock;
+
         private int _sequenceNumber;
 
         private readonly bool _useGranular;
@@ -152,6 +157,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
             _externalFlushDelegate = new RegionSignal(ExternalFlush);
             _loadDelegate = new Action<ulong, ulong>(LoadRegion);
             _modifiedDelegate = new Action<ulong, ulong>(RegionModified);
+
+            _virtualDependenciesLock = new ReaderWriterLockSlim();
         }
 
         /// <summary>
@@ -220,6 +227,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </remarks>
         /// <param name="address">Start address of the range to synchronize</param>
         /// <param name="size">Size in bytes of the range to synchronize</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void SynchronizeMemory(ulong address, ulong size)
         {
             if (_useGranular)
@@ -239,6 +247,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
                     else
                     {
                         _context.Renderer.SetBufferData(Handle, 0, _physicalMemory.GetSpan(Address, (int)Size));
+                        CopyToDependantVirtualBuffers();
                     }
 
                     _sequenceNumber = _context.SequenceNumber;
@@ -460,6 +469,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
             int offset = (int)(mAddress - Address);
 
             _context.Renderer.SetBufferData(Handle, offset, _physicalMemory.GetSpan(mAddress, (int)mSize));
+
+            CopyToDependantVirtualBuffers(mAddress, mSize);
         }
 
         /// <summary>
@@ -520,6 +531,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="dstOffset">The offset of the destination buffer to copy into</param>
         public void CopyTo(Buffer destination, int dstOffset)
         {
+            CopyFromDependantVirtualBuffers();
             _context.Renderer.Pipeline.CopyBuffer(Handle, destination.Handle, 0, dstOffset, (int)Size);
         }
 
@@ -536,7 +548,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
             using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size);
 
             // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers.
-            _physicalMemory.WriteUntracked(address, data.Get());
+            _physicalMemory.WriteUntracked(address, CopyFromDependantVirtualBuffers(data.Get(), address, size));
         }
 
         /// <summary>
@@ -618,6 +630,207 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
+        /// Adds a virtual buffer dependency, indicating that a virtual buffer depends on data from this buffer.
+        /// </summary>
+        /// <param name="virtualBuffer">Dependant virtual buffer</param>
+        public void AddVirtualDependency(MultiRangeBuffer virtualBuffer)
+        {
+            _virtualDependenciesLock.EnterWriteLock();
+
+            try
+            {
+                (_virtualDependencies ??= new()).Add(virtualBuffer);
+            }
+            finally
+            {
+                _virtualDependenciesLock.ExitWriteLock();
+            }
+        }
+
+        /// <summary>
+        /// Removes a virtual buffer dependency, indicating that a virtual buffer no longer depends on data from this buffer.
+        /// </summary>
+        /// <param name="virtualBuffer">Dependant virtual buffer</param>
+        public void RemoveVirtualDependency(MultiRangeBuffer virtualBuffer)
+        {
+            _virtualDependenciesLock.EnterWriteLock();
+
+            try
+            {
+                if (_virtualDependencies != null)
+                {
+                    _virtualDependencies.Remove(virtualBuffer);
+
+                    if (_virtualDependencies.Count == 0)
+                    {
+                        _virtualDependencies = null;
+                    }
+                }
+            }
+            finally
+            {
+                _virtualDependenciesLock.ExitWriteLock();
+            }
+        }
+
+        /// <summary>
+        /// Copies the buffer data to all virtual buffers that depends on it.
+        /// </summary>
+        public void CopyToDependantVirtualBuffers()
+        {
+            CopyToDependantVirtualBuffers(Address, Size);
+        }
+
+        /// <summary>
+        /// Copies the buffer data inside the specifide range to all virtual buffers that depends on it.
+        /// </summary>
+        /// <param name="address">Address of the range</param>
+        /// <param name="size">Size of the range in bytes</param>
+        public void CopyToDependantVirtualBuffers(ulong address, ulong size)
+        {
+            if (_virtualDependencies != null)
+            {
+                foreach (var virtualBuffer in _virtualDependencies)
+                {
+                    CopyToDependantVirtualBuffer(virtualBuffer, address, size);
+                }
+            }
+        }
+
+        /// <summary>
+        /// Copies all modified ranges from all virtual buffers back into this buffer.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public void CopyFromDependantVirtualBuffers()
+        {
+            if (_virtualDependencies != null)
+            {
+                CopyFromDependantVirtualBuffersImpl();
+            }
+        }
+
+        /// <summary>
+        /// Copies all modified ranges from all virtual buffers back into this buffer.
+        /// </summary>
+        [MethodImpl(MethodImplOptions.NoInlining)]
+        private void CopyFromDependantVirtualBuffersImpl()
+        {
+            foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber))
+            {
+                virtualBuffer.ConsumeModifiedRegion(this, (mAddress, mSize) =>
+                {
+                    // Get offset inside both this and the virtual buffer.
+                    // Note that sometimes there is no right answer for the virtual offset,
+                    // as the same physical range might be mapped multiple times inside a virtual buffer.
+                    // We just assume it does not happen in practice as it can only be implemented correctly
+                    // when the host has support for proper sparse mapping.
+
+                    ulong mEndAddress = mAddress + mSize;
+                    mAddress = Math.Max(mAddress, Address);
+                    mSize = Math.Min(mEndAddress, EndAddress) - mAddress;
+
+                    int physicalOffset = (int)(mAddress - Address);
+                    int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize));
+
+                    _context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)mSize);
+                });
+            }
+        }
+
+        /// <summary>
+        /// Copies all overlapping modified ranges from all virtual buffers back into this buffer, and returns an updated span with the data.
+        /// </summary>
+        /// <param name="dataSpan">Span where the unmodified data will be taken from for the output</param>
+        /// <param name="address">Address of the region to copy</param>
+        /// <param name="size">Size of the region to copy in bytes</param>
+        /// <returns>A span with <paramref name="dataSpan"/>, and the data for all modified ranges if any</returns>
+        private ReadOnlySpan<byte> CopyFromDependantVirtualBuffers(ReadOnlySpan<byte> dataSpan, ulong address, ulong size)
+        {
+            _virtualDependenciesLock.EnterReadLock();
+
+            try
+            {
+                if (_virtualDependencies != null)
+                {
+                    byte[] storage = dataSpan.ToArray();
+
+                    foreach (var virtualBuffer in _virtualDependencies.OrderBy(x => x.ModificationSequenceNumber))
+                    {
+                        virtualBuffer.ConsumeModifiedRegion(address, size, (mAddress, mSize) =>
+                        {
+                            // Get offset inside both this and the virtual buffer.
+                            // Note that sometimes there is no right answer for the virtual offset,
+                            // as the same physical range might be mapped multiple times inside a virtual buffer.
+                            // We just assume it does not happen in practice as it can only be implemented correctly
+                            // when the host has support for proper sparse mapping.
+
+                            ulong mEndAddress = mAddress + mSize;
+                            mAddress = Math.Max(mAddress, address);
+                            mSize = Math.Min(mEndAddress, address + size) - mAddress;
+
+                            int physicalOffset = (int)(mAddress - Address);
+                            int virtualOffset = virtualBuffer.Range.FindOffset(new(mAddress, mSize));
+
+                            _context.Renderer.Pipeline.CopyBuffer(virtualBuffer.Handle, Handle, virtualOffset, physicalOffset, (int)size);
+                            virtualBuffer.GetData(storage.AsSpan().Slice((int)(mAddress - address), (int)mSize), virtualOffset, (int)mSize);
+                        });
+                    }
+
+                    dataSpan = storage;
+                }
+            }
+            finally
+            {
+                _virtualDependenciesLock.ExitReadLock();
+            }
+
+            return dataSpan;
+        }
+
+        /// <summary>
+        /// Copies the buffer data to the specified virtual buffer.
+        /// </summary>
+        /// <param name="virtualBuffer">Virtual buffer to copy the data into</param>
+        public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer)
+        {
+            CopyToDependantVirtualBuffer(virtualBuffer, Address, Size);
+        }
+
+        /// <summary>
+        /// Copies the buffer data inside the given range to the specified virtual buffer.
+        /// </summary>
+        /// <param name="virtualBuffer">Virtual buffer to copy the data into</param>
+        /// <param name="address">Address of the range</param>
+        /// <param name="size">Size of the range in bytes</param>
+        public void CopyToDependantVirtualBuffer(MultiRangeBuffer virtualBuffer, ulong address, ulong size)
+        {
+            // Broadcast data to all ranges of the virtual buffer that are contained inside this buffer.
+
+            ulong lastOffset = 0;
+
+            while (virtualBuffer.TryGetPhysicalOffset(this, lastOffset, out ulong srcOffset, out ulong dstOffset, out ulong copySize))
+            {
+                ulong innerOffset = address - Address;
+                ulong innerEndOffset = (address + size) - Address;
+
+                lastOffset = dstOffset + copySize;
+
+                // Clamp range to the specified range.
+                ulong copySrcOffset = Math.Max(srcOffset, innerOffset);
+                ulong copySrcEndOffset = Math.Min(innerEndOffset, srcOffset + copySize);
+
+                if (copySrcEndOffset > copySrcOffset)
+                {
+                    copySize = copySrcEndOffset - copySrcOffset;
+                    dstOffset += copySrcOffset - srcOffset;
+                    srcOffset = copySrcOffset;
+
+                    _context.Renderer.Pipeline.CopyBuffer(Handle, virtualBuffer.Handle, (int)srcOffset, (int)dstOffset, (int)copySize);
+                }
+            }
+        }
+
+        /// <summary>
         /// Increments the buffer reference count.
         /// </summary>
         public void IncrementReferenceCount()
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
index bd9aa39c..c6284780 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
@@ -3,6 +3,7 @@ using Ryujinx.Memory.Range;
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Runtime.CompilerServices;
 
 namespace Ryujinx.Graphics.Gpu.Memory
 {
@@ -46,6 +47,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         private readonly Dictionary<ulong, BufferCacheEntry> _dirtyCache;
         private readonly Dictionary<ulong, BufferCacheEntry> _modifiedCache;
         private bool _pruneCaches;
+        private int _virtualModifiedSequenceNumber;
 
         public event Action NotifyBuffersModified;
 
@@ -125,7 +127,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
 
         /// <summary>
         /// Performs address translation of the GPU virtual address, and creates
-        /// new buffers, if needed, for the specified range.
+        /// new physical and virtual buffers, if needed, for the specified range.
         /// </summary>
         /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
         /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
@@ -138,12 +140,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 return new MultiRange(MemoryManager.PteUnmapped, size);
             }
 
-            bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
-
             // Fast path not taken for non-contiguous ranges,
             // since multi-range buffers are not coalesced, so a buffer that covers
             // the entire cached range might not actually exist.
-            if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
+            if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
                 range.Count == 1)
             {
                 return range;
@@ -155,6 +155,50 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
+        /// Performs address translation of the GPU virtual address, and creates
+        /// new physical buffers, if needed, for the specified range.
+        /// </summary>
+        /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
+        /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
+        /// <param name="size">Size in bytes of the buffer</param>
+        /// <returns>Physical ranges of the buffer, after address translation</returns>
+        public MultiRange TranslateAndCreateMultiBuffersPhysicalOnly(MemoryManager memoryManager, ulong gpuVa, ulong size)
+        {
+            if (gpuVa == 0)
+            {
+                return new MultiRange(MemoryManager.PteUnmapped, size);
+            }
+
+            // Fast path not taken for non-contiguous ranges,
+            // since multi-range buffers are not coalesced, so a buffer that covers
+            // the entire cached range might not actually exist.
+            if (memoryManager.VirtualRangeCache.TryGetOrAddRange(gpuVa, size, out MultiRange range) &&
+                range.Count == 1)
+            {
+                return range;
+            }
+
+            for (int i = 0; i < range.Count; i++)
+            {
+                MemoryRange subRange = range.GetSubRange(i);
+
+                if (subRange.Address != MemoryManager.PteUnmapped)
+                {
+                    if (range.Count > 1)
+                    {
+                        CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
+                    }
+                    else
+                    {
+                        CreateBuffer(subRange.Address, subRange.Size);
+                    }
+                }
+            }
+
+            return range;
+        }
+
+        /// <summary>
         /// Creates a new buffer for the specified range, if it does not yet exist.
         /// This can be used to ensure the existance of a buffer.
         /// </summary>
@@ -263,44 +307,111 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 }
             }
 
-            BufferRange[] storages = new BufferRange[range.Count];
+            MultiRangeBuffer multiRangeBuffer;
+
             MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
 
             ulong alignmentMask = SparseBufferAlignmentSize - 1;
 
-            for (int i = 0; i < range.Count; i++)
+            if (_context.Capabilities.SupportsSparseBuffer)
             {
-                MemoryRange subRange = range.GetSubRange(i);
+                BufferRange[] storages = new BufferRange[range.Count];
 
-                if (subRange.Address != MemoryManager.PteUnmapped)
+                for (int i = 0; i < range.Count; i++)
                 {
-                    ulong endAddress = subRange.Address + subRange.Size;
+                    MemoryRange subRange = range.GetSubRange(i);
 
-                    ulong alignedAddress = subRange.Address & ~alignmentMask;
-                    ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
-                    ulong alignedSize = alignedEndAddress - alignedAddress;
+                    if (subRange.Address != MemoryManager.PteUnmapped)
+                    {
+                        ulong endAddress = subRange.Address + subRange.Size;
 
-                    Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
-                    BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
+                        ulong alignedAddress = subRange.Address & ~alignmentMask;
+                        ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
+                        ulong alignedSize = alignedEndAddress - alignedAddress;
 
-                    storages[i] = bufferRange;
-                    alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+                        Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
+                        BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
+
+                        alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+                        storages[i] = bufferRange;
+                    }
+                    else
+                    {
+                        ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
+
+                        alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
+                        storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
+                    }
                 }
-                else
+
+                multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
+            }
+            else
+            {
+                for (int i = 0; i < range.Count; i++)
                 {
-                    ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
+                    MemoryRange subRange = range.GetSubRange(i);
+
+                    if (subRange.Address != MemoryManager.PteUnmapped)
+                    {
+                        ulong endAddress = subRange.Address + subRange.Size;
+
+                        ulong alignedAddress = subRange.Address & ~alignmentMask;
+                        ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
+                        ulong alignedSize = alignedEndAddress - alignedAddress;
+
+                        alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+                    }
+                    else
+                    {
+                        ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
 
-                    storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
-                    alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
+                        alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
+                    }
                 }
-            }
 
-            MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
+                multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges));
+
+                UpdateVirtualBufferDependencies(multiRangeBuffer);
+            }
 
             _multiRangeBuffers.Add(multiRangeBuffer);
         }
 
         /// <summary>
+        /// Adds two-way dependencies to all physical buffers contained within a given virtual buffer.
+        /// </summary>
+        /// <param name="virtualBuffer">Virtual buffer to have dependencies added</param>
+        private void UpdateVirtualBufferDependencies(MultiRangeBuffer virtualBuffer)
+        {
+            virtualBuffer.ClearPhysicalDependencies();
+
+            ulong dstOffset = 0;
+
+            HashSet<Buffer> physicalBuffers = new();
+
+            for (int i = 0; i < virtualBuffer.Range.Count; i++)
+            {
+                MemoryRange subRange = virtualBuffer.Range.GetSubRange(i);
+
+                if (subRange.Address != MemoryManager.PteUnmapped)
+                {
+                    Buffer buffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
+
+                    virtualBuffer.AddPhysicalDependency(buffer, subRange.Address, dstOffset, subRange.Size);
+                    physicalBuffers.Add(buffer);
+                }
+
+                dstOffset += subRange.Size;
+            }
+
+            foreach (var buffer in physicalBuffers)
+            {
+                buffer.CopyToDependantVirtualBuffer(virtualBuffer);
+            }
+        }
+
+        /// <summary>
         /// Performs address translation of the GPU virtual address, and attempts to force
         /// the buffer in the region as dirty.
         /// The buffer lookup for this function is cached in a dictionary for quick access, which
@@ -620,8 +731,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="size">Size in bytes of the copy</param>
         public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
         {
-            MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
-            MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
+            MultiRange srcRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, srcVa, size);
+            MultiRange dstRange = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, dstVa, size);
 
             if (srcRange.Count == 1 && dstRange.Count == 1)
             {
@@ -701,6 +812,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 dstBuffer.ClearModified(dstAddress, size);
                 memoryManager.Physical.WriteTrackedResource(dstAddress, memoryManager.Physical.GetSpan(srcAddress, (int)size), ResourceKind.Buffer);
             }
+
+            dstBuffer.CopyToDependantVirtualBuffers(dstAddress, size);
         }
 
         /// <summary>
@@ -715,7 +828,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// <param name="value">Value to be written into the buffer</param>
         public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
         {
-            MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
+            MultiRange range = TranslateAndCreateMultiBuffersPhysicalOnly(memoryManager, gpuVa, size);
 
             for (int index = 0; index < range.Count; index++)
             {
@@ -727,6 +840,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
 
                 memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
+
+                buffer.CopyToDependantVirtualBuffers(subRange.Address, subRange.Size);
             }
         }
 
@@ -806,6 +921,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 }
             }
 
+            if (write && buffer != null && !_context.Capabilities.SupportsSparseBuffer)
+            {
+                buffer.AddModifiedRegion(range, ++_virtualModifiedSequenceNumber);
+            }
+
             return buffer;
         }
 
@@ -825,6 +945,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
             {
                 buffer = _buffers.FindFirstOverlap(address, size);
 
+                buffer.CopyFromDependantVirtualBuffers();
                 buffer.SynchronizeMemory(address, size);
 
                 if (write)
@@ -849,14 +970,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
             if (range.Count == 1)
             {
                 MemoryRange subRange = range.GetSubRange(0);
-                SynchronizeBufferRange(subRange.Address, subRange.Size);
+                SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: true);
             }
             else
             {
                 for (int index = 0; index < range.Count; index++)
                 {
                     MemoryRange subRange = range.GetSubRange(index);
-                    SynchronizeBufferRange(subRange.Address, subRange.Size);
+                    SynchronizeBufferRange(subRange.Address, subRange.Size, copyBackVirtual: false);
                 }
             }
         }
@@ -866,12 +987,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </summary>
         /// <param name="address">Start address of the memory range</param>
         /// <param name="size">Size in bytes of the memory range</param>
-        private void SynchronizeBufferRange(ulong address, ulong size)
+        /// <param name="copyBackVirtual">Whether virtual buffers that uses this buffer as backing memory should have its data copied back if modified</param>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private void SynchronizeBufferRange(ulong address, ulong size, bool copyBackVirtual)
         {
             if (size != 0)
             {
                 Buffer buffer = _buffers.FindFirstOverlap(address, size);
 
+                if (copyBackVirtual)
+                {
+                    buffer.CopyFromDependantVirtualBuffers();
+                }
+
                 buffer.SynchronizeMemory(address, size);
             }
         }
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
index 74d52705..30f87813 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
@@ -40,9 +40,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
         internal PhysicalMemory Physical { get; }
 
         /// <summary>
-        /// Virtual buffer cache.
+        /// Virtual range cache.
         /// </summary>
-        internal VirtualBufferCache VirtualBufferCache { get; }
+        internal VirtualRangeCache VirtualRangeCache { get; }
 
         /// <summary>
         /// Cache of GPU counters.
@@ -56,12 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
         internal MemoryManager(PhysicalMemory physicalMemory)
         {
             Physical = physicalMemory;
-            VirtualBufferCache = new VirtualBufferCache(this);
+            VirtualRangeCache = new VirtualRangeCache(this);
             CounterCache = new CounterCache();
             _pageTable = new ulong[PtLvl0Size][];
             MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
             MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
-            MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler;
+            MemoryUnmapped += VirtualRangeCache.MemoryUnmappedHandler;
             MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
         }
 
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
index e039a7a4..d92b0836 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
@@ -1,6 +1,7 @@
 using Ryujinx.Graphics.GAL;
 using Ryujinx.Memory.Range;
 using System;
+using System.Collections.Generic;
 
 namespace Ryujinx.Graphics.Gpu.Memory
 {
@@ -22,11 +23,72 @@ namespace Ryujinx.Graphics.Gpu.Memory
         public MultiRange Range { get; }
 
         /// <summary>
+        /// Ever increasing counter value indicating when the buffer was modified relative to other buffers.
+        /// </summary>
+        public int ModificationSequenceNumber { get; private set; }
+
+        /// <summary>
+        /// Physical buffer dependency entry.
+        /// </summary>
+        private readonly struct PhysicalDependency
+        {
+            /// <summary>
+            /// Physical buffer.
+            /// </summary>
+            public readonly Buffer PhysicalBuffer;
+
+            /// <summary>
+            /// Offset of the range on the physical buffer.
+            /// </summary>
+            public readonly ulong PhysicalOffset;
+
+            /// <summary>
+            /// Offset of the range on the virtual buffer.
+            /// </summary>
+            public readonly ulong VirtualOffset;
+
+            /// <summary>
+            /// Size of the range.
+            /// </summary>
+            public readonly ulong Size;
+
+            /// <summary>
+            /// Creates a new physical dependency.
+            /// </summary>
+            /// <param name="physicalBuffer">Physical buffer</param>
+            /// <param name="physicalOffset">Offset of the range on the physical buffer</param>
+            /// <param name="virtualOffset">Offset of the range on the virtual buffer</param>
+            /// <param name="size">Size of the range</param>
+            public PhysicalDependency(Buffer physicalBuffer, ulong physicalOffset, ulong virtualOffset, ulong size)
+            {
+                PhysicalBuffer = physicalBuffer;
+                PhysicalOffset = physicalOffset;
+                VirtualOffset = virtualOffset;
+                Size = size;
+            }
+        }
+
+        private List<PhysicalDependency> _dependencies;
+        private BufferModifiedRangeList _modifiedRanges = null;
+
+        /// <summary>
+        /// Creates a new instance of the buffer.
+        /// </summary>
+        /// <param name="context">GPU context that the buffer belongs to</param>
+        /// <param name="range">Range of memory where the data is mapped</param>
+        public MultiRangeBuffer(GpuContext context, MultiRange range)
+        {
+            _context = context;
+            Range = range;
+            Handle = context.Renderer.CreateBuffer((int)range.GetSize());
+        }
+
+        /// <summary>
         /// Creates a new instance of the buffer.
         /// </summary>
         /// <param name="context">GPU context that the buffer belongs to</param>
         /// <param name="range">Range of memory where the data is mapped</param>
-        /// <param name="storages">Backing memory for the buffers</param>
+        /// <param name="storages">Backing memory for the buffer</param>
         public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages)
         {
             _context = context;
@@ -50,10 +112,133 @@ namespace Ryujinx.Graphics.Gpu.Memory
         }
 
         /// <summary>
+        /// Removes all physical buffer dependencies.
+        /// </summary>
+        public void ClearPhysicalDependencies()
+        {
+            _dependencies?.Clear();
+        }
+
+        /// <summary>
+        /// Adds a physical buffer dependency.
+        /// </summary>
+        /// <param name="buffer">Physical buffer to be added</param>
+        /// <param name="rangeAddress">Address inside the physical buffer where the virtual buffer range is located</param>
+        /// <param name="dstOffset">Offset inside the virtual buffer where the physical range is located</param>
+        /// <param name="rangeSize">Size of the range in bytes</param>
+        public void AddPhysicalDependency(Buffer buffer, ulong rangeAddress, ulong dstOffset, ulong rangeSize)
+        {
+            (_dependencies ??= new()).Add(new(buffer, rangeAddress - buffer.Address, dstOffset, rangeSize));
+            buffer.AddVirtualDependency(this);
+        }
+
+        /// <summary>
+        /// Tries to get the physical range corresponding to the given physical buffer.
+        /// </summary>
+        /// <param name="buffer">Physical buffer</param>
+        /// <param name="minimumVirtOffset">Minimum virtual offset that a range match can have</param>
+        /// <param name="physicalOffset">Physical offset of the match</param>
+        /// <param name="virtualOffset">Virtual offset of the match, always greater than or equal <paramref name="minimumVirtOffset"/></param>
+        /// <param name="size">Size of the range match</param>
+        /// <returns>True if a match was found for the given parameters, false otherwise</returns>
+        public bool TryGetPhysicalOffset(Buffer buffer, ulong minimumVirtOffset, out ulong physicalOffset, out ulong virtualOffset, out ulong size)
+        {
+            physicalOffset = 0;
+            virtualOffset = 0;
+            size = 0;
+
+            if (_dependencies != null)
+            {
+                foreach (var dependency in _dependencies)
+                {
+                    if (dependency.PhysicalBuffer == buffer && dependency.VirtualOffset >= minimumVirtOffset)
+                    {
+                        physicalOffset = dependency.PhysicalOffset;
+                        virtualOffset = dependency.VirtualOffset;
+                        size = dependency.Size;
+
+                        return true;
+                    }
+                }
+            }
+
+            return false;
+        }
+
+        /// <summary>
+        /// Adds a modified virtual memory range.
+        /// </summary>
+        /// <remarks>
+        /// This is only required when the host does not support sparse buffers, otherwise only physical buffers need to track modification.
+        /// </remarks>
+        /// <param name="range">Modified range</param>
+        /// <param name="modifiedSequenceNumber">ModificationSequenceNumber</param>
+        public void AddModifiedRegion(MultiRange range, int modifiedSequenceNumber)
+        {
+            _modifiedRanges ??= new(_context, null, null);
+
+            for (int i = 0; i < range.Count; i++)
+            {
+                MemoryRange subRange = range.GetSubRange(i);
+
+                _modifiedRanges.SignalModified(subRange.Address, subRange.Size);
+            }
+
+            ModificationSequenceNumber = modifiedSequenceNumber;
+        }
+
+        /// <summary>
+        /// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="buffer"/>.
+        /// </summary>
+        /// <param name="buffer">Buffer to have its range checked</param>
+        /// <param name="rangeAction">Action to perform for modified ranges</param>
+        public void ConsumeModifiedRegion(Buffer buffer, Action<ulong, ulong> rangeAction)
+        {
+            ConsumeModifiedRegion(buffer.Address, buffer.Size, rangeAction);
+        }
+
+        /// <summary>
+        /// Calls the specified <paramref name="rangeAction"/> for all modified ranges that overlaps with <paramref name="address"/> and <paramref name="size"/>.
+        /// </summary>
+        /// <param name="address">Address of the region to consume</param>
+        /// <param name="size">Size of the region to consume</param>
+        /// <param name="rangeAction">Action to perform for modified ranges</param>
+        public void ConsumeModifiedRegion(ulong address, ulong size, Action<ulong, ulong> rangeAction)
+        {
+            if (_modifiedRanges != null)
+            {
+                _modifiedRanges.GetRanges(address, size, rangeAction);
+                _modifiedRanges.Clear(address, size);
+            }
+        }
+
+        /// <summary>
+        /// Gets data from the specified region of the buffer, and places it on <paramref name="output"/>.
+        /// </summary>
+        /// <param name="output">Span to put the data into</param>
+        /// <param name="offset">Offset of the buffer to get the data from</param>
+        /// <param name="size">Size of the data in bytes</param>
+        public void GetData(Span<byte> output, int offset, int size)
+        {
+            using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, size);
+            data.Get().CopyTo(output);
+        }
+
+        /// <summary>
         /// Disposes the host buffer.
         /// </summary>
         public void Dispose()
         {
+            if (_dependencies != null)
+            {
+                foreach (var dependency in _dependencies)
+                {
+                    dependency.PhysicalBuffer.RemoveVirtualDependency(this);
+                }
+
+                _dependencies = null;
+            }
+
             _context.Renderer.DeleteBuffer(Handle);
         }
     }
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs
index 858c5e3b..889f5c9c 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualRangeCache.cs
@@ -6,9 +6,9 @@ using System.Threading;
 namespace Ryujinx.Graphics.Gpu.Memory
 {
     /// <summary>
-    /// Virtual buffer cache.
+    /// Virtual range cache.
     /// </summary>
-    class VirtualBufferCache
+    class VirtualRangeCache
     {
         private readonly MemoryManager _memoryManager;
 
@@ -68,10 +68,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
         private int _hasDeferredUnmaps;
 
         /// <summary>
-        /// Creates a new instance of the virtual buffer cache.
+        /// Creates a new instance of the virtual range cache.
         /// </summary>
-        /// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param>
-        public VirtualBufferCache(MemoryManager memoryManager)
+        /// <param name="memoryManager">Memory manager that the virtual range cache belongs to</param>
+        public VirtualRangeCache(MemoryManager memoryManager)
         {
             _memoryManager = memoryManager;
             _virtualRanges = new RangeList<VirtualRange>();
@@ -102,10 +102,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
         /// </summary>
         /// <param name="gpuVa">GPU virtual address to get the physical range from</param>
         /// <param name="size">Size in bytes of the region</param>
-        /// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param>
         /// <param name="range">Physical range for the specified GPU virtual region</param>
         /// <returns>True if the range already existed, false if a new one was created and added</returns>
-        public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range)
+        public bool TryGetOrAddRange(ulong gpuVa, ulong size, out MultiRange range)
         {
             VirtualRange[] overlaps = _virtualRangeOverlaps;
             int overlapsCount;
@@ -158,7 +157,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
                 }
                 else
                 {
-                    found = true;
+                    found = overlap0.Range.Count == 1 || IsSparseAligned(overlap0.Range);
                     range = overlap0.Range.Slice(gpuVa - overlap0.Address, size);
                 }
             }
@@ -175,11 +174,10 @@ namespace Ryujinx.Graphics.Gpu.Memory
             ShrinkOverlapsBufferIfNeeded();
 
             // If the the range is not properly aligned for sparse mapping,
-            // or if the host does not support sparse mapping, let's just
-            // force it to a single range.
+            // let's just force it to a single range.
             // This might cause issues in some applications that uses sparse
             // mappings.
-            if (!IsSparseAligned(range) || !supportsSparse)
+            if (!IsSparseAligned(range))
             {
                 range = new MultiRange(range.GetSubRange(0).Address, size);
             }
author	gdkchan <gab.dark.100@gmail.com>	2024-02-22 11:03:07 -0300
committer	GitHub <noreply@github.com>	2024-02-22 11:03:07 -0300
commit	167f50bbcd5b2378a038e540877be4d9b71a12f6 (patch)
tree	b425e6529ade9eb52dca3f9ff92ea767a78224b4 /src
parent	ba91f5d401bf5f9fc39aaa30feac2600e82f9c42 (diff)