diff options
27 files changed, 1284 insertions, 135 deletions
diff --git a/src/Ryujinx.Graphics.GAL/BufferAccess.cs b/src/Ryujinx.Graphics.GAL/BufferAccess.cs index 2f7d994f..e7d7ceb0 100644 --- a/src/Ryujinx.Graphics.GAL/BufferAccess.cs +++ b/src/Ryujinx.Graphics.GAL/BufferAccess.cs @@ -4,5 +4,6 @@ namespace Ryujinx.Graphics.GAL { Default, FlushPersistent, + Stream } } diff --git a/src/Ryujinx.Graphics.GAL/BufferRange.cs b/src/Ryujinx.Graphics.GAL/BufferRange.cs index 483747f1..fec82de2 100644 --- a/src/Ryujinx.Graphics.GAL/BufferRange.cs +++ b/src/Ryujinx.Graphics.GAL/BufferRange.cs @@ -10,12 +10,14 @@ namespace Ryujinx.Graphics.GAL public int Offset { get; } public int Size { get; } + public bool Write { get; } - public BufferRange(BufferHandle handle, int offset, int size) + public BufferRange(BufferHandle handle, int offset, int size, bool write = false) { Handle = handle; Offset = offset; Size = size; + Write = write; } } } diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs b/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs index c30df046..6377e5ea 100644 --- a/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs +++ b/src/Ryujinx.Graphics.GAL/Multithreading/BufferMap.cs @@ -113,7 +113,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading internal BufferRange MapBufferRange(BufferRange range) { - return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size); + return new BufferRange(MapBuffer(range.Handle), range.Offset, range.Size, range.Write); } internal Span<BufferRange> MapBufferRanges(Span<BufferRange> ranges) @@ -131,7 +131,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading result = BufferHandle.Null; } - range = new BufferRange(result, range.Offset, range.Size); + range = new BufferRange(result, range.Offset, range.Size, range.Write); } } @@ -154,7 +154,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading result = BufferHandle.Null; } - assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size)); + assignment = new BufferAssignment(ranges[i].Binding, new BufferRange(result, range.Offset, range.Size, range.Write)); } } @@ -176,7 +176,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading result = BufferHandle.Null; } - range = new BufferRange(result, range.Offset, range.Size); + range = new BufferRange(result, range.Offset, range.Size, range.Write); ranges[i] = new VertexBufferDescriptor(range, ranges[i].Stride, ranges[i].Divisor); } diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs b/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs index 022e12f5..6e15fcfa 100644 --- a/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs +++ b/src/Ryujinx.Graphics.Gpu/Engine/Threed/IbStreamer.cs @@ -171,7 +171,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed if (_inlineIndexBuffer == BufferHandle.Null) { - _inlineIndexBuffer = renderer.CreateBuffer(size); + _inlineIndexBuffer = renderer.CreateBuffer(size, BufferAccess.Stream); _inlineIndexBufferSize = size; } else if (_inlineIndexBufferSize < size) @@ -179,7 +179,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed BufferHandle oldBuffer = _inlineIndexBuffer; int oldSize = _inlineIndexBufferSize; - _inlineIndexBuffer = renderer.CreateBuffer(size); + _inlineIndexBuffer = renderer.CreateBuffer(size, BufferAccess.Stream); _inlineIndexBufferSize = size; renderer.Pipeline.CopyBuffer(oldBuffer, _inlineIndexBuffer, 0, 0, oldSize); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index e27c14a1..c9286a61 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -140,18 +140,21 @@ namespace Ryujinx.Graphics.Gpu.Memory } /// <summary> - /// Gets a sub-range from the buffer, from a start address till the end of the buffer. + /// Gets a sub-range from the buffer, from a start address til a page boundary after the given size. /// </summary> /// <remarks> /// This can be used to bind and use sub-ranges of the buffer on the host API. /// </remarks> /// <param name="address">Start address of the sub-range, must be greater than or equal to the buffer address</param> + /// <param name="size">Size in bytes of the sub-range, must be less than or equal to the buffer size</param> + /// <param name="write">Whether the buffer will be written to by this use</param> /// <returns>The buffer sub-range</returns> - public BufferRange GetRange(ulong address) + public BufferRange GetRangeAligned(ulong address, ulong size, bool write) { + ulong end = ((address + size + MemoryManager.PageMask) & ~MemoryManager.PageMask) - Address; ulong offset = address - Address; - return new BufferRange(Handle, (int)offset, (int)(Size - offset)); + return new BufferRange(Handle, (int)offset, (int)(end - offset), write); } /// <summary> @@ -162,12 +165,13 @@ namespace Ryujinx.Graphics.Gpu.Memory /// </remarks> /// <param name="address">Start address of the sub-range, must be greater than or equal to the buffer address</param> /// <param name="size">Size in bytes of the sub-range, must be less than or equal to the buffer size</param> + /// <param name="write">Whether the buffer will be written to by this use</param> /// <returns>The buffer sub-range</returns> - public BufferRange GetRange(ulong address, ulong size) + public BufferRange GetRange(ulong address, ulong size, bool write) { int offset = (int)(address - Address); - return new BufferRange(Handle, offset, (int)size); + return new BufferRange(Handle, offset, (int)size, write); } /// <summary> diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs index f8f572c6..05cc312c 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs @@ -372,15 +372,15 @@ namespace Ryujinx.Graphics.Gpu.Memory } /// <summary> - /// Gets a buffer sub-range starting at a given memory address. + /// Gets a buffer sub-range from a start address til a page boundary after the given size. /// </summary> /// <param name="address">Start address of the memory range</param> /// <param name="size">Size in bytes of the memory range</param> /// <param name="write">Whether the buffer will be written to by this use</param> /// <returns>The buffer sub-range starting at the given memory address</returns> - public BufferRange GetBufferRangeTillEnd(ulong address, ulong size, bool write = false) + public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false) { - return GetBuffer(address, size, write).GetRange(address); + return GetBuffer(address, size, write).GetRangeAligned(address, size, write); } /// <summary> @@ -392,7 +392,7 @@ namespace Ryujinx.Graphics.Gpu.Memory /// <returns>The buffer sub-range for the given range</returns> public BufferRange GetBufferRange(ulong address, ulong size, bool write = false) { - return GetBuffer(address, size, write).GetRange(address, size); + return GetBuffer(address, size, write).GetRange(address, size, write); } /// <summary> diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs index c656b0f6..bf4cb5d0 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs @@ -614,7 +614,7 @@ namespace Ryujinx.Graphics.Gpu.Memory if (_tfInfoBuffer == BufferHandle.Null) { - _tfInfoBuffer = _context.Renderer.CreateBuffer(TfInfoBufferSize); + _tfInfoBuffer = _context.Renderer.CreateBuffer(TfInfoBufferSize, BufferAccess.Stream); } buffers[0] = new BufferAssignment(0, new BufferRange(_tfInfoBuffer, 0, TfInfoBufferSize)); @@ -727,7 +727,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage - ? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) + ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) : bufferCache.GetBufferRange(bounds.Address, bounds.Size); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); @@ -764,7 +764,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write); var range = isStorage - ? bufferCache.GetBufferRangeTillEnd(bounds.Address, bounds.Size, isWrite) + ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite) : bufferCache.GetBufferRange(bounds.Address, bounds.Size); ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range); diff --git a/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs b/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs index 409c7a78..c1e91c54 100644 --- a/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs +++ b/src/Ryujinx.Graphics.Gpu/Memory/SupportBufferUpdater.cs @@ -228,7 +228,7 @@ namespace Ryujinx.Graphics.Gpu.Memory { if (_handle == BufferHandle.Null) { - _handle = _renderer.CreateBuffer(SupportBuffer.RequiredSize); + _handle = _renderer.CreateBuffer(SupportBuffer.RequiredSize, BufferAccess.Stream); _renderer.Pipeline.ClearBuffer(_handle, 0, SupportBuffer.RequiredSize, 0); var range = new BufferRange(_handle, 0, SupportBuffer.RequiredSize); diff --git a/src/Ryujinx.Graphics.Vulkan/Auto.cs b/src/Ryujinx.Graphics.Vulkan/Auto.cs index fdce7232..026dd2b6 100644 --- a/src/Ryujinx.Graphics.Vulkan/Auto.cs +++ b/src/Ryujinx.Graphics.Vulkan/Auto.cs @@ -18,6 +18,12 @@ namespace Ryujinx.Graphics.Vulkan void AddCommandBufferDependencies(CommandBufferScoped cbs); } + interface IMirrorable<T> where T : IDisposable + { + Auto<T> GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored); + void ClearMirrors(CommandBufferScoped cbs, int offset, int size); + } + class Auto<T> : IAutoPrivate, IDisposable where T : IDisposable { private int _referenceCount; @@ -26,6 +32,7 @@ namespace Ryujinx.Graphics.Vulkan private readonly BitMap _cbOwnership; private readonly MultiFenceHolder _waitable; private readonly IAutoPrivate[] _referencedObjs; + private readonly IMirrorable<T> _mirrorable; private bool _disposed; private bool _destroyed; @@ -37,6 +44,11 @@ namespace Ryujinx.Graphics.Vulkan _cbOwnership = new BitMap(CommandBufferPool.MaxCommandBuffers); } + public Auto(T value, IMirrorable<T> mirrorable, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value, waitable, referencedObjs) + { + _mirrorable = mirrorable; + } + public Auto(T value, MultiFenceHolder waitable, params IAutoPrivate[] referencedObjs) : this(value) { _waitable = waitable; @@ -48,9 +60,17 @@ namespace Ryujinx.Graphics.Vulkan } } - public T Get(CommandBufferScoped cbs, int offset, int size) + public T GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + var mirror = _mirrorable.GetMirrorable(cbs, ref offset, size, out mirrored); + mirror._waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, false); + return mirror.Get(cbs); + } + + public T Get(CommandBufferScoped cbs, int offset, int size, bool write = false) { - _waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size); + _mirrorable?.ClearMirrors(cbs, offset, size); + _waitable?.AddBufferUse(cbs.CommandBufferIndex, offset, size, write); return Get(cbs); } diff --git a/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs b/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs new file mode 100644 index 00000000..15672e9c --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/BitMapStruct.cs @@ -0,0 +1,263 @@ +using Ryujinx.Common.Memory; +using System; +using System.Numerics; + +namespace Ryujinx.Graphics.Vulkan +{ + interface IBitMapListener + { + void BitMapSignal(int index, int count); + } + + struct BitMapStruct<T> where T : IArray<long> + { + public const int IntSize = 64; + + private const int IntShift = 6; + private const int IntMask = IntSize - 1; + + private T _masks; + + public BitMapStruct() + { + _masks = default; + } + + public bool BecomesUnsetFrom(in BitMapStruct<T> from, ref BitMapStruct<T> into) + { + bool result = false; + + int masks = _masks.Length; + for (int i = 0; i < masks; i++) + { + long fromMask = from._masks[i]; + long unsetMask = (~fromMask) & (fromMask ^ _masks[i]); + into._masks[i] = unsetMask; + + result |= unsetMask != 0; + } + + return result; + } + + public void SetAndSignalUnset<T2>(in BitMapStruct<T> from, ref T2 listener) where T2 : struct, IBitMapListener + { + BitMapStruct<T> result = new(); + + if (BecomesUnsetFrom(from, ref result)) + { + // Iterate the set bits in the result, and signal them. + + int offset = 0; + int masks = _masks.Length; + ref T resultMasks = ref result._masks; + for (int i = 0; i < masks; i++) + { + long value = resultMasks[i]; + while (value != 0) + { + int bit = BitOperations.TrailingZeroCount((ulong)value); + + listener.BitMapSignal(offset + bit, 1); + + value &= ~(1L << bit); + } + + offset += IntSize; + } + } + + _masks = from._masks; + } + + public void SignalSet(Action<int, int> action) + { + // Iterate the set bits in the result, and signal them. + + int offset = 0; + int masks = _masks.Length; + for (int i = 0; i < masks; i++) + { + long value = _masks[i]; + while (value != 0) + { + int bit = BitOperations.TrailingZeroCount((ulong)value); + + action(offset + bit, 1); + + value &= ~(1L << bit); + } + + offset += IntSize; + } + } + + public bool AnySet() + { + for (int i = 0; i < _masks.Length; i++) + { + if (_masks[i] != 0) + { + return true; + } + } + + return false; + } + + public bool IsSet(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + return (_masks[wordIndex] & wordMask) != 0; + } + + public bool IsSet(int start, int end) + { + if (start == end) + { + return IsSet(start); + } + + int startIndex = start >> IntShift; + int startBit = start & IntMask; + long startMask = -1L << startBit; + + int endIndex = end >> IntShift; + int endBit = end & IntMask; + long endMask = (long)(ulong.MaxValue >> (IntMask - endBit)); + + if (startIndex == endIndex) + { + return (_masks[startIndex] & startMask & endMask) != 0; + } + + if ((_masks[startIndex] & startMask) != 0) + { + return true; + } + + for (int i = startIndex + 1; i < endIndex; i++) + { + if (_masks[i] != 0) + { + return true; + } + } + + if ((_masks[endIndex] & endMask) != 0) + { + return true; + } + + return false; + } + + public bool Set(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + if ((_masks[wordIndex] & wordMask) != 0) + { + return false; + } + + _masks[wordIndex] |= wordMask; + + return true; + } + + public void Set(int bit, bool value) + { + if (value) + { + Set(bit); + } + else + { + Clear(bit); + } + } + + public void SetRange(int start, int end) + { + if (start == end) + { + Set(start); + return; + } + + int startIndex = start >> IntShift; + int startBit = start & IntMask; + long startMask = -1L << startBit; + + int endIndex = end >> IntShift; + int endBit = end & IntMask; + long endMask = (long)(ulong.MaxValue >> (IntMask - endBit)); + + if (startIndex == endIndex) + { + _masks[startIndex] |= startMask & endMask; + } + else + { + _masks[startIndex] |= startMask; + + for (int i = startIndex + 1; i < endIndex; i++) + { + _masks[i] |= -1L; + } + + _masks[endIndex] |= endMask; + } + } + + public BitMapStruct<T> Union(BitMapStruct<T> other) + { + var result = new BitMapStruct<T>(); + + ref var masks = ref _masks; + ref var otherMasks = ref other._masks; + ref var newMasks = ref result._masks; + + for (int i = 0; i < masks.Length; i++) + { + newMasks[i] = masks[i] | otherMasks[i]; + } + + return result; + } + + public void Clear(int bit) + { + int wordIndex = bit >> IntShift; + int wordBit = bit & IntMask; + + long wordMask = 1L << wordBit; + + _masks[wordIndex] &= ~wordMask; + } + + public void Clear() + { + for (int i = 0; i < _masks.Length; i++) + { + _masks[i] = 0; + } + } + + public void ClearInt(int start, int end) + { + for (int i = start; i <= end; i++) + { + _masks[i] = 0; + } + } + } +} diff --git a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs index 54635631..c767a57a 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs @@ -10,7 +10,7 @@ using VkFormat = Silk.NET.Vulkan.Format; namespace Ryujinx.Graphics.Vulkan { - class BufferHolder : IDisposable + class BufferHolder : IDisposable, IMirrorable<DisposableBuffer>, IMirrorable<DisposableBufferView> { private const int MaxUpdateBufferSize = 0x10000; @@ -64,6 +64,11 @@ namespace Ryujinx.Graphics.Vulkan private List<Action> _swapActions; + private byte[] _pendingData; + private BufferMirrorRangeList _pendingDataRanges; + private Dictionary<ulong, StagingBufferReserved> _mirrors; + private bool _useMirrors; + public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType) { _gd = gd; @@ -71,7 +76,7 @@ namespace Ryujinx.Graphics.Vulkan _allocation = allocation; _allocationAuto = new Auto<MemoryAllocation>(allocation); _waitable = new MultiFenceHolder(size); - _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); + _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; Size = size; _map = allocation.HostPointer; @@ -81,6 +86,7 @@ namespace Ryujinx.Graphics.Vulkan DesiredType = currentType; _flushLock = new ReaderWriterLock(); + _useMirrors = gd.IsTBDR; } public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, Auto<MemoryAllocation> allocation, int size, BufferAllocationType type, BufferAllocationType currentType, int offset) @@ -91,7 +97,7 @@ namespace Ryujinx.Graphics.Vulkan _allocationAuto = allocation; _allocationImported = true; _waitable = new MultiFenceHolder(size); - _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, _allocationAuto); + _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; Size = size; _map = _allocation.HostPointer + offset; @@ -110,7 +116,7 @@ namespace Ryujinx.Graphics.Vulkan // Only swap if the buffer is not used in any queued command buffer. bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); - if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld) + if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld && (_pendingData == null || cbs != null)) { var currentAllocation = _allocationAuto; var currentBuffer = _buffer; @@ -120,6 +126,11 @@ namespace Ryujinx.Graphics.Vulkan if (buffer.Handle != 0) { + if (cbs != null) + { + ClearMirrors(cbs.Value, 0, Size); + } + _flushLock.AcquireWriterLock(Timeout.Infinite); ClearFlushFence(); @@ -128,7 +139,7 @@ namespace Ryujinx.Graphics.Vulkan _allocation = allocation; _allocationAuto = new Auto<MemoryAllocation>(allocation); - _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(_gd.Api, _device, buffer), _waitable, _allocationAuto); + _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(_gd.Api, _device, buffer), this, _waitable, _allocationAuto); _bufferHandle = buffer.Handle; _map = allocation.HostPointer; @@ -257,7 +268,7 @@ namespace Ryujinx.Graphics.Vulkan (_swapActions ??= new List<Action>()).Add(invalidateView); - return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer); + return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), this, _waitable, _buffer); } public void InheritMetrics(BufferHolder other) @@ -302,6 +313,82 @@ namespace Ryujinx.Graphics.Vulkan } } + private static ulong ToMirrorKey(int offset, int size) + { + return ((ulong)offset << 32) | (uint)size; + } + + private static (int offset, int size) FromMirrorKey(ulong key) + { + return ((int)(key >> 32), (int)key); + } + + private unsafe bool TryGetMirror(CommandBufferScoped cbs, ref int offset, int size, out Auto<DisposableBuffer> buffer) + { + size = Math.Min(size, Size - offset); + + // Does this binding need to be mirrored? + + if (!_pendingDataRanges.OverlapsWith(offset, size)) + { + buffer = null; + return false; + } + + var key = ToMirrorKey(offset, size); + + if (_mirrors.TryGetValue(key, out StagingBufferReserved reserved)) + { + buffer = reserved.Buffer.GetBuffer(); + offset = reserved.Offset; + + return true; + } + + // Is this mirror allowed to exist? Can't be used for write in any in-flight write. + if (_waitable.IsBufferRangeInUse(offset, size, true)) + { + // Some of the data is not mirrorable, so upload the whole range. + ClearMirrors(cbs, offset, size); + + buffer = null; + return false; + } + + // Build data for the new mirror. + + var baseData = new Span<byte>((void*)(_map + offset), size); + var modData = _pendingData.AsSpan(offset, size); + + StagingBufferReserved? newMirror = _gd.BufferManager.StagingBuffer.TryReserveData(cbs, size, (int)_gd.Capabilities.MinResourceAlignment); + + if (newMirror != null) + { + var mirror = newMirror.Value; + _pendingDataRanges.FillData(baseData, modData, offset, new Span<byte>((void*)(mirror.Buffer._map + mirror.Offset), size)); + + if (_mirrors.Count == 0) + { + _gd.PipelineInternal.RegisterActiveMirror(this); + } + + _mirrors.Add(key, mirror); + + buffer = mirror.Buffer.GetBuffer(); + offset = mirror.Offset; + + return true; + } + else + { + // Data could not be placed on the mirror, likely out of space. Force the data to flush. + ClearMirrors(cbs, offset, size); + + buffer = null; + return false; + } + } + public Auto<DisposableBuffer> GetBuffer() { return _buffer; @@ -339,6 +426,86 @@ namespace Ryujinx.Graphics.Vulkan return _buffer; } + public Auto<DisposableBuffer> GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + if (_pendingData != null && TryGetMirror(cbs, ref offset, size, out Auto<DisposableBuffer> result)) + { + mirrored = true; + return result; + } + + mirrored = false; + return _buffer; + } + + Auto<DisposableBufferView> IMirrorable<DisposableBufferView>.GetMirrorable(CommandBufferScoped cbs, ref int offset, int size, out bool mirrored) + { + // Cannot mirror buffer views right now. + + throw new NotImplementedException(); + } + + public void ClearMirrors() + { + // Clear mirrors without forcing a flush. This happens when the command buffer is switched, + // as all reserved areas on the staging buffer are released. + + if (_pendingData != null) + { + _mirrors.Clear(); + }; + } + + public void ClearMirrors(CommandBufferScoped cbs, int offset, int size) + { + // Clear mirrors in the given range, and submit overlapping pending data. + + if (_pendingData != null) + { + bool hadMirrors = _mirrors.Count > 0 && RemoveOverlappingMirrors(offset, size); + + if (_pendingDataRanges.Count() != 0) + { + UploadPendingData(cbs, offset, size); + } + + if (hadMirrors) + { + _gd.PipelineInternal.Rebind(_buffer, offset, size); + } + }; + } + + public void UseMirrors() + { + _useMirrors = true; + } + + private void UploadPendingData(CommandBufferScoped cbs, int offset, int size) + { + var ranges = _pendingDataRanges.FindOverlaps(offset, size); + + if (ranges != null) + { + _pendingDataRanges.Remove(offset, size); + + foreach (var range in ranges) + { + int rangeOffset = Math.Max(offset, range.Offset); + int rangeSize = Math.Min(offset + size, range.End) - rangeOffset; + + if (_gd.PipelineInternal.CurrentCommandBuffer.CommandBuffer.Handle == cbs.CommandBuffer.Handle) + { + SetData(rangeOffset, _pendingData.AsSpan(rangeOffset, rangeSize), cbs, _gd.PipelineInternal.EndRenderPass, false); + } + else + { + SetData(rangeOffset, _pendingData.AsSpan(rangeOffset, rangeSize), cbs, null, false); + } + } + } + } + public void SignalWrite(int offset, int size) { ConsiderBackingSwap(); @@ -472,7 +639,34 @@ namespace Ryujinx.Graphics.Vulkan throw new InvalidOperationException("The buffer is not host mapped."); } - public unsafe void SetData(int offset, ReadOnlySpan<byte> data, CommandBufferScoped? cbs = null, Action endRenderPass = null) + public bool RemoveOverlappingMirrors(int offset, int size) + { + List<ulong> toRemove = null; + foreach (var key in _mirrors.Keys) + { + (int keyOffset, int keySize) = FromMirrorKey(key); + if (!(offset + size <= keyOffset || offset >= keyOffset + keySize)) + { + toRemove ??= new List<ulong>(); + + toRemove.Add(key); + } + } + + if (toRemove != null) + { + foreach (var key in toRemove) + { + _mirrors.Remove(key); + } + + return true; + } + + return false; + } + + public unsafe void SetData(int offset, ReadOnlySpan<byte> data, CommandBufferScoped? cbs = null, Action endRenderPass = null, bool allowCbsWait = true) { int dataSize = Math.Min(data.Length, Size - offset); if (dataSize == 0) @@ -481,6 +675,7 @@ namespace Ryujinx.Graphics.Vulkan } _setCount++; + bool allowMirror = _useMirrors && allowCbsWait && cbs != null && _currentType <= BufferAllocationType.HostMapped; if (_map != IntPtr.Zero) { @@ -488,7 +683,7 @@ namespace Ryujinx.Graphics.Vulkan bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); // If the buffer is rented, take a little more time and check if the use overlaps this handle. - bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize); + bool needsFlush = isRented && _waitable.IsBufferRangeInUse(offset, dataSize, false); if (!needsFlush) { @@ -496,12 +691,48 @@ namespace Ryujinx.Graphics.Vulkan data[..dataSize].CopyTo(new Span<byte>((void*)(_map + offset), dataSize)); + if (_pendingData != null) + { + bool removed = _pendingDataRanges.Remove(offset, dataSize); + if (RemoveOverlappingMirrors(offset, dataSize) || removed) + { + // If any mirrors were removed, rebind the buffer range. + _gd.PipelineInternal.Rebind(_buffer, offset, dataSize); + } + } + SignalWrite(offset, dataSize); return; } } + // If the buffer does not have an in-flight write (including an inline update), then upload data to a pendingCopy. + if (allowMirror && !_waitable.IsBufferRangeInUse(offset, dataSize, true)) + { + if (_pendingData == null) + { + _pendingData = new byte[Size]; + _mirrors = new Dictionary<ulong, StagingBufferReserved>(); + } + + data[..dataSize].CopyTo(_pendingData.AsSpan(offset, dataSize)); + _pendingDataRanges.Add(offset, dataSize); + + // Remove any overlapping mirrors. + RemoveOverlappingMirrors(offset, dataSize); + + // Tell the graphics device to rebind any constant buffer that overlaps the newly modified range, as it should access a mirror. + _gd.PipelineInternal.Rebind(_buffer, offset, dataSize); + + return; + } + + if (_pendingData != null) + { + _pendingDataRanges.Remove(offset, dataSize); + } + if (cbs != null && _gd.PipelineInternal.RenderPassActive && !(_buffer.HasCommandBufferDependency(cbs.Value) && @@ -519,7 +750,37 @@ namespace Ryujinx.Graphics.Vulkan data.Length > MaxUpdateBufferSize || !TryPushData(cbs.Value, endRenderPass, offset, data)) { - _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data); + if (allowCbsWait) + { + _gd.BufferManager.StagingBuffer.PushData(_gd.CommandBufferPool, cbs, endRenderPass, this, offset, data); + } + else + { + bool rentCbs = cbs == null; + if (rentCbs) + { + cbs = _gd.CommandBufferPool.Rent(); + } + + if (!_gd.BufferManager.StagingBuffer.TryPushData(cbs.Value, endRenderPass, this, offset, data)) + { + // Need to do a slow upload. + BufferHolder srcHolder = _gd.BufferManager.Create(_gd, dataSize, baseType: BufferAllocationType.HostMapped); + srcHolder.SetDataUnchecked(0, data); + + var srcBuffer = srcHolder.GetBuffer(); + var dstBuffer = this.GetBuffer(cbs.Value.CommandBuffer, true); + + Copy(_gd, cbs.Value, srcBuffer, dstBuffer, 0, offset, dataSize); + + srcHolder.Dispose(); + } + + if (rentCbs) + { + cbs.Value.Dispose(); + } + } } } @@ -558,7 +819,7 @@ namespace Ryujinx.Graphics.Vulkan endRenderPass?.Invoke(); - var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value; + var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length, true).Value; _writeCount--; @@ -608,7 +869,7 @@ namespace Ryujinx.Graphics.Vulkan bool registerSrcUsage = true) { var srcBuffer = registerSrcUsage ? src.Get(cbs, srcOffset, size).Value : src.GetUnsafe().Value; - var dstBuffer = dst.Get(cbs, dstOffset, size).Value; + var dstBuffer = dst.Get(cbs, dstOffset, size, true).Value; InsertBufferBarrier( gd, diff --git a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs index b916a1ef..38096702 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs @@ -100,9 +100,10 @@ namespace Ryujinx.Graphics.Vulkan VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, - BufferHandle storageHint = default) + BufferHandle storageHint = default, + bool forceMirrors = false) { - return CreateWithHandle(gd, size, out _, baseType, storageHint); + return CreateWithHandle(gd, size, out _, baseType, storageHint, forceMirrors); } public BufferHandle CreateWithHandle( @@ -110,7 +111,8 @@ namespace Ryujinx.Graphics.Vulkan int size, out BufferHolder holder, BufferAllocationType baseType = BufferAllocationType.HostMapped, - BufferHandle storageHint = default) + BufferHandle storageHint = default, + bool forceMirrors = false) { holder = Create(gd, size, baseType: baseType, storageHint: storageHint); if (holder == null) @@ -118,6 +120,11 @@ namespace Ryujinx.Graphics.Vulkan return BufferHandle.Null; } + if (forceMirrors) + { + holder.UseMirrors(); + } + BufferCount++; ulong handle64 = (uint)_buffers.Add(holder); diff --git a/src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs b/src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs new file mode 100644 index 00000000..9e0b7244 --- /dev/null +++ b/src/Ryujinx.Graphics.Vulkan/BufferMirrorRangeList.cs @@ -0,0 +1,305 @@ +using System; +using System.Collections.Generic; + +namespace Ryujinx.Graphics.Vulkan +{ + /// <summary> + /// A structure tracking pending upload ranges for buffers. + /// Where a range is present, pending data exists that can either be used to build mirrors + /// or upload directly to the buffer. + /// </summary> + struct BufferMirrorRangeList + { + internal readonly struct Range + { + public int Offset { get; } + public int Size { get; } + + public int End => Offset + Size; + + public Range(int offset, int size) + { + Offset = offset; + Size = size; + } + + public bool OverlapsWith(int offset, int size) + { + return Offset < offset + size && offset < Offset + Size; + } + } + + private List<Range> _ranges; + + public readonly IEnumerable<Range> All() + { + return _ranges; + } + + public readonly bool Remove(int offset, int size) + { + var list = _ranges; + bool removedAny = false; + if (list != null) + { + int overlapIndex = BinarySearch(list, offset, size); + + if (overlapIndex >= 0) + { + // Overlaps with a range. Search back to find the first one it doesn't overlap with. + + while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size)) + { + overlapIndex--; + } + + int endOffset = offset + size; + int startIndex = overlapIndex; + + var currentOverlap = list[overlapIndex]; + + // Orphan the start of the overlap. + if (currentOverlap.Offset < offset) + { + list[overlapIndex] = new Range(currentOverlap.Offset, offset - currentOverlap.Offset); + currentOverlap = new Range(offset, currentOverlap.End - offset); + list.Insert(++overlapIndex, currentOverlap); + startIndex++; + + removedAny = true; + } + + // Remove any middle overlaps. + while (currentOverlap.Offset < endOffset) + { + if (currentOverlap.End > endOffset) + { + // Update the end overlap instead of removing it, if it spans beyond the removed range. + list[overlapIndex] = new Range(endOffset, currentOverlap.End - endOffset); + + removedAny = true; + break; + } + + if (++overlapIndex >= list.Count) + { + break; + } + + currentOverlap = list[overlapIndex]; + } + + int count = overlapIndex - startIndex; + + list.RemoveRange(startIndex, count); + + removedAny |= count > 0; + } + } + + return removedAny; + } + + public void Add(int offset, int size) + { + var list = _ranges; + if (list != null) + { + int overlapIndex = BinarySearch(list, offset, size); + if (overlapIndex >= 0) + { + while (overlapIndex > 0 && list[overlapIndex - 1].OverlapsWith(offset, size)) + { + overlapIndex--; + } + + int endOffset = offset + size; + int startIndex = overlapIndex; + + while (overlapIndex < list.Count && list[overlapIndex].OverlapsWith(offset, size)) + { + var currentOverlap = list[overlapIndex]; + var currentOverlapEndOffset = currentOverlap.Offset + currentOverlap.Size; + + if (offset > currentOverlap.Offset) + { + offset = currentOverlap.Offset; + } + + if (endOffset < currentOverlapEndOffset) + { + endOffset = currentOverlapEndOffset; + } + + overlapIndex++; + size = endOffset - offset; + } + + int count = overlapIndex - startIndex; + + list.RemoveRange(startIndex, count); + + overlapIndex = startIndex; + } + else + { + overlapIndex = ~overlapIndex; + } + + list.Insert(overlapIndex, new Range(offset, size)); + } + else + { + _ranges = new List<Range> + { + new Range(offset, size) + }; + } + } + + public readonly bool OverlapsWith(int offset, int size) + { + var list = _ranges; + if (list == null) + { + return false; + } + + return BinarySearch(list, offset, size) >= 0; + } + + public readonly List<Range> FindOverlaps(int offset, int size) + { + var list = _ranges; + if (list == null) + { + return null; + } + + List<Range> result = null; + + int index = BinarySearch(list, offset, size); + + if (index >= 0) + { + while (index > 0 && list[index - 1].OverlapsWith(offset, size)) + { + index--; + } + + do + { + (result ??= new List<Range>()).Add(list[index++]); + } + while (index < list.Count && list[index].OverlapsWith(offset, size)); + } + + return result; + } + + private static int BinarySearch(List<Range> list, int offset, int size) + { + int left = 0; + int right = list.Count - 1; + + while (left <= right) + { + int range = right - left; + + int middle = left + (range >> 1); + + var item = list[middle]; + + if (item.OverlapsWith(offset, size)) + { + return middle; + } + + if (offset < item.Offset) + { + right = middle - 1; + } + else + { + left = middle + 1; + } + } + + return ~left; + } + + public readonly void FillData(Span<byte> baseData, Span<byte> modData, int offset, Span<byte> result) + { + int size = baseData.Length; + int endOffset = offset + size; + + var list = _ranges; + if (list == null) + { + baseData.CopyTo(result); + } + + int srcOffset = offset; + int dstOffset = 0; + bool activeRange = false; + + for (int i = 0; i < list.Count; i++) + { + var range = list[i]; + + int rangeEnd = range.Offset + range.Size; + + if (activeRange) + { + if (range.Offset >= endOffset) + { + break; + } + } + else + { + if (rangeEnd <= offset) + { + continue; + } + + activeRange = true; + } + + int baseSize = range.Offset - srcOffset; + + if (baseSize > 0) + { + baseData.Slice(dstOffset, baseSize).CopyTo(result.Slice(dstOffset, baseSize)); + srcOffset += baseSize; + dstOffset += baseSize; + } + + int modSize = Math.Min(rangeEnd - srcOffset, endOffset - srcOffset); + if (modSize != 0) + { + modData.Slice(dstOffset, modSize).CopyTo(result.Slice(dstOffset, modSize)); + srcOffset += modSize; + dstOffset += modSize; + } + } + + int baseSizeEnd = endOffset - srcOffset; + + if (baseSizeEnd > 0) + { + baseData.Slice(dstOffset, baseSizeEnd).CopyTo(result.Slice(dstOffset, baseSizeEnd)); + } + } + + public readonly int Count() + { + return _ranges?.Count ?? 0; + } + + public void Clear() + { + _ranges = null; + } + } +} diff --git a/src/Ryujinx.Graphics.Vulkan/BufferState.cs b/src/Ryujinx.Graphics.Vulkan/BufferState.cs index ee4badd2..198ee54d 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferState.cs @@ -23,7 +23,7 @@ namespace Ryujinx.Graphics.Vulkan { if (_buffer != null) { - var buffer = _buffer.Get(cbs, _offset, _size).Value; + var buffer = _buffer.Get(cbs, _offset, _size, true).Value; gd.TransformFeedbackApi.CmdBindTransformFeedbackBuffers(cbs.CommandBuffer, binding, 1, buffer, (ulong)_offset, (ulong)_size); } @@ -40,6 +40,11 @@ namespace Ryujinx.Graphics.Vulkan } } + public readonly bool Overlaps(Auto<DisposableBuffer> buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } + public readonly void Dispose() { _buffer?.DecrementReferenceCount(); diff --git a/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs b/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs index a8ff7c28..19dcaccd 100644 --- a/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs +++ b/src/Ryujinx.Graphics.Vulkan/BufferUsageBitmap.cs @@ -6,6 +6,7 @@ private readonly int _size; private readonly int _granularity; private readonly int _bits; + private readonly int _writeBitOffset; private readonly int _intsPerCb; private readonly int _bitsPerCb; @@ -14,7 +15,11 @@ { _size = size; _granularity = granularity; - _bits = (size + (granularity - 1)) / granularity; + + // There are two sets of bits - one for read tracking, and the other for write. + int bits = (size + (granularity - 1)) / granularity; + _writeBitOffset = bits; + _bits = bits << 1; _intsPerCb = (_bits + (BitMap.IntSize - 1)) / BitMap.IntSize; _bitsPerCb = _intsPerCb * BitMap.IntSize; @@ -22,7 +27,7 @@ _bitmap = new BitMap(_bitsPerCb * CommandBufferPool.MaxCommandBuffers); } - public void Add(int cbIndex, int offset, int size) + public void Add(int cbIndex, int offset, int size, bool write) { if (size == 0) { @@ -35,32 +40,32 @@ size = _size - offset; } - int cbBase = cbIndex * _bitsPerCb; + int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0); int start = cbBase + offset / _granularity; int end = cbBase + (offset + size - 1) / _granularity; _bitmap.SetRange(start, end); } - public bool OverlapsWith(int cbIndex, int offset, int size) + public bool OverlapsWith(int cbIndex, int offset, int size, bool write = false) { if (size == 0) { return false; } - int cbBase = cbIndex * _bitsPerCb; + int cbBase = cbIndex * _bitsPerCb + (write ? _writeBitOffset : 0); int start = cbBase + offset / _granularity; int end = cbBase + (offset + size - 1) / _granularity; return _bitmap.IsSet(start, end); } - public bool OverlapsWith(int offset, int size) + public bool OverlapsWith(int offset, int size, bool write) { for (int i = 0; i < CommandBufferPool.MaxCommandBuffers; i++) { - if (OverlapsWith(i, offset, size)) + if (OverlapsWith(i, offset, size, write)) { return true; } diff --git a/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs b/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs index 45392b64..14e4c02f 100644 --- a/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs +++ b/src/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs @@ -1,9 +1,9 @@ -using Ryujinx.Graphics.GAL; +using Ryujinx.Common.Memory; +using Ryujinx.Graphics.GAL; using Ryujinx.Graphics.Shader; using Silk.NET.Vulkan; using System; using System.Runtime.CompilerServices; -using Buffer = Silk.NET.Vulkan.Buffer; using CompareOp = Ryujinx.Graphics.GAL.CompareOp; using Format = Ryujinx.Graphics.GAL.Format; using SamplerCreateInfo = Ryujinx.Graphics.GAL.SamplerCreateInfo; @@ -12,13 +12,34 @@ namespace Ryujinx.Graphics.Vulkan { class DescriptorSetUpdater { + private const ulong StorageBufferMaxMirrorable = 0x2000; + private record struct BufferRef + { + public Auto<DisposableBuffer> Buffer; + public int Offset; + public bool Write; + + public BufferRef(Auto<DisposableBuffer> buffer) + { + Buffer = buffer; + Offset = 0; + Write = true; + } + + public BufferRef(Auto<DisposableBuffer> buffer, ref BufferRange range) + { + Buffer = buffer; + Offset = range.Offset; + Write = range.Write; + } + } + private readonly VulkanRenderer _gd; private readonly PipelineBase _pipeline; - private ShaderCollection _program; - private readonly Auto<DisposableBuffer>[] _uniformBufferRefs; - private readonly Auto<DisposableBuffer>[] _storageBufferRefs; + private readonly BufferRef[] _uniformBufferRefs; + private readonly BufferRef[] _storageBufferRefs; private readonly Auto<DisposableImageView>[] _textureRefs; private readonly Auto<DisposableSampler>[] _samplerRefs; private readonly Auto<DisposableImageView>[] _imageRefs; @@ -33,8 +54,10 @@ namespace Ryujinx.Graphics.Vulkan private readonly BufferView[] _bufferTextures; private readonly BufferView[] _bufferImages; - private readonly bool[] _uniformSet; - private readonly bool[] _storageSet; + private BitMapStruct<Array2<long>> _uniformSet; + private BitMapStruct<Array2<long>> _storageSet; + private BitMapStruct<Array2<long>> _uniformMirrored; + private BitMapStruct<Array2<long>> _storageMirrored; [Flags] private enum DirtyFlags @@ -61,8 +84,8 @@ namespace Ryujinx.Graphics.Vulkan // Some of the bindings counts needs to be multiplied by 2 because we have buffer and // regular textures/images interleaved on the same descriptor set. - _uniformBufferRefs = new Auto<DisposableBuffer>[Constants.MaxUniformBufferBindings]; - _storageBufferRefs = new Auto<DisposableBuffer>[Constants.MaxStorageBufferBindings]; + _uniformBufferRefs = new BufferRef[Constants.MaxUniformBufferBindings]; + _storageBufferRefs = new BufferRef[Constants.MaxStorageBufferBindings]; _textureRefs = new Auto<DisposableImageView>[Constants.MaxTextureBindings * 2]; _samplerRefs = new Auto<DisposableSampler>[Constants.MaxTextureBindings * 2]; _imageRefs = new Auto<DisposableImageView>[Constants.MaxImageBindings * 2]; @@ -85,9 +108,6 @@ namespace Ryujinx.Graphics.Vulkan _textures.AsSpan().Fill(initialImageInfo); _images.AsSpan().Fill(initialImageInfo); - _uniformSet = new bool[Constants.MaxUniformBufferBindings]; - _storageSet = new bool[Constants.MaxStorageBufferBindings]; - if (gd.Capabilities.SupportsNullDescriptors) { // If null descriptors are supported, we can pass null as the handle. @@ -138,6 +158,63 @@ namespace Ryujinx.Graphics.Vulkan _dummyTexture.SetData(dummyTextureData); } + private static bool BindingOverlaps(ref DescriptorBufferInfo info, int bindingOffset, int offset, int size) + { + return offset < bindingOffset + (int)info.Range && (offset + size) > bindingOffset; + } + + internal void Rebind(Auto<DisposableBuffer> buffer, int offset, int size) + { + if (_program == null) + { + return; + } + + // Check stage bindings + + _uniformMirrored.Union(_uniformSet).SignalSet((int binding, int count) => + { + for (int i = 0; i < count; i++) + { + ref BufferRef bufferRef = ref _uniformBufferRefs[binding]; + if (bufferRef.Buffer == buffer) + { + ref DescriptorBufferInfo info = ref _uniformBuffers[binding]; + int bindingOffset = bufferRef.Offset; + + if (BindingOverlaps(ref info, bindingOffset, offset, size)) + { + _uniformSet.Clear(binding); + SignalDirty(DirtyFlags.Uniform); + } + } + + binding++; + } + }); + + _storageMirrored.Union(_storageSet).SignalSet((int binding, int count) => + { + for (int i = 0; i < count; i++) + { + ref BufferRef bufferRef = ref _storageBufferRefs[binding]; + if (bufferRef.Buffer == buffer) + { + ref DescriptorBufferInfo info = ref _storageBuffers[binding]; + int bindingOffset = bufferRef.Offset; + + if (BindingOverlaps(ref info, bindingOffset, offset, size)) + { + _storageSet.Clear(binding); + SignalDirty(DirtyFlags.Storage); + } + } + + binding++; + } + }); + } + public void SetProgram(ShaderCollection program) { _program = program; @@ -180,22 +257,28 @@ namespace Ryujinx.Graphics.Vulkan var buffer = assignment.Range; int index = assignment.Binding; - Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false, isSSBO: true); - ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index]; + Auto<DisposableBuffer> vkBuffer = buffer.Handle == BufferHandle.Null + ? null + : _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, buffer.Write, isSSBO: true); + + ref BufferRef currentBufferRef = ref _storageBufferRefs[index]; DescriptorBufferInfo info = new() { Offset = (ulong)buffer.Offset, Range = (ulong)buffer.Size, }; + + var newRef = new BufferRef(vkBuffer, ref buffer); + ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _storageSet[index] = false; + _storageSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -209,21 +292,24 @@ namespace Ryujinx.Graphics.Vulkan var vkBuffer = buffers[i]; int index = first + i; - ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index]; + ref BufferRef currentBufferRef = ref _storageBufferRefs[index]; DescriptorBufferInfo info = new() { Offset = 0, Range = Vk.WholeSize, }; + + BufferRef newRef = new(vkBuffer); + ref DescriptorBufferInfo currentInfo = ref _storageBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _storageSet[index] = false; + _storageSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -288,22 +374,28 @@ namespace Ryujinx.Graphics.Vulkan var buffer = assignment.Range; int index = assignment.Binding; - Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); - ref Auto<DisposableBuffer> currentVkBuffer = ref _uniformBufferRefs[index]; + Auto<DisposableBuffer> vkBuffer = buffer.Handle == BufferHandle.Null + ? null + : _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); + + ref BufferRef currentBufferRef = ref _uniformBufferRefs[index]; DescriptorBufferInfo info = new() { Offset = (ulong)buffer.Offset, Range = (ulong)buffer.Size, }; + + BufferRef newRef = new(vkBuffer, ref buffer); + ref DescriptorBufferInfo currentInfo = ref _uniformBuffers[index]; - if (vkBuffer != currentVkBuffer || currentInfo.Offset != info.Offset || currentInfo.Range != info.Range) + if (!currentBufferRef.Equals(newRef) || currentInfo.Range != info.Range) { - _uniformSet[index] = false; + _uniformSet.Clear(index); currentInfo = info; - currentVkBuffer = vkBuffer; + currentBufferRef = newRef; } } @@ -353,13 +445,26 @@ namespace Ryujinx.Graphics.Vulkan } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static void UpdateBuffer( + private static bool UpdateBuffer( CommandBufferScoped cbs, ref DescriptorBufferInfo info, - Auto<DisposableBuffer> buffer, - Auto<DisposableBuffer> dummyBuffer) + ref BufferRef buffer, + Auto<DisposableBuffer> dummyBuffer, + bool mirrorable) { - info.Buffer = buffer?.Get(cbs, (int)info.Offset, (int)info.Range).Value ?? default; + int offset = buffer.Offset; + bool mirrored = false; + + if (mirrorable) + { + info.Buffer = buffer.Buffer?.GetMirrorable(cbs, ref offset, (int)info.Range, out mirrored).Value ?? default; + } + else + { + info.Buffer = buffer.Buffer?.Get(cbs, offset, (int)info.Range, buffer.Write).Value ?? default; + } + + info.Offset = (ulong)offset; // The spec requires that buffers with null handle have offset as 0 and range as VK_WHOLE_SIZE. if (info.Buffer.Handle == 0) @@ -368,6 +473,8 @@ namespace Ryujinx.Graphics.Vulkan info.Offset = 0; info.Range = Vk.WholeSize; } + + return mirrored; } [MethodImpl(MethodImplOptions.AggressiveInlining)] @@ -404,11 +511,13 @@ namespace Ryujinx.Graphics.Vulkan { int index = binding + i; - if (!_uniformSet[index]) + if (_uniformSet.Set(index)) { - UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); + ref BufferRef buffer = ref _uniformBufferRefs[index]; - _uniformSet[index] = true; + bool mirrored = UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true); + + _uniformMirrored.Set(index, mirrored); } } @@ -421,11 +530,19 @@ namespace Ryujinx.Graphics.Vulkan { int index = binding + i; - if (!_storageSet[index]) + ref BufferRef buffer = ref _storageBufferRefs[index]; + + if (_storageSet.Set(index)) { - UpdateBuffer(cbs, ref _storageBuffers[index], _storageBufferRefs[index], dummyBuffer); + ref var info = ref _storageBuffers[index]; + + bool mirrored = UpdateBuffer(cbs, + ref info, + ref _storageBufferRefs[index], + dummyBuffer, + !buffer.Write && info.Range <= StorageBufferMaxMirrorable); - _storageSet[index] = true; + _storageMirrored.Set(index, mirrored); } } @@ -464,7 +581,7 @@ namespace Ryujinx.Graphics.Vulkan for (int i = 0; i < count; i++) { - bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs) ?? default; + bufferTextures[i] = _bufferTextureRefs[binding + i]?.GetBufferView(cbs, false) ?? default; } dsc.UpdateBufferImages(0, binding, bufferTextures[..count], DescriptorType.UniformTexelBuffer); @@ -489,7 +606,7 @@ namespace Ryujinx.Graphics.Vulkan for (int i = 0; i < count; i++) { - bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i]) ?? default; + bufferImages[i] = _bufferImageRefs[binding + i]?.GetBufferView(cbs, _bufferImageFormats[binding + i], true) ?? default; } dsc.UpdateBufferImages(0, binding, bufferImages[..count], DescriptorType.StorageTexelBuffer); @@ -546,10 +663,10 @@ namespace Ryujinx.Graphics.Vulkan { int index = binding + i; - if (!_uniformSet[index]) + if (_uniformSet.Set(index)) { - UpdateBuffer(cbs, ref _uniformBuffers[index], _uniformBufferRefs[index], dummyBuffer); - _uniformSet[index] = true; + ref BufferRef buffer = ref _uniformBufferRefs[index]; + UpdateBuffer(cbs, ref _uniformBuffers[index], ref buffer, dummyBuffer, true); doUpdate = true; } } @@ -582,17 +699,17 @@ namespace Ryujinx.Graphics.Vulkan { _dirty = DirtyFlags.All; - Array.Clear(_uniformSet); - Array.Clear(_storageSet); + _uniformSet.Clear(); + _storageSet.Clear(); } - private static void SwapBuffer(Auto<DisposableBuffer>[] list, Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) + private static void SwapBuffer(BufferRef[] list, Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) { for (int i = 0; i < list.Length; i++) { - if (list[i] == from) + if (list[i].Buffer == from) { - list[i] = to; + list[i].Buffer = to; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs index 9323fcf9..f478c58e 100644 --- a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs +++ b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs @@ -427,6 +427,7 @@ namespace Ryujinx.Graphics.Vulkan return access switch { BufferAccess.FlushPersistent => BufferAllocationType.HostMapped, + BufferAccess.Stream => BufferAllocationType.HostMapped, _ => BufferAllocationType.Auto, }; } diff --git a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs index 11d9c4fb..e76a332f 100644 --- a/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs +++ b/src/Ryujinx.Graphics.Vulkan/HardwareCapabilities.cs @@ -52,6 +52,7 @@ namespace Ryujinx.Graphics.Vulkan public readonly PortabilitySubsetFlags PortabilitySubset; public readonly uint VertexBufferAlignment; public readonly uint SubTexelPrecisionBits; + public readonly ulong MinResourceAlignment; public HardwareCapabilities( bool supportsIndexTypeUint8, @@ -89,7 +90,8 @@ namespace Ryujinx.Graphics.Vulkan SampleCountFlags supportedSampleCounts, PortabilitySubsetFlags portabilitySubset, uint vertexBufferAlignment, - uint subTexelPrecisionBits) + uint subTexelPrecisionBits, + ulong minResourceAlignment) { SupportsIndexTypeUint8 = supportsIndexTypeUint8; SupportsCustomBorderColor = supportsCustomBorderColor; @@ -127,6 +129,7 @@ namespace Ryujinx.Graphics.Vulkan PortabilitySubset = portabilitySubset; VertexBufferAlignment = vertexBufferAlignment; SubTexelPrecisionBits = subTexelPrecisionBits; + MinResourceAlignment = minResourceAlignment; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs b/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs index b839619e..b85f0c7f 100644 --- a/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/IndexBufferState.cs @@ -5,6 +5,8 @@ namespace Ryujinx.Graphics.Vulkan { internal struct IndexBufferState { + private const int IndexBufferMaxMirrorable = 0x20000; + public static IndexBufferState Null => new(BufferHandle.Null, 0, 0); private readonly int _offset; @@ -37,6 +39,7 @@ namespace Ryujinx.Graphics.Vulkan Auto<DisposableBuffer> autoBuffer; int offset, size; IndexType type = _type; + bool mirrorable = false; if (_type == IndexType.Uint8Ext && !gd.Capabilities.SupportsIndexTypeUint8) { @@ -56,6 +59,8 @@ namespace Ryujinx.Graphics.Vulkan autoBuffer = null; } + mirrorable = _size < IndexBufferMaxMirrorable; + offset = _offset; size = _size; } @@ -64,7 +69,9 @@ namespace Ryujinx.Graphics.Vulkan if (autoBuffer != null) { - gd.Api.CmdBindIndexBuffer(cbs.CommandBuffer, autoBuffer.Get(cbs, offset, size).Value, (ulong)offset, type); + DisposableBuffer buffer = mirrorable ? autoBuffer.GetMirrorable(cbs, ref offset, size, out _) : autoBuffer.Get(cbs, offset, size); + + gd.Api.CmdBindIndexBuffer(cbs.CommandBuffer, buffer.Value, (ulong)offset, type); } } @@ -155,5 +162,10 @@ namespace Ryujinx.Graphics.Vulkan _buffer = to; } } + + public readonly bool Overlaps(Auto<DisposableBuffer> buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } } } diff --git a/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs b/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs index 71769c5e..4d2d312f 100644 --- a/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs +++ b/src/Ryujinx.Graphics.Vulkan/MultiFenceHolder.cs @@ -32,14 +32,20 @@ namespace Ryujinx.Graphics.Vulkan } /// <summary> - /// Adds buffer usage information to the uses list. + /// Adds read/write buffer usage information to the uses list. /// </summary> /// <param name="cbIndex">Index of the command buffer where the buffer is used</param> /// <param name="offset">Offset of the buffer being used</param> /// <param name="size">Size of the buffer region being used, in bytes</param> - public void AddBufferUse(int cbIndex, int offset, int size) + /// <param name="write">Whether the access is a write or not</param> + public void AddBufferUse(int cbIndex, int offset, int size, bool write) { - _bufferUsageBitmap.Add(cbIndex, offset, size); + _bufferUsageBitmap.Add(cbIndex, offset, size, false); + + if (write) + { + _bufferUsageBitmap.Add(cbIndex, offset, size, true); + } } /// <summary> @@ -68,10 +74,11 @@ namespace Ryujinx.Graphics.Vulkan /// </summary> /// <param name="offset">Offset of the buffer being used</param> /// <param name="size">Size of the buffer region being used, in bytes</param> + /// <param name="write">True if only write usages should count</param> /// <returns>True if in use, false otherwise</returns> - public bool IsBufferRangeInUse(int offset, int size) + public bool IsBufferRangeInUse(int offset, int size, bool write) { - return _bufferUsageBitmap.OverlapsWith(offset, size); + return _bufferUsageBitmap.OverlapsWith(offset, size, write); } /// <summary> diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs index 8b931e52..67b16ec9 100644 --- a/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs +++ b/src/Ryujinx.Graphics.Vulkan/PipelineBase.cs @@ -193,7 +193,7 @@ namespace Ryujinx.Graphics.Vulkan { EndRenderPass(); - var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size).Value; + var dst = Gd.BufferManager.GetBuffer(CommandBuffer, destination, offset, size, true).Get(Cbs, offset, size, true).Value; BufferHolder.InsertBufferBarrier( Gd, @@ -469,6 +469,10 @@ namespace Ryujinx.Graphics.Vulkan return; } + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; + UpdateIndexBufferPattern(); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); @@ -498,10 +502,6 @@ namespace Ryujinx.Graphics.Vulkan } else { - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - ResumeTransformFeedbackInternal(); Gd.Api.CmdDrawIndexedIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); @@ -515,15 +515,19 @@ namespace Ryujinx.Graphics.Vulkan return; } + var countBuffer = Gd.BufferManager + .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) + .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; + + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; + UpdateIndexBufferPattern(); RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); DrawCount++; - var countBuffer = Gd.BufferManager - .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) - .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; - if (_indexBufferPattern != null) { // Convert the index buffer into a supported topology. @@ -570,10 +574,6 @@ namespace Ryujinx.Graphics.Vulkan } else { - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - ResumeTransformFeedbackInternal(); if (Gd.Capabilities.SupportsIndirectParameters) @@ -609,15 +609,15 @@ namespace Ryujinx.Graphics.Vulkan // TODO: Support quads and other unsupported topologies. + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value; + RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); BeginRenderPass(); ResumeTransformFeedbackInternal(); DrawCount++; - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - Gd.Api.CmdDrawIndirect(CommandBuffer, buffer, (ulong)indirectBuffer.Offset, 1, (uint)indirectBuffer.Size); } @@ -634,6 +634,14 @@ namespace Ryujinx.Graphics.Vulkan return; } + var buffer = Gd.BufferManager + .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) + .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size, false).Value; + + var countBuffer = Gd.BufferManager + .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) + .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size, false).Value; + // TODO: Support quads and other unsupported topologies. RecreatePipelineIfNeeded(PipelineBindPoint.Graphics); @@ -641,14 +649,6 @@ namespace Ryujinx.Graphics.Vulkan ResumeTransformFeedbackInternal(); DrawCount++; - var buffer = Gd.BufferManager - .GetBuffer(CommandBuffer, indirectBuffer.Handle, indirectBuffer.Offset, indirectBuffer.Size, false) - .Get(Cbs, indirectBuffer.Offset, indirectBuffer.Size).Value; - - var countBuffer = Gd.BufferManager - .GetBuffer(CommandBuffer, parameterBuffer.Handle, parameterBuffer.Offset, parameterBuffer.Size, false) - .Get(Cbs, parameterBuffer.Offset, parameterBuffer.Size).Value; - Gd.DrawIndirectCountApi.CmdDrawIndirectCount( CommandBuffer, buffer, @@ -709,6 +709,26 @@ namespace Ryujinx.Graphics.Vulkan return CommandBuffer.Handle == cb.Handle; } + internal void Rebind(Auto<DisposableBuffer> buffer, int offset, int size) + { + _descriptorSetUpdater.Rebind(buffer, offset, size); + + if (_indexBuffer.Overlaps(buffer, offset, size)) + { + _indexBuffer.BindIndexBuffer(Gd, Cbs); + } + + for (int i = 0; i < _vertexBuffers.Length; i++) + { + if (_vertexBuffers[i].Overlaps(buffer, offset, size)) + { + _vertexBuffers[i].BindVertexBuffer(Gd, Cbs, (uint)i, ref _newState, _vertexBufferUpdater); + } + } + + _vertexBufferUpdater.Commit(Cbs); + } + #pragma warning disable CA1822 // Mark member as static public void SetAlphaTest(bool enable, float reference, CompareOp op) { diff --git a/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs b/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs index dfdac52f..dcc6c530 100644 --- a/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs +++ b/src/Ryujinx.Graphics.Vulkan/PipelineFull.cs @@ -14,6 +14,7 @@ namespace Ryujinx.Graphics.Vulkan private CounterQueueEvent _activeConditionalRender; private readonly List<BufferedQuery> _pendingQueryCopies; + private readonly List<BufferHolder> _activeBufferMirrors; private ulong _byteWeight; @@ -24,6 +25,7 @@ namespace Ryujinx.Graphics.Vulkan _activeQueries = new List<(QueryPool, bool)>(); _pendingQueryCopies = new(); _backingSwaps = new(); + _activeBufferMirrors = new(); CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer; } @@ -233,6 +235,12 @@ namespace Ryujinx.Graphics.Vulkan Gd.RegisterFlush(); // Restore per-command buffer state. + foreach (BufferHolder buffer in _activeBufferMirrors) + { + buffer.ClearMirrors(); + } + + _activeBufferMirrors.Clear(); foreach ((var queryPool, var isOcclusion) in _activeQueries) { @@ -249,6 +257,11 @@ namespace Ryujinx.Graphics.Vulkan Restore(); } + public void RegisterActiveMirror(BufferHolder buffer) + { + _activeBufferMirrors.Add(buffer); + } + public void BeginQuery(BufferedQuery query, QueryPool pool, bool needsReset, bool isOcclusion, bool fromSamplePool) { if (needsReset) diff --git a/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs b/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs index 8d4fdc19..2a85429f 100644 --- a/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs +++ b/src/Ryujinx.Graphics.Vulkan/Queries/BufferedQuery.cs @@ -183,7 +183,7 @@ namespace Ryujinx.Graphics.Vulkan.Queries public void PoolCopy(CommandBufferScoped cbs) { - var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long)).Value; + var buffer = _buffer.GetBuffer(cbs.CommandBuffer, true).Get(cbs, 0, sizeof(long), true).Value; QueryResultFlags flags = QueryResultFlags.ResultWaitBit; diff --git a/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs b/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs index 00fa6477..32ec8c7c 100644 --- a/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs +++ b/src/Ryujinx.Graphics.Vulkan/StagingBuffer.cs @@ -1,12 +1,28 @@ +using Ryujinx.Common; +using Ryujinx.Common.Logging; using System; using System.Collections.Generic; using System.Diagnostics; namespace Ryujinx.Graphics.Vulkan { + readonly struct StagingBufferReserved + { + public readonly BufferHolder Buffer; + public readonly int Offset; + public readonly int Size; + + public StagingBufferReserved(BufferHolder buffer, int offset, int size) + { + Buffer = buffer; + Offset = offset; + Size = size; + } + } + class StagingBuffer : IDisposable { - private const int BufferSize = 16 * 1024 * 1024; + private const int BufferSize = 32 * 1024 * 1024; private int _freeOffset; private int _freeSize; @@ -130,13 +146,83 @@ namespace Ryujinx.Graphics.Vulkan } } - endRenderPass(); + endRenderPass?.Invoke(); PushDataImpl(cbs, dst, dstOffset, data); return true; } + private StagingBufferReserved ReserveDataImpl(CommandBufferScoped cbs, int size, int alignment) + { + // Assumes the caller has already determined that there is enough space. + int offset = BitUtils.AlignUp(_freeOffset, alignment); + int padding = offset - _freeOffset; + + int capacity = Math.Min(_freeSize, BufferSize - offset); + int reservedLength = size + padding; + if (capacity < size) + { + offset = 0; // Place at start. + reservedLength += capacity; + } + + _freeOffset = (_freeOffset + reservedLength) & (BufferSize - 1); + _freeSize -= reservedLength; + Debug.Assert(_freeSize >= 0); + + _pendingCopies.Enqueue(new PendingCopy(cbs.GetFence(), reservedLength)); + + return new StagingBufferReserved(_buffer, offset, size); + } + + private int GetContiguousFreeSize(int alignment) + { + int alignedFreeOffset = BitUtils.AlignUp(_freeOffset, alignment); + int padding = alignedFreeOffset - _freeOffset; + + // Free regions: + // - Aligned free offset to end (minimum free size - padding) + // - 0 to _freeOffset + freeSize wrapped (only if free area contains 0) + + int endOffset = (_freeOffset + _freeSize) & (BufferSize - 1); + + return Math.Max( + Math.Min(_freeSize - padding, BufferSize - alignedFreeOffset), + endOffset <= _freeOffset ? Math.Min(_freeSize, endOffset) : 0 + ); + } + + /// <summary> + /// Reserve a range on the staging buffer for the current command buffer and upload data to it. + /// </summary> + /// <param name="cbs">Command buffer to reserve the data on</param> + /// <param name="data">The data to upload</param> + /// <param name="alignment">The required alignment for the buffer offset</param> + /// <returns>The reserved range of the staging buffer</returns> + public unsafe StagingBufferReserved? TryReserveData(CommandBufferScoped cbs, int size, int alignment) + { + if (size > BufferSize) + { + return null; + } + + // Temporary reserved data cannot be fragmented. + + if (GetContiguousFreeSize(alignment) < size) + { + FreeCompleted(); + + if (GetContiguousFreeSize(alignment) < size) + { + Logger.Debug?.PrintMsg(LogClass.Gpu, $"Staging buffer out of space to reserve data of size {size}."); + return null; + } + } + + return ReserveDataImpl(cbs, size, alignment); + } + private bool WaitFreeCompleted(CommandBufferPool cbp) { if (_pendingCopies.TryPeek(out var pc)) diff --git a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs index ddcf51f6..285a5649 100644 --- a/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs +++ b/src/Ryujinx.Graphics.Vulkan/TextureBuffer.cs @@ -127,24 +127,24 @@ namespace Ryujinx.Graphics.Vulkan ReleaseImpl(); } - public BufferView GetBufferView(CommandBufferScoped cbs) + public BufferView GetBufferView(CommandBufferScoped cbs, bool write) { _bufferView ??= _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl); - return _bufferView?.Get(cbs, _offset, _size).Value ?? default; + return _bufferView?.Get(cbs, _offset, _size, write).Value ?? default; } - public BufferView GetBufferView(CommandBufferScoped cbs, Format format) + public BufferView GetBufferView(CommandBufferScoped cbs, Format format, bool write) { var vkFormat = FormatTable.GetFormat(format); if (vkFormat == VkFormat) { - return GetBufferView(cbs); + return GetBufferView(cbs, write); } if (_selfManagedViews != null && _selfManagedViews.TryGetValue(format, out var bufferView)) { - return bufferView.Get(cbs, _offset, _size).Value; + return bufferView.Get(cbs, _offset, _size, write).Value; } bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl); @@ -154,7 +154,7 @@ namespace Ryujinx.Graphics.Vulkan (_selfManagedViews ??= new Dictionary<Format, Auto<DisposableBufferView>>()).Add(format, bufferView); } - return bufferView?.Get(cbs, _offset, _size).Value ?? default; + return bufferView?.Get(cbs, _offset, _size, write).Value ?? default; } } } diff --git a/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs b/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs index cbbd829a..9a943bf9 100644 --- a/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs +++ b/src/Ryujinx.Graphics.Vulkan/VertexBufferState.cs @@ -4,6 +4,8 @@ namespace Ryujinx.Graphics.Vulkan { internal struct VertexBufferState { + private const int VertexBufferMaxMirrorable = 0x20000; + public static VertexBufferState Null => new(null, 0, 0, 0); private readonly int _offset; @@ -88,9 +90,11 @@ namespace Ryujinx.Graphics.Vulkan if (autoBuffer != null) { - var buffer = autoBuffer.Get(cbs, _offset, _size).Value; + int offset = _offset; + bool mirrorable = _size <= VertexBufferMaxMirrorable; + var buffer = mirrorable ? autoBuffer.GetMirrorable(cbs, ref offset, _size, out _).Value : autoBuffer.Get(cbs, offset, _size).Value; - updater.BindVertexBuffer(cbs, binding, buffer, (ulong)_offset, (ulong)_size, (ulong)_stride); + updater.BindVertexBuffer(cbs, binding, buffer, (ulong)offset, (ulong)_size, (ulong)_stride); } } @@ -99,6 +103,11 @@ namespace Ryujinx.Graphics.Vulkan return _buffer == buffer; } + public readonly bool Overlaps(Auto<DisposableBuffer> buffer, int offset, int size) + { + return buffer == _buffer && offset < _offset + _size && offset + size > _offset; + } + public readonly bool Matches(Auto<DisposableBuffer> buffer, int descriptorIndex, int offset, int size, int stride = 0) { return _buffer == buffer && DescriptorIndex == descriptorIndex && _offset == offset && _size == size && _stride == stride; diff --git a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs index 11c3bfe4..20b32c70 100644 --- a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs +++ b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs @@ -293,6 +293,13 @@ namespace Ryujinx.Graphics.Vulkan ref var properties = ref properties2.Properties; + ulong minResourceAlignment = Math.Max( + Math.Max( + properties.Limits.MinStorageBufferOffsetAlignment, + properties.Limits.MinUniformBufferOffsetAlignment), + properties.Limits.MinTexelBufferOffsetAlignment + ); + SampleCountFlags supportedSampleCounts = properties.Limits.FramebufferColorSampleCounts & properties.Limits.FramebufferDepthSampleCounts & @@ -334,7 +341,8 @@ namespace Ryujinx.Graphics.Vulkan supportedSampleCounts, portabilityFlags, vertexBufferAlignment, - properties.Limits.SubTexelPrecisionBits); + properties.Limits.SubTexelPrecisionBits, + minResourceAlignment); IsSharedMemory = MemoryAllocator.IsDeviceMemoryShared(_physicalDevice); @@ -397,7 +405,7 @@ namespace Ryujinx.Graphics.Vulkan public BufferHandle CreateBuffer(int size, BufferAccess access) { - return BufferManager.CreateWithHandle(this, size, access.Convert()); + return BufferManager.CreateWithHandle(this, size, access.Convert(), default, access == BufferAccess.Stream); } public BufferHandle CreateBuffer(int size, BufferHandle storageHint) |