diff options
35 files changed, 659 insertions, 166 deletions
diff --git a/Ryujinx.Graphics.GAL/IRenderer.cs b/Ryujinx.Graphics.GAL/IRenderer.cs index 1f2af559..2af7b5db 100644 --- a/Ryujinx.Graphics.GAL/IRenderer.cs +++ b/Ryujinx.Graphics.GAL/IRenderer.cs @@ -15,7 +15,12 @@ namespace Ryujinx.Graphics.GAL void BackgroundContextAction(Action action, bool alwaysBackground = false); - BufferHandle CreateBuffer(int size); + BufferHandle CreateBuffer(int size, BufferHandle storageHint); + + BufferHandle CreateBuffer(int size) + { + return CreateBuffer(size, BufferHandle.Null); + } IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info); @@ -26,7 +31,7 @@ namespace Ryujinx.Graphics.GAL void DeleteBuffer(BufferHandle buffer); - ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size); + PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size); Capabilities GetCapabilities(); ulong GetCurrentSync(); diff --git a/Ryujinx.Graphics.GAL/ITexture.cs b/Ryujinx.Graphics.GAL/ITexture.cs index 4dc93303..792c863c 100644 --- a/Ryujinx.Graphics.GAL/ITexture.cs +++ b/Ryujinx.Graphics.GAL/ITexture.cs @@ -15,8 +15,8 @@ namespace Ryujinx.Graphics.GAL ITexture CreateView(TextureCreateInfo info, int firstLayer, int firstLevel); - ReadOnlySpan<byte> GetData(); - ReadOnlySpan<byte> GetData(int layer, int level); + PinnedSpan<byte> GetData(); + PinnedSpan<byte> GetData(int layer, int level); void SetData(SpanOrArray<byte> data); void SetData(SpanOrArray<byte> data, int layer, int level); diff --git a/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetDataCommand.cs b/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetDataCommand.cs index d3a255e7..031c6153 100644 --- a/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetDataCommand.cs +++ b/Ryujinx.Graphics.GAL/Multithreading/Commands/Buffer/BufferGetDataCommand.cs @@ -21,9 +21,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Buffer public static void Run(ref BufferGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer) { - ReadOnlySpan<byte> result = renderer.GetBufferData(threaded.Buffers.MapBuffer(command._buffer), command._offset, command._size); + PinnedSpan<byte> result = renderer.GetBufferData(threaded.Buffers.MapBuffer(command._buffer), command._offset, command._size); - command._result.Get(threaded).Result = new PinnedSpan<byte>(result); + command._result.Get(threaded).Result = result; } } } diff --git a/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs b/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs index 4f01dea2..b36d8bbe 100644 --- a/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs +++ b/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs @@ -5,16 +5,25 @@ public CommandType CommandType => CommandType.CreateBuffer; private BufferHandle _threadedHandle; private int _size; + private BufferHandle _storageHint; - public void Set(BufferHandle threadedHandle, int size) + public void Set(BufferHandle threadedHandle, int size, BufferHandle storageHint) { _threadedHandle = threadedHandle; _size = size; + _storageHint = storageHint; } public static void Run(ref CreateBufferCommand command, ThreadedRenderer threaded, IRenderer renderer) { - threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size)); + BufferHandle hint = BufferHandle.Null; + + if (command._storageHint != BufferHandle.Null) + { + hint = threaded.Buffers.MapBuffer(command._storageHint); + } + + threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, hint)); } } } diff --git a/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataCommand.cs b/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataCommand.cs index 1f519ccd..91320d45 100644 --- a/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataCommand.cs +++ b/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataCommand.cs @@ -18,9 +18,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture public static void Run(ref TextureGetDataCommand command, ThreadedRenderer threaded, IRenderer renderer) { - ReadOnlySpan<byte> result = command._texture.Get(threaded).Base.GetData(); + PinnedSpan<byte> result = command._texture.Get(threaded).Base.GetData(); - command._result.Get(threaded).Result = new PinnedSpan<byte>(result); + command._result.Get(threaded).Result = result; } } } diff --git a/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataSliceCommand.cs b/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataSliceCommand.cs index 5ac05971..ec06cc4d 100644 --- a/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataSliceCommand.cs +++ b/Ryujinx.Graphics.GAL/Multithreading/Commands/Texture/TextureGetDataSliceCommand.cs @@ -22,9 +22,9 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Texture public static void Run(ref TextureGetDataSliceCommand command, ThreadedRenderer threaded, IRenderer renderer) { - ReadOnlySpan<byte> result = command._texture.Get(threaded).Base.GetData(command._layer, command._level); + PinnedSpan<byte> result = command._texture.Get(threaded).Base.GetData(command._layer, command._level); - command._result.Get(threaded).Result = new PinnedSpan<byte>(result); + command._result.Get(threaded).Result = result; } } } diff --git a/Ryujinx.Graphics.GAL/Multithreading/Model/PinnedSpan.cs b/Ryujinx.Graphics.GAL/Multithreading/Model/PinnedSpan.cs deleted file mode 100644 index 16e148c2..00000000 --- a/Ryujinx.Graphics.GAL/Multithreading/Model/PinnedSpan.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -namespace Ryujinx.Graphics.GAL.Multithreading.Model -{ - unsafe struct PinnedSpan<T> where T : unmanaged - { - private void* _ptr; - private int _size; - - public PinnedSpan(ReadOnlySpan<T> span) - { - _ptr = Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)); - _size = span.Length; - } - - public ReadOnlySpan<T> Get() - { - return new ReadOnlySpan<T>(_ptr, _size * Unsafe.SizeOf<T>()); - } - } -} diff --git a/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs b/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs index 1267ab79..ee1cfa29 100644 --- a/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs +++ b/Ryujinx.Graphics.GAL/Multithreading/Resources/ThreadedTexture.cs @@ -72,7 +72,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources return newTex; } - public ReadOnlySpan<byte> GetData() + public PinnedSpan<byte> GetData() { if (_renderer.IsGpuThread()) { @@ -80,7 +80,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources _renderer.New<TextureGetDataCommand>().Set(Ref(this), Ref(box)); _renderer.InvokeCommand(); - return box.Result.Get(); + return box.Result; } else { @@ -90,7 +90,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources } } - public ReadOnlySpan<byte> GetData(int layer, int level) + public PinnedSpan<byte> GetData(int layer, int level) { if (_renderer.IsGpuThread()) { @@ -98,7 +98,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Resources _renderer.New<TextureGetDataSliceCommand>().Set(Ref(this), Ref(box), layer, level); _renderer.InvokeCommand(); - return box.Result.Get(); + return box.Result; } else { diff --git a/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs b/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs index 74326f1d..2148f43f 100644 --- a/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs +++ b/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs @@ -265,10 +265,10 @@ namespace Ryujinx.Graphics.GAL.Multithreading } } - public BufferHandle CreateBuffer(int size) + public BufferHandle CreateBuffer(int size, BufferHandle storageHint) { BufferHandle handle = Buffers.CreateBufferHandle(); - New<CreateBufferCommand>().Set(handle, size); + New<CreateBufferCommand>().Set(handle, size, storageHint); QueueCommand(); return handle; @@ -329,7 +329,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading QueueCommand(); } - public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) + public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) { if (IsGpuThread()) { @@ -337,7 +337,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading New<BufferGetDataCommand>().Set(buffer, offset, size, Ref(box)); InvokeCommand(); - return box.Result.Get(); + return box.Result; } else { diff --git a/Ryujinx.Graphics.GAL/PinnedSpan.cs b/Ryujinx.Graphics.GAL/PinnedSpan.cs new file mode 100644 index 00000000..275b3b86 --- /dev/null +++ b/Ryujinx.Graphics.GAL/PinnedSpan.cs @@ -0,0 +1,53 @@ +using System; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; + +namespace Ryujinx.Graphics.GAL +{ + public unsafe struct PinnedSpan<T> : IDisposable where T : unmanaged + { + private void* _ptr; + private int _size; + private Action _disposeAction; + + /// <summary> + /// Creates a new PinnedSpan from an existing ReadOnlySpan. The span *must* be pinned in memory. + /// The data must be guaranteed to live until disposeAction is called. + /// </summary> + /// <param name="span">Existing span</param> + /// <param name="disposeAction">Action to call on dispose</param> + /// <remarks> + /// If a dispose action is not provided, it is safe to assume the resource will be available until the next call. + /// </remarks> + public static PinnedSpan<T> UnsafeFromSpan(ReadOnlySpan<T> span, Action disposeAction = null) + { + return new PinnedSpan<T>(Unsafe.AsPointer(ref MemoryMarshal.GetReference(span)), span.Length, disposeAction); + } + + /// <summary> + /// Creates a new PinnedSpan from an existing unsafe region. The data must be guaranteed to live until disposeAction is called. + /// </summary> + /// <param name="ptr">Pointer to the region</param> + /// <param name="size">The total items of T the region contains</param> + /// <param name="disposeAction">Action to call on dispose</param> + /// <remarks> + /// If a dispose action is not provided, it is safe to assume the resource will be available until the next call. + /// </remarks> + public PinnedSpan(void* ptr, int size, Action disposeAction = null) + { + _ptr = ptr; + _size = size; + _disposeAction = disposeAction; + } + + public ReadOnlySpan<T> Get() + { + return new ReadOnlySpan<T>(_ptr, _size * Unsafe.SizeOf<T>()); + } + + public void Dispose() + { + _disposeAction?.Invoke(); + } + } +} diff --git a/Ryujinx.Graphics.Gpu/Image/Texture.cs b/Ryujinx.Graphics.Gpu/Image/Texture.cs index f80f20ed..84808a84 100644 --- a/Ryujinx.Graphics.Gpu/Image/Texture.cs +++ b/Ryujinx.Graphics.Gpu/Image/Texture.cs @@ -1022,13 +1022,12 @@ namespace Ryujinx.Graphics.Gpu.Image /// This method should be used to retrieve data that was modified by the host GPU. /// This is not cheap, avoid doing that unless strictly needed. /// </remarks> - /// <param name="output">An output span to place the texture data into. If empty, one is generated</param> + /// <param name="output">An output span to place the texture data into</param> /// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param> /// <param name="texture">The specific host texture to flush. Defaults to this texture</param> - /// <returns>The span containing the texture data</returns> - private ReadOnlySpan<byte> GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null) + private void GetTextureDataFromGpu(Span<byte> output, bool blacklist, ITexture texture = null) { - ReadOnlySpan<byte> data; + PinnedSpan<byte> data; if (texture != null) { @@ -1054,9 +1053,9 @@ namespace Ryujinx.Graphics.Gpu.Image } } - data = ConvertFromHostCompatibleFormat(output, data); + ConvertFromHostCompatibleFormat(output, data.Get()); - return data; + data.Dispose(); } /// <summary> @@ -1071,10 +1070,9 @@ namespace Ryujinx.Graphics.Gpu.Image /// <param name="level">The level of the texture to flush</param> /// <param name="blacklist">True if the texture should be blacklisted, false otherwise</param> /// <param name="texture">The specific host texture to flush. Defaults to this texture</param> - /// <returns>The span containing the texture data</returns> - public ReadOnlySpan<byte> GetTextureDataSliceFromGpu(Span<byte> output, int layer, int level, bool blacklist, ITexture texture = null) + public void GetTextureDataSliceFromGpu(Span<byte> output, int layer, int level, bool blacklist, ITexture texture = null) { - ReadOnlySpan<byte> data; + PinnedSpan<byte> data; if (texture != null) { @@ -1100,9 +1098,9 @@ namespace Ryujinx.Graphics.Gpu.Image } } - data = ConvertFromHostCompatibleFormat(output, data, level, true); + ConvertFromHostCompatibleFormat(output, data.Get(), level, true); - return data; + data.Dispose(); } /// <summary> diff --git a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs index 3778cd82..f267dfda 100644 --- a/Ryujinx.Graphics.Gpu/Memory/Buffer.cs +++ b/Ryujinx.Graphics.Gpu/Memory/Buffer.cs @@ -82,7 +82,7 @@ namespace Ryujinx.Graphics.Gpu.Memory Address = address; Size = size; - Handle = context.Renderer.CreateBuffer((int)size); + Handle = context.Renderer.CreateBuffer((int)size, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null); _useGranular = size > GranularBufferThreshold; @@ -415,10 +415,10 @@ namespace Ryujinx.Graphics.Gpu.Memory { int offset = (int)(address - Address); - ReadOnlySpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size); + using PinnedSpan<byte> data = _context.Renderer.GetBufferData(Handle, offset, (int)size); // TODO: When write tracking shaders, they will need to be aware of changes in overlapping buffers. - _physicalMemory.WriteUntracked(address, data); + _physicalMemory.WriteUntracked(address, data.Get()); } /// <summary> diff --git a/Ryujinx.Graphics.OpenGL/Buffer.cs b/Ryujinx.Graphics.OpenGL/Buffer.cs index 68c82f95..af7d191a 100644 --- a/Ryujinx.Graphics.OpenGL/Buffer.cs +++ b/Ryujinx.Graphics.OpenGL/Buffer.cs @@ -55,11 +55,14 @@ namespace Ryujinx.Graphics.OpenGL (IntPtr)size); } - public static unsafe ReadOnlySpan<byte> GetData(OpenGLRenderer renderer, BufferHandle buffer, int offset, int size) + public static unsafe PinnedSpan<byte> GetData(OpenGLRenderer renderer, BufferHandle buffer, int offset, int size) { + // Data in the persistent buffer and host array is guaranteed to be available + // until the next time the host thread requests data. + if (HwCapabilities.UsePersistentBufferForFlush) { - return renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size); + return PinnedSpan<byte>.UnsafeFromSpan(renderer.PersistentBuffers.Default.GetBufferData(buffer, offset, size)); } else { @@ -69,7 +72,7 @@ namespace Ryujinx.Graphics.OpenGL GL.GetBufferSubData(BufferTarget.CopyReadBuffer, (IntPtr)offset, size, target); - return new ReadOnlySpan<byte>(target.ToPointer(), size); + return new PinnedSpan<byte>(target.ToPointer(), size); } } diff --git a/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs b/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs index 76d0149b..1e9e4d6b 100644 --- a/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs +++ b/Ryujinx.Graphics.OpenGL/Image/TextureBuffer.cs @@ -39,12 +39,12 @@ namespace Ryujinx.Graphics.OpenGL.Image throw new NotSupportedException(); } - public ReadOnlySpan<byte> GetData() + public PinnedSpan<byte> GetData() { return Buffer.GetData(_renderer, _buffer, _bufferOffset, _bufferSize); } - public ReadOnlySpan<byte> GetData(int layer, int level) + public PinnedSpan<byte> GetData(int layer, int level) { return GetData(); } diff --git a/Ryujinx.Graphics.OpenGL/Image/TextureView.cs b/Ryujinx.Graphics.OpenGL/Image/TextureView.cs index 44df441f..ddc5f9a3 100644 --- a/Ryujinx.Graphics.OpenGL/Image/TextureView.cs +++ b/Ryujinx.Graphics.OpenGL/Image/TextureView.cs @@ -166,7 +166,7 @@ namespace Ryujinx.Graphics.OpenGL.Image _renderer.TextureCopy.Copy(this, (TextureView)destination, srcRegion, dstRegion, linearFilter); } - public unsafe ReadOnlySpan<byte> GetData() + public unsafe PinnedSpan<byte> GetData() { int size = 0; int levels = Info.GetLevelsClamped(); @@ -196,16 +196,16 @@ namespace Ryujinx.Graphics.OpenGL.Image data = FormatConverter.ConvertD24S8ToS8D24(data); } - return data; + return PinnedSpan<byte>.UnsafeFromSpan(data); } - public unsafe ReadOnlySpan<byte> GetData(int layer, int level) + public unsafe PinnedSpan<byte> GetData(int layer, int level) { int size = Info.GetMipSize(level); if (HwCapabilities.UsePersistentBufferForFlush) { - return _renderer.PersistentBuffers.Default.GetTextureData(this, size, layer, level); + return PinnedSpan<byte>.UnsafeFromSpan(_renderer.PersistentBuffers.Default.GetTextureData(this, size, layer, level)); } else { @@ -213,7 +213,7 @@ namespace Ryujinx.Graphics.OpenGL.Image int offset = WriteTo2D(target, layer, level); - return new ReadOnlySpan<byte>(target.ToPointer(), size).Slice(offset); + return new PinnedSpan<byte>((byte*)target.ToPointer() + offset, size); } } diff --git a/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs b/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs index c79700fc..91e52178 100644 --- a/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs +++ b/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs @@ -57,7 +57,7 @@ namespace Ryujinx.Graphics.OpenGL ResourcePool = new ResourcePool(); } - public BufferHandle CreateBuffer(int size) + public BufferHandle CreateBuffer(int size, BufferHandle storageHint) { BufferCount++; @@ -96,7 +96,7 @@ namespace Ryujinx.Graphics.OpenGL return new HardwareInfo(GpuVendor, GpuRenderer); } - public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) + public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) { return Buffer.GetData(this, buffer, offset, size); } diff --git a/Ryujinx.Graphics.Vulkan/BufferAllocationType.cs b/Ryujinx.Graphics.Vulkan/BufferAllocationType.cs new file mode 100644 index 00000000..81489041 --- /dev/null +++ b/Ryujinx.Graphics.Vulkan/BufferAllocationType.cs @@ -0,0 +1,12 @@ +namespace Ryujinx.Graphics.Vulkan +{ + internal enum BufferAllocationType + { + Auto = 0, + + HostMappedNoCache, + HostMapped, + DeviceLocal, + DeviceLocalMapped + } +} diff --git a/Ryujinx.Graphics.Vulkan/BufferHolder.cs b/Ryujinx.Graphics.Vulkan/BufferHolder.cs index 055d6a59..21b81bdd 100644 --- a/Ryujinx.Graphics.Vulkan/BufferHolder.cs +++ b/Ryujinx.Graphics.Vulkan/BufferHolder.cs @@ -1,7 +1,10 @@ -using Ryujinx.Graphics.GAL; +using Ryujinx.Common.Logging; +using Ryujinx.Graphics.GAL; using Silk.NET.Vulkan; using System; +using System.Collections.Generic; using System.Runtime.CompilerServices; +using System.Threading; using VkBuffer = Silk.NET.Vulkan.Buffer; using VkFormat = Silk.NET.Vulkan.Format; @@ -11,6 +14,12 @@ namespace Ryujinx.Graphics.Vulkan { private const int MaxUpdateBufferSize = 0x10000; + private const int SetCountThreshold = 100; + private const int WriteCountThreshold = 50; + private const int FlushCountThreshold = 5; + + public const int DeviceLocalSizeThreshold = 256 * 1024; // 256kb + public const AccessFlags DefaultAccessFlags = AccessFlags.IndirectCommandReadBit | AccessFlags.ShaderReadBit | @@ -21,10 +30,10 @@ namespace Ryujinx.Graphics.Vulkan private readonly VulkanRenderer _gd; private readonly Device _device; - private readonly MemoryAllocation _allocation; - private readonly Auto<DisposableBuffer> _buffer; - private readonly Auto<MemoryAllocation> _allocationAuto; - private readonly ulong _bufferHandle; + private MemoryAllocation _allocation; + private Auto<DisposableBuffer> _buffer; + private Auto<MemoryAllocation> _allocationAuto; + private ulong _bufferHandle; private CacheByRange<BufferHolder> _cachedConvertedBuffers; @@ -32,11 +41,28 @@ namespace Ryujinx.Graphics.Vulkan private IntPtr _map; - private readonly MultiFenceHolder _waitable; + private MultiFenceHolder _waitable; private bool _lastAccessIsWrite; - public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size) + private BufferAllocationType _baseType; + private BufferAllocationType _currentType; + private bool _swapQueued; + + public BufferAllocationType DesiredType { get; private set; } + + private int _setCount; + private int _writeCount; + private int _flushCount; + private int _flushTemp; + + private ReaderWriterLock _flushLock; + private FenceHolder _flushFence; + private int _flushWaiting; + + private List<Action> _swapActions; + + public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, MemoryAllocation allocation, int size, BufferAllocationType type, BufferAllocationType currentType) { _gd = gd; _device = device; @@ -47,9 +73,153 @@ namespace Ryujinx.Graphics.Vulkan _bufferHandle = buffer.Handle; Size = size; _map = allocation.HostPointer; + + _baseType = type; + _currentType = currentType; + DesiredType = currentType; + + _flushLock = new ReaderWriterLock(); + } + + public bool TryBackingSwap(ref CommandBufferScoped? cbs) + { + if (_swapQueued && DesiredType != _currentType) + { + // Only swap if the buffer is not used in any queued command buffer. + bool isRented = _buffer.HasRentedCommandBufferDependency(_gd.CommandBufferPool); + + if (!isRented && _gd.CommandBufferPool.OwnedByCurrentThread && !_flushLock.IsReaderLockHeld) + { + var currentAllocation = _allocationAuto; + var currentBuffer = _buffer; + IntPtr currentMap = _map; + + (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, _currentType); + + if (buffer.Handle != 0) + { + _flushLock.AcquireWriterLock(Timeout.Infinite); + + ClearFlushFence(); + + _waitable = new MultiFenceHolder(Size); + + _allocation = allocation; + _allocationAuto = new Auto<MemoryAllocation>(allocation); + _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(_gd.Api, _device, buffer), _waitable, _allocationAuto); + _bufferHandle = buffer.Handle; + _map = allocation.HostPointer; + + if (_map != IntPtr.Zero && currentMap != IntPtr.Zero) + { + // Copy data directly. Readbacks don't have to wait if this is done. + + unsafe + { + new Span<byte>((void*)currentMap, Size).CopyTo(new Span<byte>((void*)_map, Size)); + } + } + else + { + if (cbs == null) + { + cbs = _gd.CommandBufferPool.Rent(); + } + + CommandBufferScoped cbsV = cbs.Value; + + Copy(_gd, cbsV, currentBuffer, _buffer, 0, 0, Size); + + // Need to wait for the data to reach the new buffer before data can be flushed. + + _flushFence = _gd.CommandBufferPool.GetFence(cbsV.CommandBufferIndex); + _flushFence.Get(); + } + + Logger.Debug?.PrintMsg(LogClass.Gpu, $"Converted {Size} buffer {_currentType} to {resultType}"); + + _currentType = resultType; + + if (_swapActions != null) + { + foreach (var action in _swapActions) + { + action(); + } + + _swapActions.Clear(); + } + + currentBuffer.Dispose(); + currentAllocation.Dispose(); + + _gd.PipelineInternal.SwapBuffer(currentBuffer, _buffer); + + _flushLock.ReleaseWriterLock(); + } + + _swapQueued = false; + + return true; + } + else + { + return false; + } + } + else + { + _swapQueued = false; + + return true; + } + } + + private void ConsiderBackingSwap() + { + if (_baseType == BufferAllocationType.Auto) + { + if (_writeCount >= WriteCountThreshold || _setCount >= SetCountThreshold || _flushCount >= FlushCountThreshold) + { + if (_flushCount > 0 || _flushTemp-- > 0) + { + // Buffers that flush should ideally be mapped in host address space for easy copies. + // If the buffer is large it will do better on GPU memory, as there will be more writes than data flushes (typically individual pages). + // If it is small, then it's likely most of the buffer will be flushed so we want it on host memory, as access is cached. + DesiredType = Size > DeviceLocalSizeThreshold ? BufferAllocationType.DeviceLocalMapped : BufferAllocationType.HostMapped; + + // It's harder for a buffer that is flushed to revert to another type of mapping. + if (_flushCount > 0) + { + _flushTemp = 1000; + } + } + else if (_writeCount >= WriteCountThreshold) + { + // Buffers that are written often should ideally be in the device local heap. (Storage buffers) + DesiredType = BufferAllocationType.DeviceLocal; + } + else if (_setCount > SetCountThreshold) + { + // Buffers that have their data set often should ideally be host mapped. (Constant buffers) + DesiredType = BufferAllocationType.HostMapped; + } + + _flushCount = 0; + _writeCount = 0; + _setCount = 0; + } + + if (!_swapQueued && DesiredType != _currentType) + { + _swapQueued = true; + + _gd.PipelineInternal.AddBackingSwap(this); + } + } } - public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size) + public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView) { var bufferViewCreateInfo = new BufferViewCreateInfo() { @@ -62,9 +232,19 @@ namespace Ryujinx.Graphics.Vulkan _gd.Api.CreateBufferView(_device, bufferViewCreateInfo, null, out var bufferView).ThrowOnError(); + (_swapActions ??= new List<Action>()).Add(invalidateView); + return new Auto<DisposableBufferView>(new DisposableBufferView(_gd.Api, _device, bufferView), _waitable, _buffer); } + public void InheritMetrics(BufferHolder other) + { + _setCount = other._setCount; + _writeCount = other._writeCount; + _flushCount = other._flushCount; + _flushTemp = other._flushTemp; + } + public unsafe void InsertBarrier(CommandBuffer commandBuffer, bool isWrite) { // If the last access is write, we always need a barrier to be sure we will read or modify @@ -104,12 +284,22 @@ namespace Ryujinx.Graphics.Vulkan return _buffer; } - public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, bool isWrite = false) + public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, bool isWrite = false, bool isSSBO = false) { if (isWrite) { + _writeCount++; + SignalWrite(0, Size); } + else if (isSSBO) + { + // Always consider SSBO access for swapping to device local memory. + + _writeCount++; + + ConsiderBackingSwap(); + } return _buffer; } @@ -118,6 +308,8 @@ namespace Ryujinx.Graphics.Vulkan { if (isWrite) { + _writeCount++; + SignalWrite(offset, size); } @@ -126,6 +318,8 @@ namespace Ryujinx.Graphics.Vulkan public void SignalWrite(int offset, int size) { + ConsiderBackingSwap(); + if (offset == 0 && size == Size) { _cachedConvertedBuffers.Clear(); @@ -147,11 +341,76 @@ namespace Ryujinx.Graphics.Vulkan return _map; } - public unsafe ReadOnlySpan<byte> GetData(int offset, int size) + private void ClearFlushFence() + { + // Asusmes _flushLock is held as writer. + + if (_flushFence != null) + { + if (_flushWaiting == 0) + { + _flushFence.Put(); + } + + _flushFence = null; + } + } + + private void WaitForFlushFence() + { + // Assumes the _flushLock is held as reader, returns in same state. + + if (_flushFence != null) + { + // If storage has changed, make sure the fence has been reached so that the data is in place. + + var cookie = _flushLock.UpgradeToWriterLock(Timeout.Infinite); + + if (_flushFence != null) + { + var fence = _flushFence; + Interlocked.Increment(ref _flushWaiting); + + // Don't wait in the lock. + + var restoreCookie = _flushLock.ReleaseLock(); + + fence.Wait(); + + _flushLock.RestoreLock(ref restoreCookie); + + if (Interlocked.Decrement(ref _flushWaiting) == 0) + { + fence.Put(); + } + + _flushFence = null; + } + + _flushLock.DowngradeFromWriterLock(ref cookie); + } + } + + public unsafe PinnedSpan<byte> GetData(int offset, int size) { + _flushLock.AcquireReaderLock(Timeout.Infinite); + + WaitForFlushFence(); + + _flushCount++; + + Span<byte> result; + if (_map != IntPtr.Zero) { - return GetDataStorage(offset, size); + result = GetDataStorage(offset, size); + + // Need to be careful here, the buffer can't be unmapped while the data is being used. + _buffer.IncrementReferenceCount(); + + _flushLock.ReleaseReaderLock(); + + return PinnedSpan<byte>.UnsafeFromSpan(result, _buffer.DecrementReferenceCount); } else { @@ -161,12 +420,17 @@ namespace Ryujinx.Graphics.Vulkan { _gd.FlushAllCommands(); - return resource.GetFlushBuffer().GetBufferData(_gd.CommandBufferPool, this, offset, size); + result = resource.GetFlushBuffer().GetBufferData(_gd.CommandBufferPool, this, offset, size); } else { - return resource.GetFlushBuffer().GetBufferData(resource.GetPool(), this, offset, size); + result = resource.GetFlushBuffer().GetBufferData(resource.GetPool(), this, offset, size); } + + _flushLock.ReleaseReaderLock(); + + // Flush buffer is pinned until the next GetBufferData on the thread, which is fine for current uses. + return PinnedSpan<byte>.UnsafeFromSpan(result); } } @@ -190,6 +454,8 @@ namespace Ryujinx.Graphics.Vulkan return; } + _setCount++; + if (_map != IntPtr.Zero) { // If persistently mapped, set the data directly if the buffer is not currently in use. @@ -268,6 +534,8 @@ namespace Ryujinx.Graphics.Vulkan var dstBuffer = GetBuffer(cbs.CommandBuffer, dstOffset, data.Length, true).Get(cbs, dstOffset, data.Length).Value; + _writeCount--; + InsertBufferBarrier( _gd, cbs.CommandBuffer, @@ -502,11 +770,19 @@ namespace Ryujinx.Graphics.Vulkan public void Dispose() { + _swapQueued = false; + _gd.PipelineInternal?.FlushCommandsIfWeightExceeding(_buffer, (ulong)Size); _buffer.Dispose(); _allocationAuto.Dispose(); _cachedConvertedBuffers.Dispose(); + + _flushLock.AcquireWriterLock(Timeout.Infinite); + + ClearFlushFence(); + + _flushLock.ReleaseWriterLock(); } } } diff --git a/Ryujinx.Graphics.Vulkan/BufferManager.cs b/Ryujinx.Graphics.Vulkan/BufferManager.cs index 49fdd75d..f8f41e5b 100644 --- a/Ryujinx.Graphics.Vulkan/BufferManager.cs +++ b/Ryujinx.Graphics.Vulkan/BufferManager.cs @@ -4,6 +4,7 @@ using System; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; using VkFormat = Silk.NET.Vulkan.Format; +using VkBuffer = Silk.NET.Vulkan.Buffer; namespace Ryujinx.Graphics.Vulkan { @@ -16,17 +17,17 @@ namespace Ryujinx.Graphics.Vulkan // Some drivers don't expose a "HostCached" memory type, // so we need those alternative flags for the allocation to succeed there. - private const MemoryPropertyFlags DefaultBufferMemoryAltFlags = + private const MemoryPropertyFlags DefaultBufferMemoryNoCacheFlags = MemoryPropertyFlags.HostVisibleBit | MemoryPropertyFlags.HostCoherentBit; private const MemoryPropertyFlags DeviceLocalBufferMemoryFlags = MemoryPropertyFlags.DeviceLocalBit; - private const MemoryPropertyFlags FlushableDeviceLocalBufferMemoryFlags = + private const MemoryPropertyFlags DeviceLocalMappedBufferMemoryFlags = + MemoryPropertyFlags.DeviceLocalBit | MemoryPropertyFlags.HostVisibleBit | - MemoryPropertyFlags.HostCoherentBit | - MemoryPropertyFlags.DeviceLocalBit; + MemoryPropertyFlags.HostCoherentBit; private const BufferUsageFlags DefaultBufferUsageFlags = BufferUsageFlags.TransferSrcBit | @@ -54,14 +55,14 @@ namespace Ryujinx.Graphics.Vulkan StagingBuffer = new StagingBuffer(gd, this); } - public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal) + public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default) { - return CreateWithHandle(gd, size, deviceLocal, out _); + return CreateWithHandle(gd, size, out _, baseType, storageHint); } - public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, bool deviceLocal, out BufferHolder holder) + public BufferHandle CreateWithHandle(VulkanRenderer gd, int size, out BufferHolder holder, BufferAllocationType baseType = BufferAllocationType.HostMapped, BufferHandle storageHint = default) { - holder = Create(gd, size, deviceLocal: deviceLocal); + holder = Create(gd, size, baseType: baseType, storageHint: storageHint); if (holder == null) { return BufferHandle.Null; @@ -74,7 +75,12 @@ namespace Ryujinx.Graphics.Vulkan return Unsafe.As<ulong, BufferHandle>(ref handle64); } - public unsafe BufferHolder Create(VulkanRenderer gd, int size, bool forConditionalRendering = false, bool deviceLocal = false) + public unsafe (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) CreateBacking( + VulkanRenderer gd, + int size, + BufferAllocationType type, + bool forConditionalRendering = false, + BufferAllocationType fallbackType = BufferAllocationType.Auto) { var usage = DefaultBufferUsageFlags; @@ -98,48 +104,106 @@ namespace Ryujinx.Graphics.Vulkan gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError(); gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements); - MemoryPropertyFlags allocateFlags; - MemoryPropertyFlags allocateFlagsAlt; + MemoryAllocation allocation; - if (deviceLocal) - { - allocateFlags = DeviceLocalBufferMemoryFlags; - allocateFlagsAlt = DeviceLocalBufferMemoryFlags; - } - else + do { - allocateFlags = DefaultBufferMemoryFlags; - allocateFlagsAlt = DefaultBufferMemoryAltFlags; + var allocateFlags = type switch + { + BufferAllocationType.HostMappedNoCache => DefaultBufferMemoryNoCacheFlags, + BufferAllocationType.HostMapped => DefaultBufferMemoryFlags, + BufferAllocationType.DeviceLocal => DeviceLocalBufferMemoryFlags, + BufferAllocationType.DeviceLocalMapped => DeviceLocalMappedBufferMemoryFlags, + _ => DefaultBufferMemoryFlags + }; + + // If an allocation with this memory type fails, fall back to the previous one. + try + { + allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, true); + } + catch (VulkanException) + { + allocation = default; + } } - - var allocation = gd.MemoryAllocator.AllocateDeviceMemory(requirements, allocateFlags, allocateFlagsAlt); + while (allocation.Memory.Handle == 0 && (--type != fallbackType)); if (allocation.Memory.Handle == 0UL) { gd.Api.DestroyBuffer(_device, buffer, null); - return null; + return default; } gd.Api.BindBufferMemory(_device, buffer, allocation.Memory, allocation.Offset); - return new BufferHolder(gd, _device, buffer, allocation, size); + return (buffer, allocation, type); } - public Auto<DisposableBufferView> CreateView(BufferHandle handle, VkFormat format, int offset, int size) + public unsafe BufferHolder Create( + VulkanRenderer gd, + int size, + bool forConditionalRendering = false, + BufferAllocationType baseType = BufferAllocationType.HostMapped, + BufferHandle storageHint = default) + { + BufferAllocationType type = baseType; + BufferHolder storageHintHolder = null; + + if (baseType == BufferAllocationType.Auto) + { + if (gd.IsSharedMemory) + { + baseType = BufferAllocationType.HostMapped; + type = baseType; + } + else + { + type = size >= BufferHolder.DeviceLocalSizeThreshold ? BufferAllocationType.DeviceLocal : BufferAllocationType.HostMapped; + } + + if (storageHint != BufferHandle.Null) + { + if (TryGetBuffer(storageHint, out storageHintHolder)) + { + type = storageHintHolder.DesiredType; + } + } + } + + (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = + CreateBacking(gd, size, type, forConditionalRendering); + + if (buffer.Handle != 0) + { + var holder = new BufferHolder(gd, _device, buffer, allocation, size, baseType, resultType); + + if (storageHintHolder != null) + { + holder.InheritMetrics(storageHintHolder); + } + + return holder; + } + + return null; + } + + public Auto<DisposableBufferView> CreateView(BufferHandle handle, VkFormat format, int offset, int size, Action invalidateView) { if (TryGetBuffer(handle, out var holder)) { - return holder.CreateView(format, offset, size); + return holder.CreateView(format, offset, size, invalidateView); } return null; } - public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite) + public Auto<DisposableBuffer> GetBuffer(CommandBuffer commandBuffer, BufferHandle handle, bool isWrite, bool isSSBO = false) { if (TryGetBuffer(handle, out var holder)) { - return holder.GetBuffer(commandBuffer, isWrite); + return holder.GetBuffer(commandBuffer, isWrite, isSSBO); } return null; @@ -332,14 +396,14 @@ namespace Ryujinx.Graphics.Vulkan return null; } - public ReadOnlySpan<byte> GetData(BufferHandle handle, int offset, int size) + public PinnedSpan<byte> GetData(BufferHandle handle, int offset, int size) { if (TryGetBuffer(handle, out var holder)) { return holder.GetData(offset, size); } - return ReadOnlySpan<byte>.Empty; + return new PinnedSpan<byte>(); } public void SetData<T>(BufferHandle handle, int offset, ReadOnlySpan<T> data) where T : unmanaged diff --git a/Ryujinx.Graphics.Vulkan/BufferState.cs b/Ryujinx.Graphics.Vulkan/BufferState.cs index f3a58469..6829f833 100644 --- a/Ryujinx.Graphics.Vulkan/BufferState.cs +++ b/Ryujinx.Graphics.Vulkan/BufferState.cs @@ -2,14 +2,14 @@ namespace Ryujinx.Graphics.Vulkan { - readonly struct BufferState : IDisposable + struct BufferState : IDisposable { public static BufferState Null => new BufferState(null, 0, 0); private readonly int _offset; private readonly int _size; - private readonly Auto<DisposableBuffer> _buffer; + private Auto<DisposableBuffer> _buffer; public BufferState(Auto<DisposableBuffer> buffer, int offset, int size) { @@ -29,6 +29,17 @@ namespace Ryujinx.Graphics.Vulkan } } + public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) + { + if (_buffer == from) + { + _buffer.DecrementReferenceCount(); + to.IncrementReferenceCount(); + + _buffer = to; + } + } + public void Dispose() { _buffer?.DecrementReferenceCount(); diff --git a/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs b/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs index 19a08502..7e126e04 100644 --- a/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs +++ b/Ryujinx.Graphics.Vulkan/DescriptorSetUpdater.cs @@ -94,7 +94,7 @@ namespace Ryujinx.Graphics.Vulkan else { // If null descriptors are not supported, we need to pass the handle of a dummy buffer on unused bindings. - _dummyBuffer = gd.BufferManager.Create(gd, 0x10000, forConditionalRendering: false, deviceLocal: true); + _dummyBuffer = gd.BufferManager.Create(gd, 0x10000, forConditionalRendering: false, baseType: BufferAllocationType.DeviceLocal); } _dummyTexture = gd.CreateTextureView(new TextureCreateInfo( @@ -178,7 +178,7 @@ namespace Ryujinx.Graphics.Vulkan var buffer = assignment.Range; int index = assignment.Binding; - Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false); + Auto<DisposableBuffer> vkBuffer = _gd.BufferManager.GetBuffer(commandBuffer, buffer.Handle, false, isSSBO: true); ref Auto<DisposableBuffer> currentVkBuffer = ref _storageBufferRefs[index]; DescriptorBufferInfo info = new DescriptorBufferInfo() @@ -640,6 +640,23 @@ namespace Ryujinx.Graphics.Vulkan Array.Clear(_storageSet); } + private void SwapBuffer(Auto<DisposableBuffer>[] list, Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) + { + for (int i = 0; i < list.Length; i++) + { + if (list[i] == from) + { + list[i] = to; + } + } + } + + public void SwapBuffer(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) + { + SwapBuffer(_uniformBufferRefs, from, to); + SwapBuffer(_storageBufferRefs, from, to); + } + protected virtual void Dispose(bool disposing) { if (disposing) diff --git a/Ryujinx.Graphics.Vulkan/Effects/FsrScalingFilter.cs b/Ryujinx.Graphics.Vulkan/Effects/FsrScalingFilter.cs index a1207059..a871679b 100644 --- a/Ryujinx.Graphics.Vulkan/Effects/FsrScalingFilter.cs +++ b/Ryujinx.Graphics.Vulkan/Effects/FsrScalingFilter.cs @@ -156,11 +156,11 @@ namespace Ryujinx.Graphics.Vulkan.Effects }; int rangeSize = dimensionsBuffer.Length * sizeof(float); - var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false); + var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize); _renderer.BufferManager.SetData(bufferHandle, 0, dimensionsBuffer); ReadOnlySpan<float> sharpeningBuffer = stackalloc float[] { 1.5f - (Level * 0.01f * 1.5f)}; - var sharpeningBufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, sizeof(float), false); + var sharpeningBufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, sizeof(float)); _renderer.BufferManager.SetData(sharpeningBufferHandle, 0, sharpeningBuffer); int threadGroupWorkRegionDim = 16; diff --git a/Ryujinx.Graphics.Vulkan/Effects/FxaaPostProcessingEffect.cs b/Ryujinx.Graphics.Vulkan/Effects/FxaaPostProcessingEffect.cs index 0f6a0a7b..9e73e1b8 100644 --- a/Ryujinx.Graphics.Vulkan/Effects/FxaaPostProcessingEffect.cs +++ b/Ryujinx.Graphics.Vulkan/Effects/FxaaPostProcessingEffect.cs @@ -87,7 +87,7 @@ namespace Ryujinx.Graphics.Vulkan.Effects ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height }; int rangeSize = resolutionBuffer.Length * sizeof(float); - var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false); + var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize); _renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer); diff --git a/Ryujinx.Graphics.Vulkan/Effects/SmaaPostProcessingEffect.cs b/Ryujinx.Graphics.Vulkan/Effects/SmaaPostProcessingEffect.cs index 4dcdaa64..bf698ade 100644 --- a/Ryujinx.Graphics.Vulkan/Effects/SmaaPostProcessingEffect.cs +++ b/Ryujinx.Graphics.Vulkan/Effects/SmaaPostProcessingEffect.cs @@ -266,7 +266,7 @@ namespace Ryujinx.Graphics.Vulkan.Effects ReadOnlySpan<float> resolutionBuffer = stackalloc float[] { view.Width, view.Height }; int rangeSize = resolutionBuffer.Length * sizeof(float); - var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize, false); + var bufferHandle = _renderer.BufferManager.CreateWithHandle(_renderer, rangeSize); _renderer.BufferManager.SetData(bufferHandle, 0, resolutionBuffer); var bufferRanges = new BufferRange(bufferHandle, 0, rangeSize); diff --git a/Ryujinx.Graphics.Vulkan/HelperShader.cs b/Ryujinx.Graphics.Vulkan/HelperShader.cs index c67389aa..8eb3088e 100644 --- a/Ryujinx.Graphics.Vulkan/HelperShader.cs +++ b/Ryujinx.Graphics.Vulkan/HelperShader.cs @@ -394,7 +394,7 @@ namespace Ryujinx.Graphics.Vulkan (region[2], region[3]) = (region[3], region[2]); } - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize); gd.BufferManager.SetData<float>(bufferHandle, 0, region); @@ -495,7 +495,7 @@ namespace Ryujinx.Graphics.Vulkan (region[2], region[3]) = (region[3], region[2]); } - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize); gd.BufferManager.SetData<float>(bufferHandle, 0, region); @@ -649,7 +649,7 @@ namespace Ryujinx.Graphics.Vulkan _pipeline.SetCommandBuffer(cbs); - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ClearColorBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ClearColorBufferSize); gd.BufferManager.SetData<float>(bufferHandle, 0, clearColor); @@ -726,7 +726,7 @@ namespace Ryujinx.Graphics.Vulkan (region[2], region[3]) = (region[3], region[2]); } - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, RegionBufferSize); gd.BufferManager.SetData<float>(bufferHandle, 0, region); @@ -802,7 +802,7 @@ namespace Ryujinx.Graphics.Vulkan shaderParams[2] = size; shaderParams[3] = srcOffset; - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); @@ -958,7 +958,7 @@ namespace Ryujinx.Graphics.Vulkan shaderParams[0] = BitOperations.Log2((uint)ratio); - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); @@ -1050,7 +1050,7 @@ namespace Ryujinx.Graphics.Vulkan (shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples); (shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples)); - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); @@ -1133,7 +1133,7 @@ namespace Ryujinx.Graphics.Vulkan (shaderParams[0], shaderParams[1]) = GetSampleCountXYLog2(samples); (shaderParams[2], shaderParams[3]) = GetSampleCountXYLog2((int)TextureStorage.ConvertToSampleCountFlags(gd.Capabilities.SupportedSampleCounts, (uint)samples)); - var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false); + var bufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize); gd.BufferManager.SetData<int>(bufferHandle, 0, shaderParams); @@ -1407,7 +1407,7 @@ namespace Ryujinx.Graphics.Vulkan pattern.OffsetIndex.CopyTo(shaderParams.Slice(0, pattern.OffsetIndex.Length)); - var patternBufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, false, out var patternBuffer); + var patternBufferHandle = gd.BufferManager.CreateWithHandle(gd, ParamsBufferSize, out var patternBuffer); var patternBufferAuto = patternBuffer.GetBuffer(); gd.BufferManager.SetData<int>(patternBufferHandle, 0, shaderParams); diff --git a/Ryujinx.Graphics.Vulkan/IndexBufferPattern.cs b/Ryujinx.Graphics.Vulkan/IndexBufferPattern.cs index 90774293..11f4ec33 100644 --- a/Ryujinx.Graphics.Vulkan/IndexBufferPattern.cs +++ b/Ryujinx.Graphics.Vulkan/IndexBufferPattern.cs @@ -82,7 +82,7 @@ namespace Ryujinx.Graphics.Vulkan } // Expand the repeating pattern to the number of requested primitives. - BufferHandle newBuffer = _gd.CreateBuffer(expectedSize * sizeof(int)); + BufferHandle newBuffer = _gd.BufferManager.CreateWithHandle(_gd, expectedSize * sizeof(int)); // Copy the old data to the new one. if (_repeatingBuffer != BufferHandle.Null) diff --git a/Ryujinx.Graphics.Vulkan/IndexBufferState.cs b/Ryujinx.Graphics.Vulkan/IndexBufferState.cs index 64b95f60..75b18456 100644 --- a/Ryujinx.Graphics.Vulkan/IndexBufferState.cs +++ b/Ryujinx.Graphics.Vulkan/IndexBufferState.cs @@ -146,5 +146,16 @@ namespace Ryujinx.Graphics.Vulkan { return _buffer == buffer; } + + public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) + { + if (_buffer == from) + { + _buffer.DecrementReferenceCount(); + to.IncrementReferenceCount(); + + _buffer = to; + } + } } } diff --git a/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs b/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs index e4dcd916..6a786a96 100644 --- a/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs +++ b/Ryujinx.Graphics.Vulkan/MemoryAllocator.cs @@ -28,32 +28,25 @@ namespace Ryujinx.Graphics.Vulkan public MemoryAllocation AllocateDeviceMemory( MemoryRequirements requirements, - MemoryPropertyFlags flags = 0) + MemoryPropertyFlags flags = 0, + bool isBuffer = false) { - return AllocateDeviceMemory(requirements, flags, flags); - } - - public MemoryAllocation AllocateDeviceMemory( - MemoryRequirements requirements, - MemoryPropertyFlags flags, - MemoryPropertyFlags alternativeFlags) - { - int memoryTypeIndex = FindSuitableMemoryTypeIndex(requirements.MemoryTypeBits, flags, alternativeFlags); + int memoryTypeIndex = FindSuitableMemoryTypeIndex(requirements.MemoryTypeBits, flags); if (memoryTypeIndex < 0) { return default; } bool map = flags.HasFlag(MemoryPropertyFlags.HostVisibleBit); - return Allocate(memoryTypeIndex, requirements.Size, requirements.Alignment, map); + return Allocate(memoryTypeIndex, requirements.Size, requirements.Alignment, map, isBuffer); } - private MemoryAllocation Allocate(int memoryTypeIndex, ulong size, ulong alignment, bool map) + private MemoryAllocation Allocate(int memoryTypeIndex, ulong size, ulong alignment, bool map, bool isBuffer) { for (int i = 0; i < _blockLists.Count; i++) { var bl = _blockLists[i]; - if (bl.MemoryTypeIndex == memoryTypeIndex) + if (bl.MemoryTypeIndex == memoryTypeIndex && bl.ForBuffer == isBuffer) { lock (bl) { @@ -62,18 +55,15 @@ namespace Ryujinx.Graphics.Vulkan } } - var newBl = new MemoryAllocatorBlockList(_api, _device, memoryTypeIndex, _blockAlignment); + var newBl = new MemoryAllocatorBlockList(_api, _device, memoryTypeIndex, _blockAlignment, isBuffer); _blockLists.Add(newBl); return newBl.Allocate(size, alignment, map); } private int FindSuitableMemoryTypeIndex( uint memoryTypeBits, - MemoryPropertyFlags flags, - MemoryPropertyFlags alternativeFlags) + MemoryPropertyFlags flags) { - int bestCandidateIndex = -1; - for (int i = 0; i < _physicalDeviceMemoryProperties.MemoryTypeCount; i++) { var type = _physicalDeviceMemoryProperties.MemoryTypes[i]; @@ -84,14 +74,27 @@ namespace Ryujinx.Graphics.Vulkan { return i; } - else if (type.PropertyFlags.HasFlag(alternativeFlags)) - { - bestCandidateIndex = i; - } } } - return bestCandidateIndex; + return -1; + } + + public static bool IsDeviceMemoryShared(Vk api, PhysicalDevice physicalDevice) + { + // The device is regarded as having shared memory if all heaps have the device local bit. + + api.GetPhysicalDeviceMemoryProperties(physicalDevice, out var properties); + + for (int i = 0; i < properties.MemoryHeapCount; i++) + { + if (!properties.MemoryHeaps[i].Flags.HasFlag(MemoryHeapFlags.DeviceLocalBit)) + { + return false; + } + } + + return true; } public void Dispose() diff --git a/Ryujinx.Graphics.Vulkan/MemoryAllocatorBlockList.cs b/Ryujinx.Graphics.Vulkan/MemoryAllocatorBlockList.cs index dc3eb598..e564cb26 100644 --- a/Ryujinx.Graphics.Vulkan/MemoryAllocatorBlockList.cs +++ b/Ryujinx.Graphics.Vulkan/MemoryAllocatorBlockList.cs @@ -162,15 +162,17 @@ namespace Ryujinx.Graphics.Vulkan private readonly Device _device; public int MemoryTypeIndex { get; } + public bool ForBuffer { get; } private readonly int _blockAlignment; - public MemoryAllocatorBlockList(Vk api, Device device, int memoryTypeIndex, int blockAlignment) + public MemoryAllocatorBlockList(Vk api, Device device, int memoryTypeIndex, int blockAlignment, bool forBuffer) { _blocks = new List<Block>(); _api = api; _device = device; MemoryTypeIndex = memoryTypeIndex; + ForBuffer = forBuffer; _blockAlignment = blockAlignment; } diff --git a/Ryujinx.Graphics.Vulkan/PipelineBase.cs b/Ryujinx.Graphics.Vulkan/PipelineBase.cs index 3abab065..6c2f1684 100644 --- a/Ryujinx.Graphics.Vulkan/PipelineBase.cs +++ b/Ryujinx.Graphics.Vulkan/PipelineBase.cs @@ -1297,6 +1297,25 @@ namespace Ryujinx.Graphics.Vulkan SignalStateChange(); } + public void SwapBuffer(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) + { + _indexBuffer.Swap(from, to); + + for (int i = 0; i < _vertexBuffers.Length; i++) + { + _vertexBuffers[i].Swap(from, to); + } + + for (int i = 0; i < _transformFeedbackBuffers.Length; i++) + { + _transformFeedbackBuffers[i].Swap(from, to); + } + + _descriptorSetUpdater.SwapBuffer(from, to); + + SignalCommandBufferChange(); + } + public unsafe void TextureBarrier() { MemoryBarrier memoryBarrier = new MemoryBarrier() diff --git a/Ryujinx.Graphics.Vulkan/PipelineFull.cs b/Ryujinx.Graphics.Vulkan/PipelineFull.cs index 6c026a07..8026103e 100644 --- a/Ryujinx.Graphics.Vulkan/PipelineFull.cs +++ b/Ryujinx.Graphics.Vulkan/PipelineFull.cs @@ -17,10 +17,13 @@ namespace Ryujinx.Graphics.Vulkan private ulong _byteWeight; + private List<BufferHolder> _backingSwaps; + public PipelineFull(VulkanRenderer gd, Device device) : base(gd, device) { _activeQueries = new List<(QueryPool, bool)>(); _pendingQueryCopies = new(); + _backingSwaps = new(); CommandBuffer = (Cbs = gd.CommandBufferPool.Rent()).CommandBuffer; } @@ -185,6 +188,20 @@ namespace Ryujinx.Graphics.Vulkan } } + private void TryBackingSwaps() + { + CommandBufferScoped? cbs = null; + + _backingSwaps.RemoveAll((holder) => holder.TryBackingSwap(ref cbs)); + + cbs?.Dispose(); + } + + public void AddBackingSwap(BufferHolder holder) + { + _backingSwaps.Add(holder); + } + public void Restore() { if (Pipeline != null) @@ -230,6 +247,8 @@ namespace Ryujinx.Graphics.Vulkan Gd.ResetCounterPool(); + TryBackingSwaps(); + Restore(); } diff --git a/Ryujinx.Graphics.Vulkan/TextureBuffer.cs b/Ryujinx.Graphics.Vulkan/TextureBuffer.cs index bf9a6ead..738bf57d 100644 --- a/Ryujinx.Graphics.Vulkan/TextureBuffer.cs +++ b/Ryujinx.Graphics.Vulkan/TextureBuffer.cs @@ -57,12 +57,12 @@ namespace Ryujinx.Graphics.Vulkan throw new NotSupportedException(); } - public ReadOnlySpan<byte> GetData() + public PinnedSpan<byte> GetData() { return _gd.GetBufferData(_bufferHandle, _offset, _size); } - public ReadOnlySpan<byte> GetData(int layer, int level) + public PinnedSpan<byte> GetData(int layer, int level) { return GetData(); } @@ -128,7 +128,7 @@ namespace Ryujinx.Graphics.Vulkan { if (_bufferView == null) { - _bufferView = _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size); + _bufferView = _gd.BufferManager.CreateView(_bufferHandle, VkFormat, _offset, _size, ReleaseImpl); } return _bufferView?.Get(cbs, _offset, _size).Value ?? default; @@ -147,7 +147,7 @@ namespace Ryujinx.Graphics.Vulkan return bufferView.Get(cbs, _offset, _size).Value; } - bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size); + bufferView = _gd.BufferManager.CreateView(_bufferHandle, vkFormat, _offset, _size, ReleaseImpl); if (bufferView != null) { diff --git a/Ryujinx.Graphics.Vulkan/TextureView.cs b/Ryujinx.Graphics.Vulkan/TextureView.cs index 264ecf5d..cd280d5f 100644 --- a/Ryujinx.Graphics.Vulkan/TextureView.cs +++ b/Ryujinx.Graphics.Vulkan/TextureView.cs @@ -531,7 +531,7 @@ namespace Ryujinx.Graphics.Vulkan return bitmap; } - public ReadOnlySpan<byte> GetData() + public PinnedSpan<byte> GetData() { BackgroundResource resources = _gd.BackgroundResources.Get(); @@ -539,15 +539,15 @@ namespace Ryujinx.Graphics.Vulkan { _gd.FlushAllCommands(); - return GetData(_gd.CommandBufferPool, resources.GetFlushBuffer()); + return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer())); } else { - return GetData(resources.GetPool(), resources.GetFlushBuffer()); + return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer())); } } - public ReadOnlySpan<byte> GetData(int layer, int level) + public PinnedSpan<byte> GetData(int layer, int level) { BackgroundResource resources = _gd.BackgroundResources.Get(); @@ -555,11 +555,11 @@ namespace Ryujinx.Graphics.Vulkan { _gd.FlushAllCommands(); - return GetData(_gd.CommandBufferPool, resources.GetFlushBuffer(), layer, level); + return PinnedSpan<byte>.UnsafeFromSpan(GetData(_gd.CommandBufferPool, resources.GetFlushBuffer(), layer, level)); } else { - return GetData(resources.GetPool(), resources.GetFlushBuffer(), layer, level); + return PinnedSpan<byte>.UnsafeFromSpan(GetData(resources.GetPool(), resources.GetFlushBuffer(), layer, level)); } } diff --git a/Ryujinx.Graphics.Vulkan/VertexBufferState.cs b/Ryujinx.Graphics.Vulkan/VertexBufferState.cs index 7a022010..c4856019 100644 --- a/Ryujinx.Graphics.Vulkan/VertexBufferState.cs +++ b/Ryujinx.Graphics.Vulkan/VertexBufferState.cs @@ -129,6 +129,17 @@ namespace Ryujinx.Graphics.Vulkan return _buffer == buffer; } + public void Swap(Auto<DisposableBuffer> from, Auto<DisposableBuffer> to) + { + if (_buffer == from) + { + _buffer.DecrementReferenceCount(); + to.IncrementReferenceCount(); + + _buffer = to; + } + } + public void Dispose() { // Only dispose if this buffer is not refetched on each bind. diff --git a/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs b/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs index 8d4e54c4..7e7d3036 100644 --- a/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs +++ b/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs @@ -80,6 +80,7 @@ namespace Ryujinx.Graphics.Vulkan internal bool IsAmdGcn { get; private set; } internal bool IsMoltenVk { get; private set; } internal bool IsTBDR { get; private set; } + internal bool IsSharedMemory { get; private set; } public string GpuVendor { get; private set; } public string GpuRenderer { get; private set; } public string GpuVersion { get; private set; } @@ -313,6 +314,8 @@ namespace Ryujinx.Graphics.Vulkan portabilityFlags, vertexBufferAlignment); + IsSharedMemory = MemoryAllocator.IsDeviceMemoryShared(Api, _physicalDevice); + MemoryAllocator = new MemoryAllocator(Api, _physicalDevice, _device, properties.Limits.MaxMemoryAllocationCount); CommandBufferPool = VulkanInitialization.CreateCommandBufferPool(Api, _device, Queue, QueueLock, queueFamilyIndex); @@ -373,9 +376,9 @@ namespace Ryujinx.Graphics.Vulkan _initialized = true; } - public BufferHandle CreateBuffer(int size) + public BufferHandle CreateBuffer(int size, BufferHandle storageHint) { - return BufferManager.CreateWithHandle(this, size, false); + return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint); } public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info) @@ -439,7 +442,7 @@ namespace Ryujinx.Graphics.Vulkan _syncManager.RegisterFlush(); } - public ReadOnlySpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) + public PinnedSpan<byte> GetBufferData(BufferHandle buffer, int offset, int size) { return BufferManager.GetData(buffer, offset, size); } |