From 1df6c07f78c4c3b8c7fc679d7466f79a10c2d496 Mon Sep 17 00:00:00 2001
From: gdkchan <gab.dark.100@gmail.com>
Date: Mon, 4 Dec 2023 16:30:19 -0300
Subject: Implement support for multi-range buffers using Vulkan sparse
mappings (#5427)
* Pass MultiRange to BufferManager
* Implement support for multi-range buffers using Vulkan sparse mappings
* Use multi-range for remaining buffers, delete old methods
* Assume that more buffers are contiguous
* Dispose multi-range buffers after they are removed from the list
* Properly init BufferBounds for constant and storage buffers
* Do not try reading zero bytes data from an unmapped address on the shader cache + PR feedback
* Fix misaligned sparse buffer offsets
* Null check can be simplified
* PR feedback
---
src/Ryujinx.Graphics.GAL/BufferAccess.cs | 10 +-
src/Ryujinx.Graphics.GAL/Capabilities.cs | 3 +
src/Ryujinx.Graphics.GAL/IRenderer.cs | 9 +-
.../Multithreading/CommandHelper.cs | 1 +
.../Multithreading/CommandType.cs | 1 +
.../Commands/Renderer/CreateBufferCommand.cs | 6 +-
.../Commands/Renderer/CreateBufferSparseCommand.cs | 25 +
.../Multithreading/ThreadedRenderer.cs | 17 +-
src/Ryujinx.Graphics.Gpu/Engine/MME/MacroHLE.cs | 15 +-
.../Engine/Threed/ComputeDraw/VtgAsComputeState.cs | 6 +-
.../Engine/Threed/DrawManager.cs | 15 +-
.../Engine/Threed/ThreedClass.cs | 13 +-
.../Image/TextureBindingsManager.cs | 8 +-
src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs | 19 +-
src/Ryujinx.Graphics.Gpu/Memory/BufferBounds.cs | 21 +-
src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs | 557 ++++++++++++++++++---
src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs | 136 ++---
.../Memory/BufferTextureBinding.cs | 19 +-
src/Ryujinx.Graphics.Gpu/Memory/IndexBuffer.cs | 5 +-
src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs | 12 +
.../Memory/MultiRangeBuffer.cs | 60 +++
src/Ryujinx.Graphics.Gpu/Memory/VertexBuffer.cs | 5 +-
.../Memory/VirtualBufferCache.cs | 238 +++++++++
src/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs | 23 +-
.../Shader/ShaderSpecializationState.cs | 4 +-
src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs | 18 +-
.../BufferAllocationType.cs | 1 +
src/Ryujinx.Graphics.Vulkan/BufferHolder.cs | 40 +-
src/Ryujinx.Graphics.Vulkan/BufferManager.cs | 119 ++++-
src/Ryujinx.Graphics.Vulkan/Constants.cs | 2 +
src/Ryujinx.Graphics.Vulkan/EnumConversion.cs | 10 +-
src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs | 15 +-
src/Ryujinx.Memory/Range/MultiRange.cs | 21 +-
33 files changed, 1231 insertions(+), 223 deletions(-)
create mode 100644 src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferSparseCommand.cs
create mode 100644 src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
create mode 100644 src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs
(limited to 'src')
diff --git a/src/Ryujinx.Graphics.GAL/BufferAccess.cs b/src/Ryujinx.Graphics.GAL/BufferAccess.cs
index e7d7ceb0..faefa518 100644
--- a/src/Ryujinx.Graphics.GAL/BufferAccess.cs
+++ b/src/Ryujinx.Graphics.GAL/BufferAccess.cs
@@ -1,9 +1,13 @@
+using System;
+
namespace Ryujinx.Graphics.GAL
{
+ [Flags]
public enum BufferAccess
{
- Default,
- FlushPersistent,
- Stream
+ Default = 0,
+ FlushPersistent = 1 << 0,
+ Stream = 1 << 1,
+ SparseCompatible = 1 << 2,
}
}
diff --git a/src/Ryujinx.Graphics.GAL/Capabilities.cs b/src/Ryujinx.Graphics.GAL/Capabilities.cs
index 8959bf93..dc927eab 100644
--- a/src/Ryujinx.Graphics.GAL/Capabilities.cs
+++ b/src/Ryujinx.Graphics.GAL/Capabilities.cs
@@ -23,6 +23,7 @@ namespace Ryujinx.Graphics.GAL
public readonly bool SupportsR4G4B4A4Format;
public readonly bool SupportsScaledVertexFormats;
public readonly bool SupportsSnormBufferTextureFormat;
+ public readonly bool SupportsSparseBuffer;
public readonly bool Supports5BitComponentFormat;
public readonly bool SupportsBlendEquationAdvanced;
public readonly bool SupportsFragmentShaderInterlock;
@@ -79,6 +80,7 @@ namespace Ryujinx.Graphics.GAL
bool supportsScaledVertexFormats,
bool supportsSnormBufferTextureFormat,
bool supports5BitComponentFormat,
+ bool supportsSparseBuffer,
bool supportsBlendEquationAdvanced,
bool supportsFragmentShaderInterlock,
bool supportsFragmentShaderOrderingIntel,
@@ -130,6 +132,7 @@ namespace Ryujinx.Graphics.GAL
SupportsScaledVertexFormats = supportsScaledVertexFormats;
SupportsSnormBufferTextureFormat = supportsSnormBufferTextureFormat;
Supports5BitComponentFormat = supports5BitComponentFormat;
+ SupportsSparseBuffer = supportsSparseBuffer;
SupportsBlendEquationAdvanced = supportsBlendEquationAdvanced;
SupportsFragmentShaderInterlock = supportsFragmentShaderInterlock;
SupportsFragmentShaderOrderingIntel = supportsFragmentShaderOrderingIntel;
diff --git a/src/Ryujinx.Graphics.GAL/IRenderer.cs b/src/Ryujinx.Graphics.GAL/IRenderer.cs
index 1dabbdae..3bf56465 100644
--- a/src/Ryujinx.Graphics.GAL/IRenderer.cs
+++ b/src/Ryujinx.Graphics.GAL/IRenderer.cs
@@ -16,13 +16,10 @@ namespace Ryujinx.Graphics.GAL
void BackgroundContextAction(Action action, bool alwaysBackground = false);
- BufferHandle CreateBuffer(int size, BufferHandle storageHint);
- BufferHandle CreateBuffer(int size)
- {
- return CreateBuffer(size, BufferHandle.Null);
- }
+ BufferHandle CreateBuffer(int size, BufferAccess access = BufferAccess.Default);
+ BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint);
BufferHandle CreateBuffer(nint pointer, int size);
- BufferHandle CreateBuffer(int size, BufferAccess access);
+ BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers);
IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info);
diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs b/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs
index 8feeacf4..5bf3d328 100644
--- a/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs
+++ b/src/Ryujinx.Graphics.GAL/Multithreading/CommandHelper.cs
@@ -44,6 +44,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
Register<ActionCommand>(CommandType.Action);
Register<CreateBufferCommand>(CommandType.CreateBuffer);
Register<CreateBufferAccessCommand>(CommandType.CreateBufferAccess);
+ Register<CreateBufferSparseCommand>(CommandType.CreateBufferSparse);
Register<CreateHostBufferCommand>(CommandType.CreateHostBuffer);
Register<CreateProgramCommand>(CommandType.CreateProgram);
Register<CreateSamplerCommand>(CommandType.CreateSampler);
diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs b/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs
index 55a04573..6be63925 100644
--- a/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs
+++ b/src/Ryujinx.Graphics.GAL/Multithreading/CommandType.cs
@@ -5,6 +5,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading
Action,
CreateBuffer,
CreateBufferAccess,
+ CreateBufferSparse,
CreateHostBuffer,
CreateProgram,
CreateSampler,
diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs
index 353227b6..60a6e4bf 100644
--- a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs
+++ b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferCommand.cs
@@ -5,12 +5,14 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
public readonly CommandType CommandType => CommandType.CreateBuffer;
private BufferHandle _threadedHandle;
private int _size;
+ private BufferAccess _access;
private BufferHandle _storageHint;
- public void Set(BufferHandle threadedHandle, int size, BufferHandle storageHint)
+ public void Set(BufferHandle threadedHandle, int size, BufferAccess access, BufferHandle storageHint)
{
_threadedHandle = threadedHandle;
_size = size;
+ _access = access;
_storageHint = storageHint;
}
@@ -23,7 +25,7 @@ namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
hint = threaded.Buffers.MapBuffer(command._storageHint);
}
- threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, hint));
+ threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBuffer(command._size, command._access, hint));
}
}
}
diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferSparseCommand.cs b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferSparseCommand.cs
new file mode 100644
index 00000000..965529ad
--- /dev/null
+++ b/src/Ryujinx.Graphics.GAL/Multithreading/Commands/Renderer/CreateBufferSparseCommand.cs
@@ -0,0 +1,25 @@
+using Ryujinx.Graphics.GAL.Multithreading.Model;
+using System;
+
+namespace Ryujinx.Graphics.GAL.Multithreading.Commands.Renderer
+{
+ struct CreateBufferSparseCommand : IGALCommand, IGALCommand<CreateBufferSparseCommand>
+ {
+ public readonly CommandType CommandType => CommandType.CreateBufferSparse;
+ private BufferHandle _threadedHandle;
+ private SpanRef<BufferRange> _buffers;
+
+ public void Set(BufferHandle threadedHandle, SpanRef<BufferRange> buffers)
+ {
+ _threadedHandle = threadedHandle;
+ _buffers = buffers;
+ }
+
+ public static void Run(ref CreateBufferSparseCommand command, ThreadedRenderer threaded, IRenderer renderer)
+ {
+ Span<BufferRange> buffers = command._buffers.Get(threaded);
+ threaded.Buffers.AssignBuffer(command._threadedHandle, renderer.CreateBufferSparse(threaded.Buffers.MapBufferRanges(buffers)));
+ command._buffers.Dispose(threaded);
+ }
+ }
+}
diff --git a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs
index 0e0031b0..830fbf2d 100644
--- a/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs
+++ b/src/Ryujinx.Graphics.GAL/Multithreading/ThreadedRenderer.cs
@@ -263,10 +263,19 @@ namespace Ryujinx.Graphics.GAL.Multithreading
}
}
- public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
+ public BufferHandle CreateBuffer(int size, BufferAccess access)
{
BufferHandle handle = Buffers.CreateBufferHandle();
- New<CreateBufferCommand>().Set(handle, size, storageHint);
+ New<CreateBufferAccessCommand>().Set(handle, size, access);
+ QueueCommand();
+
+ return handle;
+ }
+
+ public BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint)
+ {
+ BufferHandle handle = Buffers.CreateBufferHandle();
+ New<CreateBufferCommand>().Set(handle, size, access, storageHint);
QueueCommand();
return handle;
@@ -281,10 +290,10 @@ namespace Ryujinx.Graphics.GAL.Multithreading
return handle;
}
- public BufferHandle CreateBuffer(int size, BufferAccess access)
+ public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
{
BufferHandle handle = Buffers.CreateBufferHandle();
- New<CreateBufferAccessCommand>().Set(handle, size, access);
+ New<CreateBufferSparseCommand>().Set(handle, CopySpan(storageBuffers));
QueueCommand();
return handle;
diff --git a/src/Ryujinx.Graphics.Gpu/Engine/MME/MacroHLE.cs b/src/Ryujinx.Graphics.Gpu/Engine/MME/MacroHLE.cs
index 7d9e1ec0..7f3772f4 100644
--- a/src/Ryujinx.Graphics.Gpu/Engine/MME/MacroHLE.cs
+++ b/src/Ryujinx.Graphics.Gpu/Engine/MME/MacroHLE.cs
@@ -5,6 +5,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Engine.GPFifo;
using Ryujinx.Graphics.Gpu.Engine.Threed;
using Ryujinx.Graphics.Gpu.Engine.Types;
+using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
@@ -392,12 +393,12 @@ namespace Ryujinx.Graphics.Gpu.Engine.MME
_processor.ThreedClass.DrawIndirect(
topology,
- indirectBufferAddress,
- 0,
+ new MultiRange(indirectBufferAddress, IndirectIndexedDataEntrySize),
+ default,
1,
IndirectIndexedDataEntrySize,
indexCount,
- Threed.IndirectDrawType.DrawIndexedIndirect);
+ IndirectDrawType.DrawIndexedIndirect);
}
else
{
@@ -494,13 +495,13 @@ namespace Ryujinx.Graphics.Gpu.Engine.MME
ulong indirectBufferSize = (ulong)maxDrawCount * (ulong)stride;
- ulong indirectBufferAddress = bufferCache.TranslateAndCreateBuffer(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize);
- ulong parameterBufferAddress = bufferCache.TranslateAndCreateBuffer(_processor.MemoryManager, parameterBufferGpuVa, 4);
+ MultiRange indirectBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, indirectBufferGpuVa, indirectBufferSize);
+ MultiRange parameterBufferRange = bufferCache.TranslateAndCreateMultiBuffers(_processor.MemoryManager, parameterBufferGpuVa, 4);
_processor.ThreedClass.DrawIndirect(
topology,
- indirectBufferAddress,
- parameterBufferAddress,
+ indirectBufferRange,
+ parameterBufferRange,
maxDrawCount,
stride,
indexCount,
diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Threed/ComputeDraw/VtgAsComputeState.cs b/src/Ryujinx.Graphics.Gpu/Engine/Threed/ComputeDraw/VtgAsComputeState.cs
index d1a333a7..6324e6a1 100644
--- a/src/Ryujinx.Graphics.Gpu/Engine/Threed/ComputeDraw/VtgAsComputeState.cs
+++ b/src/Ryujinx.Graphics.Gpu/Engine/Threed/ComputeDraw/VtgAsComputeState.cs
@@ -370,8 +370,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw
{
var memoryManager = _channel.MemoryManager;
- address = memoryManager.Translate(address);
- BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(address, size);
+ BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address, size));
ITexture bufferTexture = _vacContext.EnsureBufferTexture(index + 2, format);
bufferTexture.SetStorage(range);
@@ -412,9 +411,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw
var memoryManager = _channel.MemoryManager;
- address = memoryManager.Translate(address + indexOffset);
ulong misalign = address & ((ulong)_context.Capabilities.TextureBufferOffsetAlignment - 1);
- BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(address - misalign, size + misalign);
+ BufferRange range = memoryManager.Physical.BufferCache.GetBufferRange(memoryManager.GetPhysicalRegions(address + indexOffset - misalign, size + misalign));
misalignedOffset = (int)misalign >> shift;
SetIndexBufferTexture(reservations, range, format);
diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs b/src/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs
index 399ecdd7..8c72663f 100644
--- a/src/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs
+++ b/src/Ryujinx.Graphics.Gpu/Engine/Threed/DrawManager.cs
@@ -3,6 +3,7 @@ using Ryujinx.Graphics.Gpu.Engine.Threed.ComputeDraw;
using Ryujinx.Graphics.Gpu.Engine.Types;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Memory;
+using Ryujinx.Memory.Range;
using System;
namespace Ryujinx.Graphics.Gpu.Engine.Threed
@@ -630,8 +631,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
/// </summary>
/// <param name="engine">3D engine where this method is being called</param>
/// <param name="topology">Primitive topology</param>
- /// <param name="indirectBufferAddress">Address of the buffer with the draw parameters, such as count, first index, etc</param>
- /// <param name="parameterBufferAddress">Address of the buffer with the draw count</param>
+ /// <param name="indirectBufferRange">Memory range of the buffer with the draw parameters, such as count, first index, etc</param>
+ /// <param name="parameterBufferRange">Memory range of the buffer with the draw count</param>
/// <param name="maxDrawCount">Maximum number of draws that can be made</param>
/// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param>
/// <param name="indexCount">Maximum number of indices that the draw can consume</param>
@@ -639,8 +640,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
public void DrawIndirect(
ThreedClass engine,
PrimitiveTopology topology,
- ulong indirectBufferAddress,
- ulong parameterBufferAddress,
+ MultiRange indirectBufferRange,
+ MultiRange parameterBufferRange,
int maxDrawCount,
int stride,
int indexCount,
@@ -681,8 +682,8 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
if (hasCount)
{
- var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferAddress, (ulong)maxDrawCount * (ulong)stride);
- var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferAddress, 4);
+ var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
+ var parameterBuffer = memory.BufferCache.GetBufferRange(parameterBufferRange);
if (indexed)
{
@@ -695,7 +696,7 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
}
else
{
- var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferAddress, (ulong)stride);
+ var indirectBuffer = memory.BufferCache.GetBufferRange(indirectBufferRange);
if (indexed)
{
diff --git a/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs b/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
index 0ce2f7b5..ab1d27a1 100644
--- a/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
+++ b/src/Ryujinx.Graphics.Gpu/Engine/Threed/ThreedClass.cs
@@ -6,6 +6,7 @@ using Ryujinx.Graphics.Gpu.Engine.InlineToMemory;
using Ryujinx.Graphics.Gpu.Engine.Threed.Blender;
using Ryujinx.Graphics.Gpu.Engine.Types;
using Ryujinx.Graphics.Gpu.Synchronization;
+using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
@@ -803,22 +804,22 @@ namespace Ryujinx.Graphics.Gpu.Engine.Threed
/// Performs a indirect draw, with parameters from a GPU buffer.
/// </summary>
/// <param name="topology">Primitive topology</param>
- /// <param name="indirectBufferAddress">Address of the buffer with the draw parameters, such as count, first index, etc</param>
- /// <param name="parameterBufferAddress">Address of the buffer with the draw count</param>
+ /// <param name="indirectBufferRange">Memory range of the buffer with the draw parameters, such as count, first index, etc</param>
+ /// <param name="parameterBufferRange">Memory range of the buffer with the draw count</param>
/// <param name="maxDrawCount">Maximum number of draws that can be made</param>
- /// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferAddress"/></param>
+ /// <param name="stride">Distance in bytes between each entry on the data pointed to by <paramref name="indirectBufferRange"/></param>
/// <param name="indexCount">Maximum number of indices that the draw can consume</param>
/// <param name="drawType">Type of the indirect draw, which can be indexed or non-indexed, with or without a draw count</param>
public void DrawIndirect(
PrimitiveTopology topology,
- ulong indirectBufferAddress,
- ulong parameterBufferAddress,
+ MultiRange indirectBufferRange,
+ MultiRange parameterBufferRange,
int maxDrawCount,
int stride,
int indexCount,
IndirectDrawType drawType)
{
- _drawManager.DrawIndirect(this, topology, indirectBufferAddress, parameterBufferAddress, maxDrawCount, stride, indexCount, drawType);
+ _drawManager.DrawIndirect(this, topology, indirectBufferRange, parameterBufferRange, maxDrawCount, stride, indexCount, drawType);
}
/// <summary>
diff --git a/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs b/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs
index 8eca18b4..963bd947 100644
--- a/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs
+++ b/src/Ryujinx.Graphics.Gpu/Image/TextureBindingsManager.cs
@@ -382,7 +382,7 @@ namespace Ryujinx.Graphics.Gpu.Image
{
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, textureBufferIndex);
- cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
+ cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedTextureBufferIndex = textureBufferIndex;
if (samplerBufferIndex == textureBufferIndex)
@@ -396,7 +396,7 @@ namespace Ryujinx.Graphics.Gpu.Image
{
ref BufferBounds bounds = ref _channel.BufferManager.GetUniformBufferBounds(_isCompute, stageIndex, samplerBufferIndex);
- cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
+ cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(_channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedSamplerBufferIndex = samplerBufferIndex;
}
}
@@ -524,7 +524,7 @@ namespace Ryujinx.Graphics.Gpu.Image
// Ensure that the buffer texture is using the correct buffer as storage.
// Buffers are frequently re-created to accommodate larger data, so we need to re-bind
// to ensure we're not using a old buffer that was already deleted.
- _channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, bindingInfo.Format, false);
+ _channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range, bindingInfo, bindingInfo.Format, false);
// Cache is not used for buffer texture, it must always rebind.
state.CachedTexture = null;
@@ -661,7 +661,7 @@ namespace Ryujinx.Graphics.Gpu.Image
format = texture.Format;
}
- _channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range.GetSubRange(0).Address, texture.Size, bindingInfo, format, true);
+ _channel.BufferManager.SetBufferTextureStorage(stage, hostTexture, texture.Range, bindingInfo, format, true);
// Cache is not used for buffer texture, it must always rebind.
state.CachedTexture = null;
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
index c9286a61..12461e96 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/Buffer.cs
@@ -43,6 +43,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
public int UnmappedSequence { get; private set; }
+ /// <summary>
+ /// Indicates if the buffer can be used in a sparse buffer mapping.
+ /// </summary>
+ public bool SparseCompatible { get; }
+
/// <summary>
/// Ranges of the buffer that have been modified on the GPU.
/// Ranges defined here cannot be updated from CPU until a CPU waiting sync point is reached.
@@ -77,15 +82,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="physicalMemory">Physical memory where the buffer is mapped</param>
/// <param name="address">Start address of the buffer</param>
/// <param name="size">Size of the buffer in bytes</param>
+ /// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
/// <param name="baseBuffers">Buffers which this buffer contains, and will inherit tracking handles from</param>
- public Buffer(GpuContext context, PhysicalMemory physicalMemory, ulong address, ulong size, IEnumerable<Buffer> baseBuffers = null)
+ public Buffer(
+ GpuContext context,
+ PhysicalMemory physicalMemory,
+ ulong address,
+ ulong size,
+ bool sparseCompatible,
+ IEnumerable<Buffer> baseBuffers = null)
{
_context = context;
_physicalMemory = physicalMemory;
Address = address;
Size = size;
+ SparseCompatible = sparseCompatible;
+
+ BufferAccess access = sparseCompatible ? BufferAccess.SparseCompatible : BufferAccess.Default;
- Handle = context.Renderer.CreateBuffer((int)size, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
+ Handle = context.Renderer.CreateBuffer((int)size, access, baseBuffers?.MaxBy(x => x.Size).Handle ?? BufferHandle.Null);
_useGranular = size > GranularBufferThreshold;
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferBounds.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferBounds.cs
index a9ea04ce..aed3268a 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/BufferBounds.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferBounds.cs
@@ -1,4 +1,5 @@
using Ryujinx.Graphics.Shader;
+using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory
{
@@ -8,30 +9,28 @@ namespace Ryujinx.Graphics.Gpu.Memory
readonly struct BufferBounds
{
/// <summary>
- /// Region virtual address.
+ /// Physical memory ranges where the buffer is mapped.
/// </summary>
- public ulong Address { get; }
+ public MultiRange Range { get; }
/// <summary>
- /// Region size in bytes.
+ /// Buffer usage flags.
/// </summary>
- public ulong Size { get; }
+ public BufferUsageFlags Flags { get; }
/// <summary>
- /// Buffer usage flags.
+ /// Indicates that the backing memory for the buffer does not exist.
/// </summary>
- public BufferUsageFlags Flags { get; }
+ public bool IsUnmapped => Range.IsUnmapped;
/// <summary>
/// Creates a new buffer region.
/// </summary>
- /// <param name="address">Region address</param>
- /// <param name="size">Region size</param>
+ /// <param name="range">Physical memory ranges where the buffer is mapped</param>
/// <param name="flags">Buffer usage flags</param>
- public BufferBounds(ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
+ public BufferBounds(MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
{
- Address = address;
- Size = size;
+ Range = range;
Flags = flags;
}
}
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
index 05cc312c..bd9aa39c 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferCache.cs
@@ -11,12 +11,24 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
class BufferCache : IDisposable
{
- private const int OverlapsBufferInitialCapacity = 10;
- private const int OverlapsBufferMaxCapacity = 10000;
+ /// <summary>
+ /// Initial size for the array holding overlaps.
+ /// </summary>
+ public const int OverlapsBufferInitialCapacity = 10;
+
+ /// <summary>
+ /// Maximum size that an array holding overlaps may have after trimming.
+ /// </summary>
+ public const int OverlapsBufferMaxCapacity = 10000;
private const ulong BufferAlignmentSize = 0x1000;
private const ulong BufferAlignmentMask = BufferAlignmentSize - 1;
+ /// <summary>
+ /// Alignment required for sparse buffer mappings.
+ /// </summary>
+ public const ulong SparseBufferAlignmentSize = 0x10000;
+
private const ulong MaxDynamicGrowthSize = 0x100000;
private readonly GpuContext _context;
@@ -27,6 +39,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// Must lock for any access from other threads.
/// </remarks>
private readonly RangeList<Buffer> _buffers;
+ private readonly MultiRangeList<MultiRangeBuffer> _multiRangeBuffers;
private Buffer[] _bufferOverlaps;
@@ -47,6 +60,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_physicalMemory = physicalMemory;
_buffers = new RangeList<Buffer>();
+ _multiRangeBuffers = new MultiRangeList<MultiRangeBuffer>();
_bufferOverlaps = new Buffer[OverlapsBufferInitialCapacity];
@@ -66,45 +80,100 @@ namespace Ryujinx.Graphics.Gpu.Memory
Buffer[] overlaps = new Buffer[10];
int overlapCount;
- ulong address = ((MemoryManager)sender).Translate(e.Address);
- ulong size = e.Size;
+ MultiRange range = ((MemoryManager)sender).GetPhysicalRegions(e.Address, e.Size);
- lock (_buffers)
+ for (int index = 0; index < range.Count; index++)
{
- overlapCount = _buffers.FindOverlaps(address, size, ref overlaps);
- }
+ MemoryRange subRange = range.GetSubRange(index);
- for (int i = 0; i < overlapCount; i++)
- {
- overlaps[i].Unmapped(address, size);
+ lock (_buffers)
+ {
+ overlapCount = _buffers.FindOverlaps(subRange.Address, subRange.Size, ref overlaps);
+ }
+
+ for (int i = 0; i < overlapCount; i++)
+ {
+ overlaps[i].Unmapped(subRange.Address, subRange.Size);
+ }
}
}
/// <summary>
/// Performs address translation of the GPU virtual address, and creates a
- /// new buffer, if needed, for the specified range.
+ /// new buffer, if needed, for the specified contiguous range.
/// </summary>
/// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
/// <param name="gpuVa">Start GPU virtual address of the buffer</param>
/// <param name="size">Size in bytes of the buffer</param>
- /// <returns>CPU virtual address of the buffer, after address translation</returns>
- public ulong TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
+ /// <returns>Contiguous physical range of the buffer, after address translation</returns>
+ public MultiRange TranslateAndCreateBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size)
{
if (gpuVa == 0)
{
- return 0;
+ return new MultiRange(MemoryManager.PteUnmapped, size);
}
ulong address = memoryManager.Translate(gpuVa);
- if (address == MemoryManager.PteUnmapped)
+ if (address != MemoryManager.PteUnmapped)
{
- return 0;
+ CreateBuffer(address, size);
}
- CreateBuffer(address, size);
+ return new MultiRange(address, size);
+ }
- return address;
+ /// <summary>
+ /// Performs address translation of the GPU virtual address, and creates
+ /// new buffers, if needed, for the specified range.
+ /// </summary>
+ /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
+ /// <param name="gpuVa">Start GPU virtual address of the buffer</param>
+ /// <param name="size">Size in bytes of the buffer</param>
+ /// <returns>Physical ranges of the buffer, after address translation</returns>
+ public MultiRange TranslateAndCreateMultiBuffers(MemoryManager memoryManager, ulong gpuVa, ulong size)
+ {
+ if (gpuVa == 0)
+ {
+ return new MultiRange(MemoryManager.PteUnmapped, size);
+ }
+
+ bool supportsSparse = _context.Capabilities.SupportsSparseBuffer;
+
+ // Fast path not taken for non-contiguous ranges,
+ // since multi-range buffers are not coalesced, so a buffer that covers
+ // the entire cached range might not actually exist.
+ if (memoryManager.VirtualBufferCache.TryGetOrAddRange(gpuVa, size, supportsSparse, out MultiRange range) &&
+ range.Count == 1)
+ {
+ return range;
+ }
+
+ CreateBuffer(range);
+
+ return range;
+ }
+
+ /// <summary>
+ /// Creates a new buffer for the specified range, if it does not yet exist.
+ /// This can be used to ensure the existance of a buffer.
+ /// </summary>
+ /// <param name="range">Physical ranges of memory where the buffer data is located</param>
+ public void CreateBuffer(MultiRange range)
+ {
+ if (range.Count > 1)
+ {
+ CreateMultiRangeBuffer(range);
+ }
+ else
+ {
+ MemoryRange subRange = range.GetSubRange(0);
+
+ if (subRange.Address != MemoryManager.PteUnmapped)
+ {
+ CreateBuffer(subRange.Address, subRange.Size);
+ }
+ }
}
/// <summary>
@@ -118,7 +187,6 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong endAddress = address + size;
ulong alignedAddress = address & ~BufferAlignmentMask;
-
ulong alignedEndAddress = (endAddress + BufferAlignmentMask) & ~BufferAlignmentMask;
// The buffer must have the size of at least one page.
@@ -130,6 +198,108 @@ namespace Ryujinx.Graphics.Gpu.Memory
CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress);
}
+ /// <summary>
+ /// Creates a new buffer for the specified range, if it does not yet exist.
+ /// This can be used to ensure the existance of a buffer.
+ /// </summary>
+ /// <param name="address">Address of the buffer in memory</param>
+ /// <param name="size">Size of the buffer in bytes</param>
+ /// <param name="alignment">Alignment of the start address of the buffer in bytes</param>
+ public void CreateBuffer(ulong address, ulong size, ulong alignment)
+ {
+ ulong alignmentMask = alignment - 1;
+ ulong pageAlignmentMask = BufferAlignmentMask;
+ ulong endAddress = address + size;
+
+ ulong alignedAddress = address & ~alignmentMask;
+ ulong alignedEndAddress = (endAddress + pageAlignmentMask) & ~pageAlignmentMask;
+
+ // The buffer must have the size of at least one page.
+ if (alignedEndAddress == alignedAddress)
+ {
+ alignedEndAddress += pageAlignmentMask;
+ }
+
+ CreateBufferAligned(alignedAddress, alignedEndAddress - alignedAddress, alignment);
+ }
+
+ /// <summary>
+ /// Creates a buffer for a memory region composed of multiple physical ranges,
+ /// if it does not exist yet.
+ /// </summary>
+ /// <param name="range">Physical ranges of memory</param>
+ private void CreateMultiRangeBuffer(MultiRange range)
+ {
+ // Ensure all non-contiguous buffer we might use are sparse aligned.
+ for (int i = 0; i < range.Count; i++)
+ {
+ MemoryRange subRange = range.GetSubRange(i);
+
+ if (subRange.Address != MemoryManager.PteUnmapped)
+ {
+ CreateBuffer(subRange.Address, subRange.Size, SparseBufferAlignmentSize);
+ }
+ }
+
+ // Create sparse buffer.
+ MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
+
+ int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
+
+ for (int index = 0; index < overlapCount; index++)
+ {
+ if (overlaps[index].Range.Contains(range))
+ {
+ return;
+ }
+ }
+
+ for (int index = 0; index < overlapCount; index++)
+ {
+ if (range.Contains(overlaps[index].Range))
+ {
+ _multiRangeBuffers.Remove(overlaps[index]);
+ overlaps[index].Dispose();
+ }
+ }
+
+ BufferRange[] storages = new BufferRange[range.Count];
+ MemoryRange[] alignedSubRanges = new MemoryRange[range.Count];
+
+ ulong alignmentMask = SparseBufferAlignmentSize - 1;
+
+ for (int i = 0; i < range.Count; i++)
+ {
+ MemoryRange subRange = range.GetSubRange(i);
+
+ if (subRange.Address != MemoryManager.PteUnmapped)
+ {
+ ulong endAddress = subRange.Address + subRange.Size;
+
+ ulong alignedAddress = subRange.Address & ~alignmentMask;
+ ulong alignedEndAddress = (endAddress + alignmentMask) & ~alignmentMask;
+ ulong alignedSize = alignedEndAddress - alignedAddress;
+
+ Buffer buffer = _buffers.FindFirstOverlap(alignedAddress, alignedSize);
+ BufferRange bufferRange = buffer.GetRange(alignedAddress, alignedSize, false);
+
+ storages[i] = bufferRange;
+ alignedSubRanges[i] = new MemoryRange(alignedAddress, alignedSize);
+ }
+ else
+ {
+ ulong alignedSize = (subRange.Size + alignmentMask) & ~alignmentMask;
+
+ storages[i] = new BufferRange(BufferHandle.Null, 0, (int)alignedSize);
+ alignedSubRanges[i] = new MemoryRange(MemoryManager.PteUnmapped, alignedSize);
+ }
+ }
+
+ MultiRangeBuffer multiRangeBuffer = new(_context, new MultiRange(alignedSubRanges), storages);
+
+ _multiRangeBuffers.Add(multiRangeBuffer);
+ }
+
/// <summary>
/// Performs address translation of the GPU virtual address, and attempts to force
/// the buffer in the region as dirty.
@@ -150,7 +320,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
result.EndGpuAddress < gpuVa + size ||
result.UnmappedSequence != result.Buffer.UnmappedSequence)
{
- ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
+ MultiRange range = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
+ ulong address = range.GetSubRange(0).Address;
result = new BufferCacheEntry(address, gpuVa, GetBuffer(address, size));
_dirtyCache[gpuVa] = result;
@@ -184,7 +355,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
result.EndGpuAddress < alignedEndGpuVa ||
result.UnmappedSequence != result.Buffer.UnmappedSequence)
{
- ulong address = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
+ MultiRange range = TranslateAndCreateBuffer(memoryManager, alignedGpuVa, size);
+ ulong address = range.GetSubRange(0).Address;
result = new BufferCacheEntry(address, alignedGpuVa, GetBuffer(address, size));
_modifiedCache[alignedGpuVa] = result;
@@ -204,7 +376,8 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the buffer</param>
private void CreateBufferAligned(ulong address, ulong size)
{
- int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
+ Buffer[] overlaps = _bufferOverlaps;
+ int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
if (overlapsCount != 0)
{
@@ -215,9 +388,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
// old buffer(s) to the new buffer.
ulong endAddress = address + size;
+ Buffer overlap0 = overlaps[0];
- if (_bufferOverlaps[0].Address > address || _bufferOverlaps[0].EndAddress < endAddress)
+ if (overlap0.Address > address || overlap0.EndAddress < endAddress)
{
+ bool anySparseCompatible = false;
+
// Check if the following conditions are met:
// - We have a single overlap.
// - The overlap starts at or before the requested range. That is, the overlap happens at the end.
@@ -228,23 +404,25 @@ namespace Ryujinx.Graphics.Gpu.Memory
// Allowing for 2 pages (rather than just one) is necessary to catch cases where the
// range crosses a page, and after alignment, ends having a size of 2 pages.
if (overlapsCount == 1 &&
- address >= _bufferOverlaps[0].Address &&
- endAddress - _bufferOverlaps[0].EndAddress <= BufferAlignmentSize * 2)
+ address >= overlap0.Address &&
+ endAddress - overlap0.EndAddress <= BufferAlignmentSize * 2)
{
// Try to grow the buffer by 1.5x of its current size.
// This improves performance in the cases where the buffer is resized often by small amounts.
- ulong existingSize = _bufferOverlaps[0].Size;
+ ulong existingSize = overlap0.Size;
ulong growthSize = (existingSize + Math.Min(existingSize >> 1, MaxDynamicGrowthSize)) & ~BufferAlignmentMask;
size = Math.Max(size, growthSize);
endAddress = address + size;
- overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref _bufferOverlaps);
+ overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
}
for (int index = 0; index < overlapsCount; index++)
{
- Buffer buffer = _bufferOverlaps[index];
+ Buffer buffer = overlaps[index];
+
+ anySparseCompatible |= buffer.SparseCompatible;
address = Math.Min(address, buffer.Address);
endAddress = Math.Max(endAddress, buffer.EndAddress);
@@ -257,35 +435,91 @@ namespace Ryujinx.Graphics.Gpu.Memory
ulong newSize = endAddress - address;
- Buffer newBuffer = new(_context, _physicalMemory, address, newSize, _bufferOverlaps.Take(overlapsCount));
+ CreateBufferAligned(address, newSize, anySparseCompatible, overlaps, overlapsCount);
+ }
+ }
+ else
+ {
+ // No overlap, just create a new buffer.
+ Buffer buffer = new(_context, _physicalMemory, address, size, sparseCompatible: false);
- lock (_buffers)
- {
- _buffers.Add(newBuffer);
- }
+ lock (_buffers)
+ {
+ _buffers.Add(buffer);
+ }
+ }
- for (int index = 0; index < overlapsCount; index++)
+ ShrinkOverlapsBufferIfNeeded();
+ }
+
+ /// <summary>
+ /// Creates a new buffer for the specified range, if needed.
+ /// If a buffer where this range can be fully contained already exists,
+ /// then the creation of a new buffer is not necessary.
+ /// </summary>
+ /// <param name="address">Address of the buffer in guest memory</param>
+ /// <param name="size">Size in bytes of the buffer</param>
+ /// <param name="alignment">Alignment of the start address of the buffer</param>
+ private void CreateBufferAligned(ulong address, ulong size, ulong alignment)
+ {
+ Buffer[] overlaps = _bufferOverlaps;
+ int overlapsCount = _buffers.FindOverlapsNonOverlapping(address, size, ref overlaps);
+ bool sparseAligned = alignment >= SparseBufferAlignmentSize;
+
+ if (overlapsCount != 0)
+ {
+ // If the buffer already exists, make sure if covers the entire range,
+ // and make sure it is properly aligned, otherwise sparse mapping may fail.
+
+ ulong endAddress = address + size;
+ Buffer overlap0 = overlaps[0];
+
+ if (overlap0.Address > address ||
+ overlap0.EndAddress < endAddress ||
+ (overlap0.Address & (alignment - 1)) != 0 ||
+ (!overlap0.SparseCompatible && sparseAligned))
+ {
+ // We need to make sure the new buffer is properly aligned.
+ // However, after the range is aligned, it is possible that it
+ // overlaps more buffers, so try again after each extension
+ // and ensure we cover all overlaps.
+
+ int oldOverlapsCount;
+
+ do
{
- Buffer buffer = _bufferOverlaps[index];
+ for (int index = 0; index < overlapsCount; index++)
+ {
+ Buffer buffer = overlaps[index];
- int dstOffset = (int)(buffer.Address - newBuffer.Address);
+ address = Math.Min(address, buffer.Address);
+ endAddress = Math.Max(endAddress, buffer.EndAddress);
+ }
- buffer.CopyTo(newBuffer, dstOffset);
- newBuffer.InheritModifiedRanges(buffer);
+ address &= ~(alignment - 1);
- buffer.DecrementReferenceCount();
+ oldOverlapsCount = overlapsCount;
+ overlapsCount = _buffers.FindOverlapsNonOverlapping(address, endAddress - address, ref overlaps);
}
+ while (oldOverlapsCount != overlapsCount);
- newBuffer.SynchronizeMemory(address, newSize);
+ lock (_buffers)
+ {
+ for (int index = 0; index < overlapsCount; index++)
+ {
+ _buffers.Remove(overlaps[index]);
+ }
+ }
+
+ ulong newSize = endAddress - address;
- // Existing buffers were modified, we need to rebind everything.
- NotifyBuffersModified?.Invoke();
+ CreateBufferAligned(address, newSize, sparseAligned, overlaps, overlapsCount);
}
}
else
{
// No overlap, just create a new buffer.
- Buffer buffer = new(_context, _physicalMemory, address, size);
+ Buffer buffer = new(_context, _physicalMemory, address, size, sparseAligned);
lock (_buffers)
{
@@ -296,6 +530,73 @@ namespace Ryujinx.Graphics.Gpu.Memory
ShrinkOverlapsBufferIfNeeded();
}
+ /// <summary>
+ /// Creates a new buffer for the specified range, if needed.
+ /// If a buffer where this range can be fully contained already exists,
+ /// then the creation of a new buffer is not necessary.
+ /// </summary>
+ /// <param name="address">Address of the buffer in guest memory</param>
+ /// <param name="size">Size in bytes of the buffer</param>
+ /// <param name="sparseCompatible">Indicates if the buffer can be used in a sparse buffer mapping</param>
+ /// <param name="overlaps">Buffers overlapping the range</param>
+ /// <param name="overlapsCount">Total of overlaps</param>
+ private void CreateBufferAligned(ulong address, ulong size, bool sparseCompatible, Buffer[] overlaps, int overlapsCount)
+ {
+ Buffer newBuffer = new Buffer(_context, _physicalMemory, address, size, sparseCompatible, overlaps.Take(overlapsCount));
+
+ lock (_buffers)
+ {
+ _buffers.Add(newBuffer);
+ }
+
+ for (int index = 0; index < overlapsCount; index++)
+ {
+ Buffer buffer = overlaps[index];
+
+ int dstOffset = (int)(buffer.Address - newBuffer.Address);
+
+ buffer.CopyTo(newBuffer, dstOffset);
+ newBuffer.InheritModifiedRanges(buffer);
+
+ buffer.DecrementReferenceCount();
+ }
+
+ newBuffer.SynchronizeMemory(address, size);
+
+ // Existing buffers were modified, we need to rebind everything.
+ NotifyBuffersModified?.Invoke();
+
+ RecreateMultiRangeBuffers(address, size);
+ }
+
+ /// <summary>
+ /// Recreates all the multi-range buffers that overlaps a given physical memory range.
+ /// </summary>
+ /// <param name="address">Start address of the range</param>
+ /// <param name="size">Size of the range in bytes</param>
+ private void RecreateMultiRangeBuffers(ulong address, ulong size)
+ {
+ if ((address & (SparseBufferAlignmentSize - 1)) != 0 || (size & (SparseBufferAlignmentSize - 1)) != 0)
+ {
+ return;
+ }
+
+ MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
+
+ int overlapCount = _multiRangeBuffers.FindOverlaps(address, size, ref overlaps);
+
+ for (int index = 0; index < overlapCount; index++)
+ {
+ _multiRangeBuffers.Remove(overlaps[index]);
+ overlaps[index].Dispose();
+ }
+
+ for (int index = 0; index < overlapCount; index++)
+ {
+ CreateMultiRangeBuffer(overlaps[index].Range);
+ }
+ }
+
/// <summary>
/// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
/// </summary>
@@ -319,9 +620,63 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the copy</param>
public void CopyBuffer(MemoryManager memoryManager, ulong srcVa, ulong dstVa, ulong size)
{
- ulong srcAddress = TranslateAndCreateBuffer(memoryManager, srcVa, size);
- ulong dstAddress = TranslateAndCreateBuffer(memoryManager, dstVa, size);
+ MultiRange srcRange = TranslateAndCreateMultiBuffers(memoryManager, srcVa, size);
+ MultiRange dstRange = TranslateAndCreateMultiBuffers(memoryManager, dstVa, size);
+
+ if (srcRange.Count == 1 && dstRange.Count == 1)
+ {
+ CopyBufferSingleRange(memoryManager, srcRange.GetSubRange(0).Address, dstRange.GetSubRange(0).Address, size);
+ }
+ else
+ {
+ ulong copiedSize = 0;
+ ulong srcOffset = 0;
+ ulong dstOffset = 0;
+ int srcRangeIndex = 0;
+ int dstRangeIndex = 0;
+
+ while (copiedSize < size)
+ {
+ if (srcRange.GetSubRange(srcRangeIndex).Size == srcOffset)
+ {
+ srcRangeIndex++;
+ srcOffset = 0;
+ }
+
+ if (dstRange.GetSubRange(dstRangeIndex).Size == dstOffset)
+ {
+ dstRangeIndex++;
+ dstOffset = 0;
+ }
+
+ MemoryRange srcSubRange = srcRange.GetSubRange(srcRangeIndex);
+ MemoryRange dstSubRange = dstRange.GetSubRange(dstRangeIndex);
+
+ ulong srcSize = srcSubRange.Size - srcOffset;
+ ulong dstSize = dstSubRange.Size - dstOffset;
+ ulong copySize = Math.Min(srcSize, dstSize);
+
+ CopyBufferSingleRange(memoryManager, srcSubRange.Address + srcOffset, dstSubRange.Address + dstOffset, copySize);
+
+ srcOffset += copySize;
+ dstOffset += copySize;
+ copiedSize += copySize;
+ }
+ }
+ }
+ /// <summary>
+ /// Copy a buffer data from a given address to another.
+ /// </summary>
+ /// <remarks>
+ /// This does a GPU side copy.
+ /// </remarks>
+ /// <param name="memoryManager">GPU memory manager where the buffer is mapped</param>
+ /// <param name="srcAddress">Physical address of the copy source</param>
+ /// <param name="dstAddress">Physical address of the copy destination</param>
+ /// <param name="size">Size in bytes of the copy</param>
+ private void CopyBufferSingleRange(MemoryManager memoryManager, ulong srcAddress, ulong dstAddress, ulong size)
+ {
Buffer srcBuffer = GetBuffer(srcAddress, size);
Buffer dstBuffer = GetBuffer(dstAddress, size);
@@ -360,39 +715,98 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="value">Value to be written into the buffer</param>
public void ClearBuffer(MemoryManager memoryManager, ulong gpuVa, ulong size, uint value)
{
- ulong address = TranslateAndCreateBuffer(memoryManager, gpuVa, size);
+ MultiRange range = TranslateAndCreateMultiBuffers(memoryManager, gpuVa, size);
- Buffer buffer = GetBuffer(address, size);
+ for (int index = 0; index < range.Count; index++)
+ {
+ MemoryRange subRange = range.GetSubRange(index);
+ Buffer buffer = GetBuffer(subRange.Address, subRange.Size);
- int offset = (int)(address - buffer.Address);
+ int offset = (int)(subRange.Address - buffer.Address);
- _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)size, value);
+ _context.Renderer.Pipeline.ClearBuffer(buffer.Handle, offset, (int)subRange.Size, value);
- memoryManager.Physical.FillTrackedResource(address, size, value, ResourceKind.Buffer);
+ memoryManager.Physical.FillTrackedResource(subRange.Address, subRange.Size, value, ResourceKind.Buffer);
+ }
}
/// <summary>
- /// Gets a buffer sub-range from a start address til a page boundary after the given size.
+ /// Gets a buffer sub-range starting at a given memory address, aligned to the next page boundary.
/// </summary>
- /// <param name="address">Start address of the memory range</param>
- /// <param name="size">Size in bytes of the memory range</param>
+ /// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range starting at the given memory address</returns>
- public BufferRange GetBufferRangeAligned(ulong address, ulong size, bool write = false)
+ public BufferRange GetBufferRangeAligned(MultiRange range, bool write = false)
{
- return GetBuffer(address, size, write).GetRangeAligned(address, size, write);
+ if (range.Count > 1)
+ {
+ return GetBuffer(range, write).GetRange(range);
+ }
+ else
+ {
+ MemoryRange subRange = range.GetSubRange(0);
+ return GetBuffer(subRange.Address, subRange.Size, write).GetRangeAligned(subRange.Address, subRange.Size, write);
+ }
}
/// <summary>
/// Gets a buffer sub-range for a given memory range.
/// </summary>
- /// <param name="address">Start address of the memory range</param>
- /// <param name="size">Size in bytes of the memory range</param>
+ /// <param name="range">Physical regions of memory where the buffer is mapped</param>
/// <param name="write">Whether the buffer will be written to by this use</param>
/// <returns>The buffer sub-range for the given range</returns>
- public BufferRange GetBufferRange(ulong address, ulong size, bool write = false)
+ public BufferRange GetBufferRange(MultiRange range, bool write = false)
+ {
+ if (range.Count > 1)
+ {
+ return GetBuffer(range, write).GetRange(range);
+ }
+ else
+ {
+ MemoryRange subRange = range.GetSubRange(0);
+ return GetBuffer(subRange.Address, subRange.Size, write).GetRange(subRange.Address, subRange.Size, write);
+ }
+ }
+
+ /// <summary>
+ /// Gets a buffer for a given memory range.
+ /// A buffer overlapping with the specified range is assumed to already exist on the cache.
+ /// </summary>
+ /// <param name="range">Physical regions of memory where the buffer is mapped</param>
+ /// <param name="write">Whether the buffer will be written to by this use</param>
+ /// <returns>The buffer where the range is fully contained</returns>
+ private MultiRangeBuffer GetBuffer(MultiRange range, bool write = false)
{
- return GetBuffer(address, size, write).GetRange(address, size, write);
+ for (int i = 0; i < range.Count; i++)
+ {
+ MemoryRange subRange = range.GetSubRange(i);
+
+ Buffer subBuffer = _buffers.FindFirstOverlap(subRange.Address, subRange.Size);
+
+ subBuffer.SynchronizeMemory(subRange.Address, subRange.Size);
+
+ if (write)
+ {
+ subBuffer.SignalModified(subRange.Address, subRange.Size);
+ }
+ }
+
+ MultiRangeBuffer[] overlaps = new MultiRangeBuffer[10];
+
+ int overlapCount = _multiRangeBuffers.FindOverlaps(range, ref overlaps);
+
+ MultiRangeBuffer buffer = null;
+
+ for (int i = 0; i < overlapCount; i++)
+ {
+ if (overlaps[i].Range.Contains(range))
+ {
+ buffer = overlaps[i];
+ break;
+ }
+ }
+
+ return buffer;
}
/// <summary>
@@ -426,12 +840,33 @@ namespace Ryujinx.Graphics.Gpu.Memory
return buffer;
}
+ /// <summary>
+ /// Performs guest to host memory synchronization of a given memory range.
+ /// </summary>
+ /// <param name="range">Physical regions of memory where the buffer is mapped</param>
+ public void SynchronizeBufferRange(MultiRange range)
+ {
+ if (range.Count == 1)
+ {
+ MemoryRange subRange = range.GetSubRange(0);
+ SynchronizeBufferRange(subRange.Address, subRange.Size);
+ }
+ else
+ {
+ for (int index = 0; index < range.Count; index++)
+ {
+ MemoryRange subRange = range.GetSubRange(index);
+ SynchronizeBufferRange(subRange.Address, subRange.Size);
+ }
+ }
+ }
+
/// <summary>
/// Performs guest to host memory synchronization of a given memory range.
/// </summary>
/// <param name="address">Start address of the memory range</param>
/// <param name="size">Size in bytes of the memory range</param>
- public void SynchronizeBufferRange(ulong address, ulong size)
+ private void SynchronizeBufferRange(ulong address, ulong size)
{
if (size != 0)
{
@@ -491,7 +926,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <summary>
/// Disposes all buffers in the cache.
- /// It's an error to use the buffer manager after disposal.
+ /// It's an error to use the buffer cache after disposal.
/// </summary>
public void Dispose()
{
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
index 01f59477..c65602b5 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferManager.cs
@@ -3,6 +3,7 @@ using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Shader;
using Ryujinx.Graphics.Shader;
+using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
@@ -62,18 +63,19 @@ namespace Ryujinx.Graphics.Gpu.Memory
Bindings = new BufferDescriptor[count];
Buffers = new BufferBounds[count];
Unaligned = new bool[count];
+
+ Buffers.AsSpan().Fill(new BufferBounds(new MultiRange(MemoryManager.PteUnmapped, 0UL)));
}
/// <summary>
/// Sets the region of a buffer at a given slot.
/// </summary>
/// <param name="index">Buffer slot</param>
- /// <param name="address">Region virtual address</param>
- /// <param name="size">Region size in bytes</param>
+ /// <param name="range">Physical memory regions where the buffer is mapped</param>
/// <param name="flags">Buffer usage flags</param>
- public void SetBounds(int index, ulong address, ulong size, BufferUsageFlags flags = BufferUsageFlags.None)
+ public void SetBounds(int index, MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
{
- Buffers[index] = new BufferBounds(address, size, flags);
+ Buffers[index] = new BufferBounds(range, flags);
}
/// <summary>
@@ -120,6 +122,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
_context = context;
_channel = channel;
+ _indexBuffer.Range = new MultiRange(MemoryManager.PteUnmapped, 0UL);
_vertexBuffers = new VertexBuffer[Constants.TotalVertexBuffers];
_transformFeedbackBuffers = new BufferBounds[Constants.TotalTransformFeedbackBuffers];
@@ -150,10 +153,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="type">Type of each index buffer element</param>
public void SetIndexBuffer(ulong gpuVa, ulong size, IndexType type)
{
- ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
+ MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
- _indexBuffer.Address = address;
- _indexBuffer.Size = size;
+ _indexBuffer.Range = range;
_indexBuffer.Type = type;
_indexBufferDirty = true;
@@ -181,16 +183,15 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="divisor">Vertex divisor of the buffer, for instanced draws</param>
public void SetVertexBuffer(int index, ulong gpuVa, ulong size, int stride, int divisor)
{
- ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
+ MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
- _vertexBuffers[index].Address = address;
- _vertexBuffers[index].Size = size;
+ _vertexBuffers[index].Range = range;
_vertexBuffers[index].Stride = stride;
_vertexBuffers[index].Divisor = divisor;
_vertexBuffersDirty = true;
- if (address != 0)
+ if (!range.IsUnmapped)
{
_vertexBuffersEnableMask |= 1u << index;
}
@@ -209,9 +210,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the transform feedback buffer</param>
public void SetTransformFeedbackBuffer(int index, ulong gpuVa, ulong size)
{
- ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
+ MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
- _transformFeedbackBuffers[index] = new BufferBounds(address, size);
+ _transformFeedbackBuffers[index] = new BufferBounds(range);
_transformFeedbackBuffersDirty = true;
}
@@ -256,9 +257,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
- ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
+ MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
- _cpStorageBuffers.SetBounds(index, address, size, flags);
+ _cpStorageBuffers.SetBounds(index, range, flags);
}
/// <summary>
@@ -280,15 +281,14 @@ namespace Ryujinx.Graphics.Gpu.Memory
gpuVa = BitUtils.AlignDown<ulong>(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
- ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
+ MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
- if (buffers.Buffers[index].Address != address ||
- buffers.Buffers[index].Size != size)
+ if (!buffers.Buffers[index].Range.Equals(range))
{
_gpStorageBuffersDirty = true;
}
- buffers.SetBounds(index, address, size, flags);
+ buffers.SetBounds(index, range, flags);
}
/// <summary>
@@ -300,9 +300,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the storage buffer</param>
public void SetComputeUniformBuffer(int index, ulong gpuVa, ulong size)
{
- ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
+ MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
- _cpUniformBuffers.SetBounds(index, address, size);
+ _cpUniformBuffers.SetBounds(index, range);
}
/// <summary>
@@ -315,9 +315,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <param name="size">Size in bytes of the storage buffer</param>
public void SetGraphicsUniformBuffer(int stage, int index, ulong gpuVa, ulong size)
{
- ulong address = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
+ MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
- _gpUniformBuffers[stage].SetBounds(index, address, size);
+ _gpUniformBuffers[stage].SetBounds(index, range);
_gpUniformBuffersDirty = true;
}
@@ -379,7 +379,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < _cpUniformBuffers.Buffers.Length; i++)
{
- if (_cpUniformBuffers.Buffers[i].Address != 0)
+ if (!_cpUniformBuffers.Buffers[i].IsUnmapped)
{
mask |= 1u << i;
}
@@ -399,7 +399,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
for (int i = 0; i < _gpUniformBuffers[stage].Buffers.Length; i++)
{
- if (_gpUniformBuffers[stage].Buffers[i].Address != 0)
+ if (!_gpUniformBuffers[stage].Buffers[i].IsUnmapped)
{
mask |= 1u << i;
}
@@ -415,7 +415,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
public ulong GetComputeUniformBufferAddress(int index)
{
- return _cpUniformBuffers.Buffers[index].Address;
+ return _cpUniformBuffers.Buffers[index].Range.GetSubRange(0).Address;
}
/// <summary>
@@ -426,7 +426,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// <returns>The uniform buffer address, or an undefined value if the buffer is not currently bound</returns>
public ulong GetGraphicsUniformBufferAddress(int stage, int index)
{
- return _gpUniformBuffers[stage].Buffers[index].Address;
+ return _gpUniformBuffers[stage].Buffers[index].Range.GetSubRange(0).Address;
}
/// <summary>
@@ -477,7 +477,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
foreach (var binding in _bufferTextures)
{
var isStore = binding.BindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore);
- var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Address, binding.Size, isStore);
+ var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Range, isStore);
binding.Texture.SetStorage(range);
// The texture must be rebound to use the new storage if it was updated.
@@ -511,16 +511,16 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
_indexBufferDirty = false;
- if (_indexBuffer.Address != 0)
+ if (!_indexBuffer.Range.IsUnmapped)
{
- BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Address, _indexBuffer.Size);
+ BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Range);
_context.Renderer.Pipeline.SetIndexBuffer(buffer, _indexBuffer.Type);
}
}
- else if (_indexBuffer.Address != 0)
+ else if (!_indexBuffer.Range.IsUnmapped)
{
- bufferCache.SynchronizeBufferRange(_indexBuffer.Address, _indexBuffer.Size);
+ bufferCache.SynchronizeBufferRange(_indexBuffer.Range);
}
}
else if (_rebind)
@@ -540,12 +540,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
VertexBuffer vb = _vertexBuffers[index];
- if (vb.Address == 0)
+ if (vb.Range.IsUnmapped)
{
continue;
}
- BufferRange buffer = bufferCache.GetBufferRange(vb.Address, vb.Size);
+ BufferRange buffer = bufferCache.GetBufferRange(vb.Range);
vertexBuffers[index] = new VertexBufferDescriptor(buffer, vb.Stride, vb.Divisor);
}
@@ -558,12 +558,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
VertexBuffer vb = _vertexBuffers[index];
- if (vb.Address == 0)
+ if (vb.Range.IsUnmapped)
{
continue;
}
- bufferCache.SynchronizeBufferRange(vb.Address, vb.Size);
+ bufferCache.SynchronizeBufferRange(vb.Range);
}
}
@@ -579,13 +579,13 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
BufferBounds tfb = _transformFeedbackBuffers[index];
- if (tfb.Address == 0)
+ if (tfb.IsUnmapped)
{
tfbs[index] = BufferRange.Empty;
continue;
}
- tfbs[index] = bufferCache.GetBufferRange(tfb.Address, tfb.Size, write: true);
+ tfbs[index] = bufferCache.GetBufferRange(tfb.Range, write: true);
}
_context.Renderer.Pipeline.SetTransformFeedbackBuffers(tfbs);
@@ -600,21 +600,39 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
BufferBounds tfb = _transformFeedbackBuffers[index];
- if (tfb.Address == 0)
+ if (tfb.IsUnmapped)
{
buffers[index] = new BufferAssignment(index, BufferRange.Empty);
}
else
{
- ulong endAddress = tfb.Address + tfb.Size;
- ulong address = BitUtils.AlignDown(tfb.Address, (ulong)alignment);
- ulong size = endAddress - address;
+ MultiRange range = tfb.Range;
+ ulong address0 = range.GetSubRange(0).Address;
+ ulong address = BitUtils.AlignDown(address0, (ulong)alignment);
+
+ if (range.Count == 1)
+ {
+ range = new MultiRange(address, range.GetSubRange(0).Size + (address0 - address));
+ }
+ else
+ {
+ MemoryRange[] subRanges = new MemoryRange[range.Count];
+
+ subRanges[0] = new MemoryRange(address, range.GetSubRange(0).Size + (address0 - address));
+
+ for (int i = 1; i < range.Count; i++)
+ {
+ subRanges[i] = range.GetSubRange(i);
+ }
+
+ range = new MultiRange(subRanges);
+ }
- int tfeOffset = ((int)tfb.Address & (alignment - 1)) / 4;
+ int tfeOffset = ((int)address0 & (alignment - 1)) / 4;
_context.SupportBufferUpdater.SetTfeOffset(index, tfeOffset);
- buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(address, size, write: true));
+ buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(range, write: true));
}
}
@@ -627,12 +645,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
{
BufferBounds tfb = _transformFeedbackBuffers[index];
- if (tfb.Address == 0)
+ if (tfb.IsUnmapped)
{
continue;
}
- bufferCache.SynchronizeBufferRange(tfb.Address, tfb.Size);
+ bufferCache.SynchronizeBufferRange(tfb.Range);
}
}
@@ -688,12 +706,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
- if (bounds.Address != 0)
+ if (!bounds.IsUnmapped)
{
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage
- ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
- : bufferCache.GetBufferRange(bounds.Address, bounds.Size);
+ ? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
+ : bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
}
@@ -725,12 +743,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
- if (bounds.Address != 0)
+ if (!bounds.IsUnmapped)
{
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage
- ? bufferCache.GetBufferRangeAligned(bounds.Address, bounds.Size, isWrite)
- : bufferCache.GetBufferRange(bounds.Address, bounds.Size);
+ ? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
+ : bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
}
@@ -778,12 +796,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
BufferBounds bounds = buffers.Buffers[binding.Slot];
- if (bounds.Address == 0)
+ if (bounds.IsUnmapped)
{
continue;
}
- _channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Address, bounds.Size);
+ _channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Range);
}
}
}
@@ -793,23 +811,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
/// <param name="stage">Shader stage accessing the texture</param>
/// <param name="texture">Buffer texture</param>
- /// <param name="address">Address of the buffer in memory</param>
- /// <param name="size">Size of the buffer in bytes</param>
+ /// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
/// <param name="bindingInfo">Binding info for the buffer texture</param>
/// <param name="format">Format of the buffer texture</param>
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
public void SetBufferTextureStorage(
ShaderStage stage,
ITexture texture,
- ulong address,
- ulong size,
+ MultiRange range,
TextureBindingInfo bindingInfo,
Format format,
bool isImage)
{
- _channel.MemoryManager.Physical.BufferCache.CreateBuffer(address, size);
+ _channel.MemoryManager.Physical.BufferCache.CreateBuffer(range);
- _bufferTextures.Add(new BufferTextureBinding(stage, texture, address, size, bindingInfo, format, isImage));
+ _bufferTextures.Add(new BufferTextureBinding(stage, texture, range, bindingInfo, format, isImage));
}
/// <summary>
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/BufferTextureBinding.cs b/src/Ryujinx.Graphics.Gpu/Memory/BufferTextureBinding.cs
index c5f0401b..bf0beffa 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/BufferTextureBinding.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/BufferTextureBinding.cs
@@ -1,6 +1,7 @@
using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Shader;
+using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory
{
@@ -20,14 +21,9 @@ namespace Ryujinx.Graphics.Gpu.Memory
public ITexture Texture { get; }
/// <summary>
- /// The base address of the buffer binding.
+ /// Physical ranges of memory where the buffer texture data is located.
/// </summary>
- public ulong Address { get; }
-
- /// <summary>
- /// The size of the buffer binding in bytes.
- /// </summary>
- public ulong Size { get; }
+ public MultiRange Range { get; }
/// <summary>
/// The image or sampler binding info for the buffer texture.
@@ -49,24 +45,21 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
/// <param name="stage">Shader stage accessing the texture</param>
/// <param name="texture">Buffer texture</param>
- /// <param name="address">Base address</param>
- /// <param name="size">Size in bytes</param>
+ /// <param name="range">Physical ranges of memory where the buffer texture data is located</param>
/// <param name="bindingInfo">Binding info</param>
/// <param name="format">Binding format</param>
/// <param name="isImage">Whether the binding is for an image or a sampler</param>
public BufferTextureBinding(
ShaderStage stage,
ITexture texture,
- ulong address,
- ulong size,
+ MultiRange range,
TextureBindingInfo bindingInfo,
Format format,
bool isImage)
{
Stage = stage;
Texture = texture;
- Address = address;
- Size = size;
+ Range = range;
BindingInfo = bindingInfo;
Format = format;
IsImage = isImage;
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/IndexBuffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/IndexBuffer.cs
index c72fa50e..04114a95 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/IndexBuffer.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/IndexBuffer.cs
@@ -1,4 +1,5 @@
using Ryujinx.Graphics.GAL;
+using Ryujinx.Memory.Range;
namespace Ryujinx.Graphics.Gpu.Memory
{
@@ -7,9 +8,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
struct IndexBuffer
{
- public ulong Address;
- public ulong Size;
-
+ public MultiRange Range;
public IndexType Type;
}
}
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
index 6af12de1..5e19bddc 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/MemoryManager.cs
@@ -39,6 +39,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
internal PhysicalMemory Physical { get; }
+ /// <summary>
+ /// Virtual buffer cache.
+ /// </summary>
+ internal VirtualBufferCache VirtualBufferCache { get; }
+
/// <summary>
/// Cache of GPU counters.
/// </summary>
@@ -51,10 +56,12 @@ namespace Ryujinx.Graphics.Gpu.Memory
internal MemoryManager(PhysicalMemory physicalMemory)
{
Physical = physicalMemory;
+ VirtualBufferCache = new VirtualBufferCache(this);
CounterCache = new CounterCache();
_pageTable = new ulong[PtLvl0Size][];
MemoryUnmapped += Physical.TextureCache.MemoryUnmappedHandler;
MemoryUnmapped += Physical.BufferCache.MemoryUnmappedHandler;
+ MemoryUnmapped += VirtualBufferCache.MemoryUnmappedHandler;
MemoryUnmapped += CounterCache.MemoryUnmappedHandler;
}
@@ -508,6 +515,11 @@ namespace Ryujinx.Graphics.Gpu.Memory
regionSize += Math.Min(endVa - va, PageSize);
}
+ if (regions.Count == 0)
+ {
+ return new MultiRange(regionStart, regionSize);
+ }
+
regions.Add(new MemoryRange(regionStart, regionSize));
return new MultiRange(regions.ToArray());
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
new file mode 100644
index 00000000..e039a7a4
--- /dev/null
+++ b/src/Ryujinx.Graphics.Gpu/Memory/MultiRangeBuffer.cs
@@ -0,0 +1,60 @@
+using Ryujinx.Graphics.GAL;
+using Ryujinx.Memory.Range;
+using System;
+
+namespace Ryujinx.Graphics.Gpu.Memory
+{
+ /// <summary>
+ /// Buffer, used to store vertex and index data, uniform and storage buffers, and others.
+ /// </summary>
+ class MultiRangeBuffer : IMultiRangeItem, IDisposable
+ {
+ private readonly GpuContext _context;
+
+ /// <summary>
+ /// Host buffer handle.
+ /// </summary>
+ public BufferHandle Handle { get; }
+
+ /// <summary>
+ /// Range of memory where the data is located.
+ /// </summary>
+ public MultiRange Range { get; }
+
+ /// <summary>
+ /// Creates a new instance of the buffer.
+ /// </summary>
+ /// <param name="context">GPU context that the buffer belongs to</param>
+ /// <param name="range">Range of memory where the data is mapped</param>
+ /// <param name="storages">Backing memory for the buffers</param>
+ public MultiRangeBuffer(GpuContext context, MultiRange range, ReadOnlySpan<BufferRange> storages)
+ {
+ _context = context;
+ Range = range;
+ Handle = context.Renderer.CreateBufferSparse(storages);
+ }
+
+ /// <summary>
+ /// Gets a sub-range from the buffer.
+ /// </summary>
+ /// <remarks>
+ /// This can be used to bind and use sub-ranges of the buffer on the host API.
+ /// </remarks>
+ /// <param name="range">Range of memory where the data is mapped</param>
+ /// <returns>The buffer sub-range</returns>
+ public BufferRange GetRange(MultiRange range)
+ {
+ int offset = Range.FindOffset(range);
+
+ return new BufferRange(Handle, offset, (int)range.GetSize());
+ }
+
+ /// <summary>
+ /// Disposes the host buffer.
+ /// </summary>
+ public void Dispose()
+ {
+ _context.Renderer.DeleteBuffer(Handle);
+ }
+ }
+}
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VertexBuffer.cs b/src/Ryujinx.Graphics.Gpu/Memory/VertexBuffer.cs
index ac334881..206e1b48 100644
--- a/src/Ryujinx.Graphics.Gpu/Memory/VertexBuffer.cs
+++ b/src/Ryujinx.Graphics.Gpu/Memory/VertexBuffer.cs
@@ -1,3 +1,5 @@
+using Ryujinx.Memory.Range;
+
namespace Ryujinx.Graphics.Gpu.Memory
{
/// <summary>
@@ -5,8 +7,7 @@ namespace Ryujinx.Graphics.Gpu.Memory
/// </summary>
struct VertexBuffer
{
- public ulong Address;
- public ulong Size;
+ public MultiRange Range;
public int Stride;
public int Divisor;
}
diff --git a/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs b/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs
new file mode 100644
index 00000000..858c5e3b
--- /dev/null
+++ b/src/Ryujinx.Graphics.Gpu/Memory/VirtualBufferCache.cs
@@ -0,0 +1,238 @@
+using Ryujinx.Memory.Range;
+using System;
+using System.Collections.Concurrent;
+using System.Threading;
+
+namespace Ryujinx.Graphics.Gpu.Memory
+{
+ /// <summary>
+ /// Virtual buffer cache.
+ /// </summary>
+ class VirtualBufferCache
+ {
+ private readonly MemoryManager _memoryManager;
+
+ /// <summary>
+ /// Represents a GPU virtual memory range.
+ /// </summary>
+ private readonly struct VirtualRange : IRange
+ {
+ /// <summary>
+ /// GPU virtual address where the range starts.
+ /// </summary>
+ public ulong Address { get; }
+
+ /// <summary>
+ /// Size of the range in bytes.
+ /// </summary>
+ public ulong Size { get; }
+
+ /// <summary>
+ /// GPU virtual address where the range ends.
+ /// </summary>
+ public ulong EndAddress => Address + Size;
+
+ /// <summary>
+ /// Physical regions where the GPU virtual region is mapped.
+ /// </summary>
+ public MultiRange Range { get; }
+
+ /// <summary>
+ /// Creates a new virtual memory range.
+ /// </summary>
+ /// <param name="address">GPU virtual address where the range starts</param>
+ /// <param name="size">Size of the range in bytes</param>
+ /// <param name="range">Physical regions where the GPU virtual region is mapped</param>
+ public VirtualRange(ulong address, ulong size, MultiRange range)
+ {
+ Address = address;
+ Size = size;
+ Range = range;
+ }
+
+ /// <summary>
+ /// Checks if a given range overlaps with the buffer.
+ /// </summary>
+ /// <param name="address">Start address of the range</param>
+ /// <param name="size">Size in bytes of the range</param>
+ /// <returns>True if the range overlaps, false otherwise</returns>
+ public bool OverlapsWith(ulong address, ulong size)
+ {
+ return Address < address + size && address < EndAddress;
+ }
+ }
+
+ private readonly RangeList<VirtualRange> _virtualRanges;
+ private VirtualRange[] _virtualRangeOverlaps;
+ private readonly ConcurrentQueue<VirtualRange> _deferredUnmaps;
+ private int _hasDeferredUnmaps;
+
+ /// <summary>
+ /// Creates a new instance of the virtual buffer cache.
+ /// </summary>
+ /// <param name="memoryManager">Memory manager that the virtual buffer cache belongs to</param>
+ public VirtualBufferCache(MemoryManager memoryManager)
+ {
+ _memoryManager = memoryManager;
+ _virtualRanges = new RangeList<VirtualRange>();
+ _virtualRangeOverlaps = new VirtualRange[BufferCache.OverlapsBufferInitialCapacity];
+ _deferredUnmaps = new ConcurrentQueue<VirtualRange>();
+ }
+
+ /// <summary>
+ /// Handles removal of buffers written to a memory region being unmapped.
+ /// </summary>
+ /// <param name="sender">Sender object</param>
+ /// <param name="e">Event arguments</param>
+ public void MemoryUnmappedHandler(object sender, UnmapEventArgs e)
+ {
+ void EnqueueUnmap()
+ {
+ _deferredUnmaps.Enqueue(new VirtualRange(e.Address, e.Size, default));
+
+ Interlocked.Exchange(ref _hasDeferredUnmaps, 1);
+ }
+
+ e.AddRemapAction(EnqueueUnmap);
+ }
+
+ /// <summary>
+ /// Tries to get a existing, cached physical range for the specified virtual region.
+ /// If no cached range is found, a new one is created and added.
+ /// </summary>
+ /// <param name="gpuVa">GPU virtual address to get the physical range from</param>
+ /// <param name="size">Size in bytes of the region</param>
+ /// <param name="supportsSparse">Indicates host support for sparse buffer mapping of non-contiguous ranges</param>
+ /// <param name="range">Physical range for the specified GPU virtual region</param>
+ /// <returns>True if the range already existed, false if a new one was created and added</returns>
+ public bool TryGetOrAddRange(ulong gpuVa, ulong size, bool supportsSparse, out MultiRange range)
+ {
+ VirtualRange[] overlaps = _virtualRangeOverlaps;
+ int overlapsCount;
+
+ if (Interlocked.Exchange(ref _hasDeferredUnmaps, 0) != 0)
+ {
+ while (_deferredUnmaps.TryDequeue(out VirtualRange unmappedRange))
+ {
+ overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(unmappedRange.Address, unmappedRange.Size, ref overlaps);
+
+ for (int index = 0; index < overlapsCount; index++)
+ {
+ _virtualRanges.Remove(overlaps[index]);
+ }
+ }
+ }
+
+ bool found = false;
+
+ ulong originalVa = gpuVa;
+
+ overlapsCount = _virtualRanges.FindOverlapsNonOverlapping(gpuVa, size, ref overlaps);
+
+ if (overlapsCount != 0)
+ {
+ // The virtual range already exists. We just need to check if our range fits inside
+ // the existing one, and if not, we must extend the existing one.
+
+ ulong endAddress = gpuVa + size;
+ VirtualRange overlap0 = overlaps[0];
+
+ if (overlap0.Address > gpuVa || overlap0.EndAddress < endAddress)
+ {
+ for (int index = 0; index < overlapsCount; index++)
+ {
+ VirtualRange virtualRange = overlaps[index];
+
+ gpuVa = Math.Min(gpuVa, virtualRange.Address);
+ endAddress = Math.Max(endAddress, virtualRange.EndAddress);
+
+ _virtualRanges.Remove(virtualRange);
+ }
+
+ ulong newSize = endAddress - gpuVa;
+ MultiRange newRange = _memoryManager.GetPhysicalRegions(gpuVa, newSize);
+
+ _virtualRanges.Add(new(gpuVa, newSize, newRange));
+
+ range = newRange.Slice(originalVa - gpuVa, size);
+ }
+ else
+ {
+ found = true;
+ range = overlap0.Range.Slice(gpuVa - overlap0.Address, size);
+ }
+ }
+ else
+ {
+ // No overlap, just create a new virtual range.
+ range = _memoryManager.GetPhysicalRegions(gpuVa, size);
+
+ VirtualRange virtualRange = new(gpuVa, size, range);
+
+ _virtualRanges.Add(virtualRange);
+ }
+
+ ShrinkOverlapsBufferIfNeeded();
+
+ // If the the range is not properly aligned for sparse mapping,
+ // or if the host does not support sparse mapping, let's just
+ // force it to a single range.
+ // This might cause issues in some applications that uses sparse
+ // mappings.
+ if (!IsSparseAligned(range) || !supportsSparse)
+ {
+ range = new MultiRange(range.GetSubRange(0).Address, size);
+ }
+
+ return found;
+ }
+
+ /// <summary>
+ /// Checks if the physical memory ranges are valid for sparse mapping,
+ /// which requires all sub-ranges to be 64KB aligned.
+ /// </summary>
+ /// <param name="range">Range to check</param>
+ /// <returns>True if the range is valid for sparse mapping, false otherwise</returns>
+ private static bool IsSparseAligned(MultiRange range)
+ {
+ if (range.Count == 1)
+ {
+ return (range.GetSubRange(0).Address & (BufferCache.SparseBufferAlignmentSize - 1)) == 0;
+ }
+
+ for (int i = 0; i < range.Count; i++)
+ {
+ MemoryRange subRange = range.GetSubRange(i);
+
+ // Check if address is aligned. The address of the first sub-range can
+ // be misaligned as it is at the start.
+ if (i > 0 &&
+ subRange.Address != MemoryManager.PteUnmapped &&
+ (subRange.Address & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
+ {
+ return false;
+ }
+
+ // Check if the size is aligned. The size of the last sub-range can
+ // be misaligned as it is at the end.
+ if (i < range.Count - 1 && (subRange.Size & (BufferCache.SparseBufferAlignmentSize - 1)) != 0)
+ {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ /// <summary>
+ /// Resizes the temporary buffer used for range list intersection results, if it has grown too much.
+ /// </summary>
+ private void ShrinkOverlapsBufferIfNeeded()
+ {
+ if (_virtualRangeOverlaps.Length > BufferCache.OverlapsBufferMaxCapacity)
+ {
+ Array.Resize(ref _virtualRangeOverlaps, BufferCache.OverlapsBufferMaxCapacity);
+ }
+ }
+ }
+}
diff --git a/src/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs b/src/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
index 38be262a..af682e42 100644
--- a/src/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
+++ b/src/Ryujinx.Graphics.Gpu/Shader/ShaderCache.cs
@@ -730,8 +730,8 @@ namespace Ryujinx.Graphics.Gpu.Shader
codeA ??= memoryManager.GetSpan(vertexA.Address, vertexA.Size).ToArray();
codeB ??= memoryManager.GetSpan(currentStage.Address, currentStage.Size).ToArray();
- byte[] cb1DataA = memoryManager.Physical.GetSpan(cb1DataAddress, vertexA.Cb1DataSize).ToArray();
- byte[] cb1DataB = memoryManager.Physical.GetSpan(cb1DataAddress, currentStage.Cb1DataSize).ToArray();
+ byte[] cb1DataA = ReadArray(memoryManager, cb1DataAddress, vertexA.Cb1DataSize);
+ byte[] cb1DataB = ReadArray(memoryManager, cb1DataAddress, currentStage.Cb1DataSize);
ShaderDumpPaths pathsA = default;
ShaderDumpPaths pathsB = default;
@@ -770,7 +770,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
? channel.BufferManager.GetComputeUniformBufferAddress(1)
: channel.BufferManager.GetGraphicsUniformBufferAddress(StageToStageIndex(context.Stage), 1);
- byte[] cb1Data = memoryManager.Physical.GetSpan(cb1DataAddress, context.Cb1DataSize).ToArray();
+ byte[] cb1Data = ReadArray(memoryManager, cb1DataAddress, context.Cb1DataSize);
code ??= memoryManager.GetSpan(context.Address, context.Size).ToArray();
ShaderDumpPaths paths = dumper?.Dump(code, context.Stage == ShaderStage.Compute) ?? default;
@@ -781,6 +781,23 @@ namespace Ryujinx.Graphics.Gpu.Shader
return new TranslatedShader(new CachedShaderStage(program.Info, code, cb1Data), program);
}
+ /// <summary>
+ /// Reads data from physical memory, returns an empty array if the memory is unmapped or size is 0.
+ /// </summary>
+ /// <param name="memoryManager">Memory manager with the physical memory to read from</param>
+ /// <param name="address">Physical address of the region to read</param>
+ /// <param name="size">Size in bytes of the data</param>
+ /// <returns>An array with the data at the specified memory location</returns>
+ private static byte[] ReadArray(MemoryManager memoryManager, ulong address, int size)
+ {
+ if (address == MemoryManager.PteUnmapped || size == 0)
+ {
+ return Array.Empty<byte>();
+ }
+
+ return memoryManager.Physical.GetSpan(address, size).ToArray();
+ }
+
/// <summary>
/// Gets the index of a stage from a <see cref="ShaderStage"/>.
/// </summary>
diff --git a/src/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs b/src/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs
index a41f761b..1477b738 100644
--- a/src/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs
+++ b/src/Ryujinx.Graphics.Gpu/Shader/ShaderSpecializationState.cs
@@ -611,7 +611,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
{
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, textureBufferIndex);
- cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
+ cachedTextureBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedTextureBufferIndex = textureBufferIndex;
if (samplerBufferIndex == textureBufferIndex)
@@ -625,7 +625,7 @@ namespace Ryujinx.Graphics.Gpu.Shader
{
ref BufferBounds bounds = ref channel.BufferManager.GetUniformBufferBounds(isCompute, stageIndex, samplerBufferIndex);
- cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Address, (int)bounds.Size));
+ cachedSamplerBuffer = MemoryMarshal.Cast<byte, int>(channel.MemoryManager.Physical.GetSpan(bounds.Range));
cachedSamplerBufferIndex = samplerBufferIndex;
}
diff --git a/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs b/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs
index b3c26171..64ba4e3e 100644
--- a/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs
+++ b/src/Ryujinx.Graphics.OpenGL/OpenGLRenderer.cs
@@ -57,16 +57,11 @@ namespace Ryujinx.Graphics.OpenGL
ResourcePool = new ResourcePool();
}
- public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
- {
- return CreateBuffer(size, GAL.BufferAccess.Default);
- }
-
public BufferHandle CreateBuffer(int size, GAL.BufferAccess access)
{
BufferCount++;
- if (access == GAL.BufferAccess.FlushPersistent)
+ if (access.HasFlag(GAL.BufferAccess.FlushPersistent))
{
BufferHandle handle = Buffer.CreatePersistent(size);
@@ -80,11 +75,21 @@ namespace Ryujinx.Graphics.OpenGL
}
}
+ public BufferHandle CreateBuffer(int size, GAL.BufferAccess access, BufferHandle storageHint)
+ {
+ return CreateBuffer(size, access);
+ }
+
public BufferHandle CreateBuffer(nint pointer, int size)
{
throw new NotSupportedException();
}
+ public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
+ {
+ throw new NotSupportedException();
+ }
+
public IProgram CreateProgram(ShaderSource[] shaders, ShaderInfo info)
{
return new Program(shaders, info.FragmentOutputMap);
@@ -148,6 +153,7 @@ namespace Ryujinx.Graphics.OpenGL
supportsR4G4B4A4Format: true,
supportsSnormBufferTextureFormat: false,
supports5BitComponentFormat: true,
+ supportsSparseBuffer: false,
supportsBlendEquationAdvanced: HwCapabilities.SupportsBlendEquationAdvanced,
supportsFragmentShaderInterlock: HwCapabilities.SupportsFragmentShaderInterlock,
supportsFragmentShaderOrderingIntel: HwCapabilities.SupportsFragmentShaderOrdering,
diff --git a/src/Ryujinx.Graphics.Vulkan/BufferAllocationType.cs b/src/Ryujinx.Graphics.Vulkan/BufferAllocationType.cs
index 7987017e..345191f1 100644
--- a/src/Ryujinx.Graphics.Vulkan/BufferAllocationType.cs
+++ b/src/Ryujinx.Graphics.Vulkan/BufferAllocationType.cs
@@ -8,5 +8,6 @@ namespace Ryujinx.Graphics.Vulkan
HostMapped,
DeviceLocal,
DeviceLocalMapped,
+ Sparse,
}
}
diff --git a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs
index b1887eaa..b54ff3ab 100644
--- a/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs
+++ b/src/Ryujinx.Graphics.Vulkan/BufferHolder.cs
@@ -46,7 +46,7 @@ namespace Ryujinx.Graphics.Vulkan
private bool _lastAccessIsWrite;
- private readonly BufferAllocationType _baseType;
+ private BufferAllocationType _baseType;
private BufferAllocationType _currentType;
private bool _swapQueued;
@@ -109,6 +109,22 @@ namespace Ryujinx.Graphics.Vulkan
_flushLock = new ReaderWriterLockSlim();
}
+ public BufferHolder(VulkanRenderer gd, Device device, VkBuffer buffer, int size, Auto<MemoryAllocation>[] storageAllocations)
+ {
+ _gd = gd;
+ _device = device;
+ _waitable = new MultiFenceHolder(size);
+ _buffer = new Auto<DisposableBuffer>(new DisposableBuffer(gd.Api, device, buffer), _waitable, storageAllocations);
+ _bufferHandle = buffer.Handle;
+ Size = size;
+
+ _baseType = BufferAllocationType.Sparse;
+ _currentType = BufferAllocationType.Sparse;
+ DesiredType = BufferAllocationType.Sparse;
+
+ _flushLock = new ReaderWriterLockSlim();
+ }
+
public bool TryBackingSwap(ref CommandBufferScoped? cbs)
{
if (_swapQueued && DesiredType != _currentType)
@@ -122,7 +138,7 @@ namespace Ryujinx.Graphics.Vulkan
var currentBuffer = _buffer;
IntPtr currentMap = _map;
- (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, _currentType);
+ (VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) = _gd.BufferManager.CreateBacking(_gd, Size, DesiredType, false, false, _currentType);
if (buffer.Handle != 0)
{
@@ -253,6 +269,14 @@ namespace Ryujinx.Graphics.Vulkan
}
}
+ public void Pin()
+ {
+ if (_baseType == BufferAllocationType.Auto)
+ {
+ _baseType = _currentType;
+ }
+ }
+
public unsafe Auto<DisposableBufferView> CreateView(VkFormat format, int offset, int size, Action invalidateView)
{
var bufferViewCreateInfo = new BufferViewCreateInfo
@@ -506,6 +530,16 @@ namespace Ryujinx.Graphics.Vulkan
}
}
+ public Auto<MemoryAllocation> GetAllocation()
+ {
+ return _allocationAuto;
+ }
+
+ public (DeviceMemory, ulong) GetDeviceMemoryAndOffset()
+ {
+ return (_allocation.Memory, _allocation.Offset);
+ }
+
public void SignalWrite(int offset, int size)
{
ConsiderBackingSwap();
@@ -1072,7 +1106,7 @@ namespace Ryujinx.Graphics.Vulkan
}
else
{
- _allocationAuto.Dispose();
+ _allocationAuto?.Dispose();
}
_flushLock.EnterWriteLock();
diff --git a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs
index 20e00338..e9ac9884 100644
--- a/src/Ryujinx.Graphics.Vulkan/BufferManager.cs
+++ b/src/Ryujinx.Graphics.Vulkan/BufferManager.cs
@@ -96,25 +96,131 @@ namespace Ryujinx.Graphics.Vulkan
return Unsafe.As<ulong, BufferHandle>(ref handle64);
}
+ public unsafe BufferHandle CreateSparse(VulkanRenderer gd, ReadOnlySpan<BufferRange> storageBuffers)
+ {
+ var usage = DefaultBufferUsageFlags;
+
+ if (gd.Capabilities.SupportsIndirectParameters)
+ {
+ usage |= BufferUsageFlags.IndirectBufferBit;
+ }
+
+ ulong size = 0;
+
+ foreach (BufferRange range in storageBuffers)
+ {
+ size += (ulong)range.Size;
+ }
+
+ var bufferCreateInfo = new BufferCreateInfo()
+ {
+ SType = StructureType.BufferCreateInfo,
+ Size = size,
+ Usage = usage,
+ SharingMode = SharingMode.Exclusive,
+ Flags = BufferCreateFlags.SparseBindingBit | BufferCreateFlags.SparseAliasedBit
+ };
+
+ gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
+
+ var memoryBinds = new SparseMemoryBind[storageBuffers.Length];
+ var storageAllocations = new Auto<MemoryAllocation>[storageBuffers.Length];
+ int storageAllocationsCount = 0;
+
+ ulong dstOffset = 0;
+
+ for (int index = 0; index < storageBuffers.Length; index++)
+ {
+ BufferRange range = storageBuffers[index];
+
+ if (TryGetBuffer(range.Handle, out var existingHolder))
+ {
+ // Since this buffer now also owns the memory from the referenced buffer,
+ // we pin it to ensure the memory location will not change.
+ existingHolder.Pin();
+
+ (var memory, var offset) = existingHolder.GetDeviceMemoryAndOffset();
+
+ memoryBinds[index] = new SparseMemoryBind()
+ {
+ ResourceOffset = dstOffset,
+ Size = (ulong)range.Size,
+ Memory = memory,
+ MemoryOffset = offset + (ulong)range.Offset,
+ Flags = SparseMemoryBindFlags.None
+ };
+
+ storageAllocations[storageAllocationsCount++] = existingHolder.GetAllocation();
+ }
+ else
+ {
+ memoryBinds[index] = new SparseMemoryBind()
+ {
+ ResourceOffset = dstOffset,
+ Size = (ulong)range.Size,
+ Memory = default,
+ MemoryOffset = 0UL,
+ Flags = SparseMemoryBindFlags.None
+ };
+ }
+
+ dstOffset += (ulong)range.Size;
+ }
+
+ if (storageAllocations.Length != storageAllocationsCount)
+ {
+ Array.Resize(ref storageAllocations, storageAllocationsCount);
+ }
+
+ fixed (SparseMemoryBind* pMemoryBinds = memoryBinds)
+ {
+ SparseBufferMemoryBindInfo bufferBind = new SparseBufferMemoryBindInfo()
+ {
+ Buffer = buffer,
+ BindCount = (uint)memoryBinds.Length,
+ PBinds = pMemoryBinds
+ };
+
+ BindSparseInfo bindSparseInfo = new BindSparseInfo()
+ {
+ SType = StructureType.BindSparseInfo,
+ BufferBindCount = 1,
+ PBufferBinds = &bufferBind
+ };
+
+ gd.Api.QueueBindSparse(gd.Queue, 1, bindSparseInfo, default).ThrowOnError();
+ }
+
+ var holder = new BufferHolder(gd, _device, buffer, (int)size, storageAllocations);
+
+ BufferCount++;
+
+ ulong handle64 = (uint)_buffers.Add(holder);
+
+ return Unsafe.As<ulong, BufferHandle>(ref handle64);
+ }
+
public BufferHandle CreateWithHandle(
VulkanRenderer gd,
int size,
+ bool sparseCompatible = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default,
bool forceMirrors = false)
{
- return CreateWithHandle(gd, size, out _, baseType, storageHint, forceMirrors);
+ return CreateWithHandle(gd, size, out _, sparseCompatible, baseType, storageHint, forceMirrors);
}
public BufferHandle CreateWithHandle(
VulkanRenderer gd,
int size,
out BufferHolder holder,
+ bool sparseCompatible = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default,
bool forceMirrors = false)
{
- holder = Create(gd, size, baseType: baseType, storageHint: storageHint);
+ holder = Create(gd, size, forConditionalRendering: false, sparseCompatible, baseType, storageHint);
if (holder == null)
{
return BufferHandle.Null;
@@ -163,6 +269,7 @@ namespace Ryujinx.Graphics.Vulkan
int size,
BufferAllocationType type,
bool forConditionalRendering = false,
+ bool sparseCompatible = false,
BufferAllocationType fallbackType = BufferAllocationType.Auto)
{
var usage = DefaultBufferUsageFlags;
@@ -187,6 +294,11 @@ namespace Ryujinx.Graphics.Vulkan
gd.Api.CreateBuffer(_device, in bufferCreateInfo, null, out var buffer).ThrowOnError();
gd.Api.GetBufferMemoryRequirements(_device, buffer, out var requirements);
+ if (sparseCompatible)
+ {
+ requirements.Alignment = Math.Max(requirements.Alignment, Constants.SparseBufferAlignment);
+ }
+
MemoryAllocation allocation;
do
@@ -227,6 +339,7 @@ namespace Ryujinx.Graphics.Vulkan
VulkanRenderer gd,
int size,
bool forConditionalRendering = false,
+ bool sparseCompatible = false,
BufferAllocationType baseType = BufferAllocationType.HostMapped,
BufferHandle storageHint = default)
{
@@ -255,7 +368,7 @@ namespace Ryujinx.Graphics.Vulkan
}
(VkBuffer buffer, MemoryAllocation allocation, BufferAllocationType resultType) =
- CreateBacking(gd, size, type, forConditionalRendering);
+ CreateBacking(gd, size, type, forConditionalRendering, sparseCompatible);
if (buffer.Handle != 0)
{
diff --git a/src/Ryujinx.Graphics.Vulkan/Constants.cs b/src/Ryujinx.Graphics.Vulkan/Constants.cs
index 1bf8c580..cd612211 100644
--- a/src/Ryujinx.Graphics.Vulkan/Constants.cs
+++ b/src/Ryujinx.Graphics.Vulkan/Constants.cs
@@ -16,5 +16,7 @@ namespace Ryujinx.Graphics.Vulkan
public const int MaxStorageBufferBindings = MaxStorageBuffersPerStage * MaxShaderStages;
public const int MaxTextureBindings = MaxTexturesPerStage * MaxShaderStages;
public const int MaxImageBindings = MaxImagesPerStage * MaxShaderStages;
+
+ public const ulong SparseBufferAlignment = 0x10000;
}
}
diff --git a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs
index b7787601..e1002705 100644
--- a/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs
+++ b/src/Ryujinx.Graphics.Vulkan/EnumConversion.cs
@@ -424,12 +424,12 @@ namespace Ryujinx.Graphics.Vulkan
public static BufferAllocationType Convert(this BufferAccess access)
{
- return access switch
+ if (access.HasFlag(BufferAccess.FlushPersistent) || access.HasFlag(BufferAccess.Stream))
{
- BufferAccess.FlushPersistent => BufferAllocationType.HostMapped,
- BufferAccess.Stream => BufferAllocationType.HostMapped,
- _ => BufferAllocationType.Auto,
- };
+ return BufferAllocationType.HostMapped;
+ }
+
+ return BufferAllocationType.Auto;
}
private static T2 LogInvalidAndReturn<T1, T2>(T1 value, string name, T2 defaultValue = default)
diff --git a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs
index 7240bcad..893ecf1a 100644
--- a/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs
+++ b/src/Ryujinx.Graphics.Vulkan/VulkanRenderer.cs
@@ -392,6 +392,8 @@ namespace Ryujinx.Graphics.Vulkan
LoadFeatures(maxQueueCount, queueFamilyIndex);
+ QueueFamilyIndex = queueFamilyIndex;
+
_window = new Window(this, _surface, _physicalDevice.PhysicalDevice, _device);
_initialized = true;
@@ -399,12 +401,12 @@ namespace Ryujinx.Graphics.Vulkan
public BufferHandle CreateBuffer(int size, BufferAccess access)
{
- return BufferManager.CreateWithHandle(this, size, access.Convert(), default, access == BufferAccess.Stream);
+ return BufferManager.CreateWithHandle(this, size, access.HasFlag(BufferAccess.SparseCompatible), access.Convert(), default, access == BufferAccess.Stream);
}
- public BufferHandle CreateBuffer(int size, BufferHandle storageHint)
+ public BufferHandle CreateBuffer(int size, BufferAccess access, BufferHandle storageHint)
{
- return BufferManager.CreateWithHandle(this, size, BufferAllocationType.Auto, storageHint);
+ return BufferManager.CreateWithHandle(this, size, access.HasFlag(BufferAccess.SparseCompatible), access.Convert(), storageHint);
}
public BufferHandle CreateBuffer(nint pointer, int size)
@@ -412,6 +414,11 @@ namespace Ryujinx.Graphics.Vulkan
return BufferManager.CreateHostImported(this, pointer, size);
}
+ public BufferHandle CreateBufferSparse(ReadOnlySpan<BufferRange> storageBuffers)
+ {
+ return BufferManager.CreateSparse(this, storageBuffers);
+ }
+
public IProgram CreateProgram(ShaderSource[] sources, ShaderInfo info)
{
bool isCompute = sources.Length == 1 && sources[0].Stage == ShaderStage.Compute;
@@ -571,6 +578,7 @@ namespace Ryujinx.Graphics.Vulkan
Api.GetPhysicalDeviceFeatures2(_physicalDevice.PhysicalDevice, &features2);
var limits = _physicalDevice.PhysicalDeviceProperties.Limits;
+ var mainQueueProperties = _physicalDevice.QueueFamilyProperties[QueueFamilyIndex];
return new Capabilities(
api: TargetApi.Vulkan,
@@ -590,6 +598,7 @@ namespace Ryujinx.Graphics.Vulkan
supportsR4G4B4A4Format: supportsR4G4B4A4Format,
supportsSnormBufferTextureFormat: true,
supports5BitComponentFormat: supports5BitComponentFormat,
+ supportsSparseBuffer: features2.Features.SparseBinding && mainQueueProperties.QueueFlags.HasFlag(QueueFlags.SparseBindingBit),
supportsBlendEquationAdvanced: Capabilities.SupportsBlendEquationAdvanced,
supportsFragmentShaderInterlock: Capabilities.SupportsFragmentShaderInterlock,
supportsFragmentShaderOrderingIntel: false,
diff --git a/src/Ryujinx.Memory/Range/MultiRange.cs b/src/Ryujinx.Memory/Range/MultiRange.cs
index 798bc911..093e2190 100644
--- a/src/Ryujinx.Memory/Range/MultiRange.cs
+++ b/src/Ryujinx.Memory/Range/MultiRange.cs
@@ -15,6 +15,11 @@ namespace Ryujinx.Memory.Range
private bool HasSingleRange => _ranges == null;
+ /// <summary>
+ /// Indicates that the range is fully unmapped.
+ /// </summary>
+ public bool IsUnmapped => HasSingleRange && _singleRange.Address == InvalidAddress;
+
/// <summary>
/// Total of physical sub-ranges on the virtual memory region.
/// </summary>
@@ -38,8 +43,18 @@ namespace Ryujinx.Memory.Range
/// <exception cref="ArgumentNullException"><paramref name="ranges"/> is null</exception>
public MultiRange(MemoryRange[] ranges)
{
- _singleRange = MemoryRange.Empty;
- _ranges = ranges ?? throw new ArgumentNullException(nameof(ranges));
+ ArgumentNullException.ThrowIfNull(ranges);
+
+ if (ranges.Length == 1)
+ {
+ _singleRange = ranges[0];
+ _ranges = null;
+ }
+ else
+ {
+ _singleRange = MemoryRange.Empty;
+ _ranges = ranges;
+ }
}
/// <summary>
@@ -91,7 +106,7 @@ namespace Ryujinx.Memory.Range
offset -= range.Size;
}
- return new MultiRange(ranges.ToArray());
+ return ranges.Count == 1 ? new MultiRange(ranges[0].Address, ranges[0].Size) : new MultiRange(ranges.ToArray());
}
}
--
cgit v1.2.3-70-g09d2