using Ryujinx.Common;
using Ryujinx.Graphics.GAL;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Shader;
using Ryujinx.Graphics.Shader;
using Ryujinx.Memory.Range;
using System;
using System.Collections.Generic;
using System.Runtime.CompilerServices;
namespace Ryujinx.Graphics.Gpu.Memory
{
///
/// Buffer manager.
///
class BufferManager
{
private readonly GpuContext _context;
private readonly GpuChannel _channel;
private int _unalignedStorageBuffers;
public bool HasUnalignedStorageBuffers => _unalignedStorageBuffers > 0;
public bool HasTransformFeedbackOutputs { get; set; }
private IndexBuffer _indexBuffer;
private readonly VertexBuffer[] _vertexBuffers;
private readonly BufferBounds[] _transformFeedbackBuffers;
private readonly List _bufferTextures;
private readonly BufferAssignment[] _ranges;
///
/// Holds shader stage buffer state and binding information.
///
private class BuffersPerStage
{
///
/// Shader buffer binding information.
///
public BufferDescriptor[] Bindings { get; private set; }
///
/// Buffer regions.
///
public BufferBounds[] Buffers { get; }
///
/// Flag indicating if this binding is unaligned.
///
public bool[] Unaligned { get; }
///
/// Total amount of buffers used on the shader.
///
public int Count { get; private set; }
///
/// Creates a new instance of the shader stage buffer information.
///
/// Maximum amount of buffers that the shader stage can use
public BuffersPerStage(int count)
{
Bindings = new BufferDescriptor[count];
Buffers = new BufferBounds[count];
Unaligned = new bool[count];
Buffers.AsSpan().Fill(new BufferBounds(new MultiRange(MemoryManager.PteUnmapped, 0UL)));
}
///
/// Sets the region of a buffer at a given slot.
///
/// Buffer slot
/// Physical memory regions where the buffer is mapped
/// Buffer usage flags
public void SetBounds(int index, MultiRange range, BufferUsageFlags flags = BufferUsageFlags.None)
{
Buffers[index] = new BufferBounds(range, flags);
}
///
/// Sets shader buffer binding information.
///
/// Buffer binding information
public void SetBindings(BufferDescriptor[] descriptors)
{
if (descriptors == null)
{
Count = 0;
return;
}
if ((Count = descriptors.Length) != 0)
{
Bindings = descriptors;
}
}
}
private readonly BuffersPerStage _cpStorageBuffers;
private readonly BuffersPerStage _cpUniformBuffers;
private readonly BuffersPerStage[] _gpStorageBuffers;
private readonly BuffersPerStage[] _gpUniformBuffers;
private bool _gpStorageBuffersDirty;
private bool _gpUniformBuffersDirty;
private bool _indexBufferDirty;
private bool _vertexBuffersDirty;
private uint _vertexBuffersEnableMask;
private bool _transformFeedbackBuffersDirty;
private bool _rebind;
///
/// Creates a new instance of the buffer manager.
///
/// GPU context that the buffer manager belongs to
/// GPU channel that the buffer manager belongs to
public BufferManager(GpuContext context, GpuChannel channel)
{
_context = context;
_channel = channel;
_indexBuffer.Range = new MultiRange(MemoryManager.PteUnmapped, 0UL);
_vertexBuffers = new VertexBuffer[Constants.TotalVertexBuffers];
_transformFeedbackBuffers = new BufferBounds[Constants.TotalTransformFeedbackBuffers];
_cpStorageBuffers = new BuffersPerStage(Constants.TotalCpStorageBuffers);
_cpUniformBuffers = new BuffersPerStage(Constants.TotalCpUniformBuffers);
_gpStorageBuffers = new BuffersPerStage[Constants.ShaderStages];
_gpUniformBuffers = new BuffersPerStage[Constants.ShaderStages];
for (int index = 0; index < Constants.ShaderStages; index++)
{
_gpStorageBuffers[index] = new BuffersPerStage(Constants.TotalGpStorageBuffers);
_gpUniformBuffers[index] = new BuffersPerStage(Constants.TotalGpUniformBuffers);
}
_bufferTextures = new List();
_ranges = new BufferAssignment[Constants.TotalGpUniformBuffers * Constants.ShaderStages];
}
///
/// Sets the memory range with the index buffer data, to be used for subsequent draw calls.
///
/// Start GPU virtual address of the index buffer
/// Size, in bytes, of the index buffer
/// Type of each index buffer element
public void SetIndexBuffer(ulong gpuVa, ulong size, IndexType type)
{
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_indexBuffer.Range = range;
_indexBuffer.Type = type;
_indexBufferDirty = true;
}
///
/// Sets a new index buffer that overrides the one set on the call to .
///
/// Buffer to be used as index buffer
/// Type of each index buffer element
public void SetIndexBuffer(BufferRange buffer, IndexType type)
{
_context.Renderer.Pipeline.SetIndexBuffer(buffer, type);
_indexBufferDirty = true;
}
///
/// Sets the memory range with vertex buffer data, to be used for subsequent draw calls.
///
/// Index of the vertex buffer (up to 16)
/// GPU virtual address of the buffer
/// Size in bytes of the buffer
/// Stride of the buffer, defined as the number of bytes of each vertex
/// Vertex divisor of the buffer, for instanced draws
public void SetVertexBuffer(int index, ulong gpuVa, ulong size, int stride, int divisor)
{
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_vertexBuffers[index].Range = range;
_vertexBuffers[index].Stride = stride;
_vertexBuffers[index].Divisor = divisor;
_vertexBuffersDirty = true;
if (!range.IsUnmapped)
{
_vertexBuffersEnableMask |= 1u << index;
}
else
{
_vertexBuffersEnableMask &= ~(1u << index);
}
}
///
/// Sets a transform feedback buffer on the graphics pipeline.
/// The output from the vertex transformation stages are written into the feedback buffer.
///
/// Index of the transform feedback buffer
/// Start GPU virtual address of the buffer
/// Size in bytes of the transform feedback buffer
public void SetTransformFeedbackBuffer(int index, ulong gpuVa, ulong size)
{
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
_transformFeedbackBuffers[index] = new BufferBounds(range);
_transformFeedbackBuffersDirty = true;
}
///
/// Records the alignment of a storage buffer.
/// Unaligned storage buffers disable some optimizations on the shader.
///
/// The binding list to modify
/// Index of the storage buffer
/// Start GPU virtual address of the buffer
private void RecordStorageAlignment(BuffersPerStage buffers, int index, ulong gpuVa)
{
bool unaligned = (gpuVa & ((ulong)_context.Capabilities.StorageBufferOffsetAlignment - 1)) != 0;
if (unaligned || HasUnalignedStorageBuffers)
{
// Check if the alignment changed for this binding.
ref bool currentUnaligned = ref buffers.Unaligned[index];
if (currentUnaligned != unaligned)
{
currentUnaligned = unaligned;
_unalignedStorageBuffers += unaligned ? 1 : -1;
}
}
}
///
/// Sets a storage buffer on the compute pipeline.
/// Storage buffers can be read and written to on shaders.
///
/// Index of the storage buffer
/// Start GPU virtual address of the buffer
/// Size in bytes of the storage buffer
/// Buffer usage flags
public void SetComputeStorageBuffer(int index, ulong gpuVa, ulong size, BufferUsageFlags flags)
{
size += gpuVa & ((ulong)_context.Capabilities.StorageBufferOffsetAlignment - 1);
RecordStorageAlignment(_cpStorageBuffers, index, gpuVa);
gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
_cpStorageBuffers.SetBounds(index, range, flags);
}
///
/// Sets a storage buffer on the graphics pipeline.
/// Storage buffers can be read and written to on shaders.
///
/// Index of the shader stage
/// Index of the storage buffer
/// Start GPU virtual address of the buffer
/// Size in bytes of the storage buffer
/// Buffer usage flags
public void SetGraphicsStorageBuffer(int stage, int index, ulong gpuVa, ulong size, BufferUsageFlags flags)
{
size += gpuVa & ((ulong)_context.Capabilities.StorageBufferOffsetAlignment - 1);
BuffersPerStage buffers = _gpStorageBuffers[stage];
RecordStorageAlignment(buffers, index, gpuVa);
gpuVa = BitUtils.AlignDown(gpuVa, (ulong)_context.Capabilities.StorageBufferOffsetAlignment);
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateMultiBuffers(_channel.MemoryManager, gpuVa, size);
if (!buffers.Buffers[index].Range.Equals(range))
{
_gpStorageBuffersDirty = true;
}
buffers.SetBounds(index, range, flags);
}
///
/// Sets a uniform buffer on the compute pipeline.
/// Uniform buffers are read-only from shaders, and have a small capacity.
///
/// Index of the uniform buffer
/// Start GPU virtual address of the buffer
/// Size in bytes of the storage buffer
public void SetComputeUniformBuffer(int index, ulong gpuVa, ulong size)
{
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_cpUniformBuffers.SetBounds(index, range);
}
///
/// Sets a uniform buffer on the graphics pipeline.
/// Uniform buffers are read-only from shaders, and have a small capacity.
///
/// Index of the shader stage
/// Index of the uniform buffer
/// Start GPU virtual address of the buffer
/// Size in bytes of the storage buffer
public void SetGraphicsUniformBuffer(int stage, int index, ulong gpuVa, ulong size)
{
MultiRange range = _channel.MemoryManager.Physical.BufferCache.TranslateAndCreateBuffer(_channel.MemoryManager, gpuVa, size);
_gpUniformBuffers[stage].SetBounds(index, range);
_gpUniformBuffersDirty = true;
}
///
/// Sets the number of vertices per instance on a instanced draw. Used for transform feedback emulation.
///
/// Vertex count per instance
public void SetInstancedDrawVertexCount(int vertexCount)
{
if (!_context.Capabilities.SupportsTransformFeedback && HasTransformFeedbackOutputs)
{
_context.SupportBufferUpdater.SetTfeVertexCount(vertexCount);
_context.SupportBufferUpdater.Commit();
}
}
///
/// Forces transform feedback and storage buffers to be updated on the next draw.
///
public void ForceTransformFeedbackAndStorageBuffersDirty()
{
_transformFeedbackBuffersDirty = true;
_gpStorageBuffersDirty = true;
}
///
/// Sets the binding points for the storage buffers bound on the compute pipeline.
///
/// Bindings for the active shader
public void SetComputeBufferBindings(CachedShaderBindings bindings)
{
_cpStorageBuffers.SetBindings(bindings.StorageBufferBindings[0]);
_cpUniformBuffers.SetBindings(bindings.ConstantBufferBindings[0]);
}
///
/// Sets the binding points for the storage buffers bound on the graphics pipeline.
///
/// Bindings for the active shader
public void SetGraphicsBufferBindings(CachedShaderBindings bindings)
{
for (int i = 0; i < Constants.ShaderStages; i++)
{
_gpStorageBuffers[i].SetBindings(bindings.StorageBufferBindings[i]);
_gpUniformBuffers[i].SetBindings(bindings.ConstantBufferBindings[i]);
}
_gpStorageBuffersDirty = true;
_gpUniformBuffersDirty = true;
}
///
/// Gets a bit mask indicating which compute uniform buffers are currently bound.
///
/// Mask where each bit set indicates a bound constant buffer
public uint GetComputeUniformBufferUseMask()
{
uint mask = 0;
for (int i = 0; i < _cpUniformBuffers.Buffers.Length; i++)
{
if (!_cpUniformBuffers.Buffers[i].IsUnmapped)
{
mask |= 1u << i;
}
}
return mask;
}
///
/// Gets a bit mask indicating which graphics uniform buffers are currently bound.
///
/// Index of the shader stage
/// Mask where each bit set indicates a bound constant buffer
public uint GetGraphicsUniformBufferUseMask(int stage)
{
uint mask = 0;
for (int i = 0; i < _gpUniformBuffers[stage].Buffers.Length; i++)
{
if (!_gpUniformBuffers[stage].Buffers[i].IsUnmapped)
{
mask |= 1u << i;
}
}
return mask;
}
///
/// Gets the address of the compute uniform buffer currently bound at the given index.
///
/// Index of the uniform buffer binding
/// The uniform buffer address, or an undefined value if the buffer is not currently bound
public ulong GetComputeUniformBufferAddress(int index)
{
return _cpUniformBuffers.Buffers[index].Range.GetSubRange(0).Address;
}
///
/// Gets the address of the graphics uniform buffer currently bound at the given index.
///
/// Index of the shader stage
/// Index of the uniform buffer binding
/// The uniform buffer address, or an undefined value if the buffer is not currently bound
public ulong GetGraphicsUniformBufferAddress(int stage, int index)
{
return _gpUniformBuffers[stage].Buffers[index].Range.GetSubRange(0).Address;
}
///
/// Gets the bounds of the uniform buffer currently bound at the given index.
///
/// Indicates whenever the uniform is requested by the 3D or compute engine
/// Index of the shader stage, if the uniform is for the 3D engine
/// Index of the uniform buffer binding
/// The uniform buffer bounds, or an undefined value if the buffer is not currently bound
public ref BufferBounds GetUniformBufferBounds(bool isCompute, int stage, int index)
{
if (isCompute)
{
return ref _cpUniformBuffers.Buffers[index];
}
else
{
return ref _gpUniformBuffers[stage].Buffers[index];
}
}
///
/// Ensures that the compute engine bindings are visible to the host GPU.
/// Note: this actually performs the binding using the host graphics API.
///
public void CommitComputeBindings()
{
var bufferCache = _channel.MemoryManager.Physical.BufferCache;
BindBuffers(bufferCache, _cpStorageBuffers, isStorage: true);
BindBuffers(bufferCache, _cpUniformBuffers, isStorage: false);
CommitBufferTextureBindings();
// Force rebind after doing compute work.
Rebind();
_context.SupportBufferUpdater.Commit();
}
///
/// Commit any queued buffer texture bindings.
///
private void CommitBufferTextureBindings()
{
if (_bufferTextures.Count > 0)
{
foreach (var binding in _bufferTextures)
{
var isStore = binding.BindingInfo.Flags.HasFlag(TextureUsageFlags.ImageStore);
var range = _channel.MemoryManager.Physical.BufferCache.GetBufferRange(binding.Range, isStore);
binding.Texture.SetStorage(range);
// The texture must be rebound to use the new storage if it was updated.
if (binding.IsImage)
{
_context.Renderer.Pipeline.SetImage(binding.Stage, binding.BindingInfo.Binding, binding.Texture, binding.Format);
}
else
{
_context.Renderer.Pipeline.SetTextureAndSampler(binding.Stage, binding.BindingInfo.Binding, binding.Texture, null);
}
}
_bufferTextures.Clear();
}
}
///
/// Ensures that the graphics engine bindings are visible to the host GPU.
/// Note: this actually performs the binding using the host graphics API.
///
/// True if the index buffer is in use
public void CommitGraphicsBindings(bool indexed)
{
var bufferCache = _channel.MemoryManager.Physical.BufferCache;
if (indexed)
{
if (_indexBufferDirty || _rebind)
{
_indexBufferDirty = false;
if (!_indexBuffer.Range.IsUnmapped)
{
BufferRange buffer = bufferCache.GetBufferRange(_indexBuffer.Range);
_context.Renderer.Pipeline.SetIndexBuffer(buffer, _indexBuffer.Type);
}
}
else if (!_indexBuffer.Range.IsUnmapped)
{
bufferCache.SynchronizeBufferRange(_indexBuffer.Range);
}
}
else if (_rebind)
{
_indexBufferDirty = true;
}
uint vbEnableMask = _vertexBuffersEnableMask;
if (_vertexBuffersDirty || _rebind)
{
_vertexBuffersDirty = false;
Span vertexBuffers = stackalloc VertexBufferDescriptor[Constants.TotalVertexBuffers];
for (int index = 0; (vbEnableMask >> index) != 0; index++)
{
VertexBuffer vb = _vertexBuffers[index];
if (vb.Range.IsUnmapped)
{
continue;
}
BufferRange buffer = bufferCache.GetBufferRange(vb.Range);
vertexBuffers[index] = new VertexBufferDescriptor(buffer, vb.Stride, vb.Divisor);
}
_context.Renderer.Pipeline.SetVertexBuffers(vertexBuffers);
}
else
{
for (int index = 0; (vbEnableMask >> index) != 0; index++)
{
VertexBuffer vb = _vertexBuffers[index];
if (vb.Range.IsUnmapped)
{
continue;
}
bufferCache.SynchronizeBufferRange(vb.Range);
}
}
if (_transformFeedbackBuffersDirty || _rebind)
{
_transformFeedbackBuffersDirty = false;
if (_context.Capabilities.SupportsTransformFeedback)
{
Span tfbs = stackalloc BufferRange[Constants.TotalTransformFeedbackBuffers];
for (int index = 0; index < Constants.TotalTransformFeedbackBuffers; index++)
{
BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.IsUnmapped)
{
tfbs[index] = BufferRange.Empty;
continue;
}
tfbs[index] = bufferCache.GetBufferRange(tfb.Range, write: true);
}
_context.Renderer.Pipeline.SetTransformFeedbackBuffers(tfbs);
}
else if (HasTransformFeedbackOutputs)
{
Span buffers = stackalloc BufferAssignment[Constants.TotalTransformFeedbackBuffers];
int alignment = _context.Capabilities.StorageBufferOffsetAlignment;
for (int index = 0; index < Constants.TotalTransformFeedbackBuffers; index++)
{
BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.IsUnmapped)
{
buffers[index] = new BufferAssignment(index, BufferRange.Empty);
}
else
{
MultiRange range = tfb.Range;
ulong address0 = range.GetSubRange(0).Address;
ulong address = BitUtils.AlignDown(address0, (ulong)alignment);
if (range.Count == 1)
{
range = new MultiRange(address, range.GetSubRange(0).Size + (address0 - address));
}
else
{
MemoryRange[] subRanges = new MemoryRange[range.Count];
subRanges[0] = new MemoryRange(address, range.GetSubRange(0).Size + (address0 - address));
for (int i = 1; i < range.Count; i++)
{
subRanges[i] = range.GetSubRange(i);
}
range = new MultiRange(subRanges);
}
int tfeOffset = ((int)address0 & (alignment - 1)) / 4;
_context.SupportBufferUpdater.SetTfeOffset(index, tfeOffset);
buffers[index] = new BufferAssignment(index, bufferCache.GetBufferRange(range, write: true));
}
}
_context.Renderer.Pipeline.SetStorageBuffers(buffers);
}
}
else
{
for (int index = 0; index < Constants.TotalTransformFeedbackBuffers; index++)
{
BufferBounds tfb = _transformFeedbackBuffers[index];
if (tfb.IsUnmapped)
{
continue;
}
bufferCache.SynchronizeBufferRange(tfb.Range);
}
}
if (_gpStorageBuffersDirty || _rebind)
{
_gpStorageBuffersDirty = false;
BindBuffers(bufferCache, _gpStorageBuffers, isStorage: true);
}
else
{
UpdateBuffers(_gpStorageBuffers);
}
if (_gpUniformBuffersDirty || _rebind)
{
_gpUniformBuffersDirty = false;
BindBuffers(bufferCache, _gpUniformBuffers, isStorage: false);
}
else
{
UpdateBuffers(_gpUniformBuffers);
}
CommitBufferTextureBindings();
_rebind = false;
_context.SupportBufferUpdater.Commit();
}
///
/// Bind respective buffer bindings on the host API.
///
/// Buffer cache holding the buffers for the specified ranges
/// Buffer memory ranges to bind
/// True to bind as storage buffer, false to bind as uniform buffer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void BindBuffers(BufferCache bufferCache, BuffersPerStage[] bindings, bool isStorage)
{
int rangesCount = 0;
Span ranges = _ranges;
for (ShaderStage stage = ShaderStage.Vertex; stage <= ShaderStage.Fragment; stage++)
{
ref var buffers = ref bindings[(int)stage - 1];
for (int index = 0; index < buffers.Count; index++)
{
ref var bindingInfo = ref buffers.Bindings[index];
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
if (!bounds.IsUnmapped)
{
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
: bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
}
}
}
if (rangesCount != 0)
{
SetHostBuffers(ranges, rangesCount, isStorage);
}
}
///
/// Bind respective buffer bindings on the host API.
///
/// Buffer cache holding the buffers for the specified ranges
/// Buffer memory ranges to bind
/// True to bind as storage buffer, false to bind as uniform buffer
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void BindBuffers(BufferCache bufferCache, BuffersPerStage buffers, bool isStorage)
{
int rangesCount = 0;
Span ranges = _ranges;
for (int index = 0; index < buffers.Count; index++)
{
ref var bindingInfo = ref buffers.Bindings[index];
BufferBounds bounds = buffers.Buffers[bindingInfo.Slot];
if (!bounds.IsUnmapped)
{
var isWrite = bounds.Flags.HasFlag(BufferUsageFlags.Write);
var range = isStorage
? bufferCache.GetBufferRangeAligned(bounds.Range, isWrite)
: bufferCache.GetBufferRange(bounds.Range);
ranges[rangesCount++] = new BufferAssignment(bindingInfo.Binding, range);
}
}
if (rangesCount != 0)
{
SetHostBuffers(ranges, rangesCount, isStorage);
}
}
///
/// Bind respective buffer bindings on the host API.
///
/// Host buffers to bind, with their offsets and sizes
/// First binding point
/// Number of bindings
/// Indicates if the buffers are storage or uniform buffers
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private void SetHostBuffers(ReadOnlySpan ranges, int count, bool isStorage)
{
if (isStorage)
{
_context.Renderer.Pipeline.SetStorageBuffers(ranges[..count]);
}
else
{
_context.Renderer.Pipeline.SetUniformBuffers(ranges[..count]);
}
}
///
/// Updates data for the already bound buffer bindings.
///
/// Bindings to update
private void UpdateBuffers(BuffersPerStage[] bindings)
{
for (ShaderStage stage = ShaderStage.Vertex; stage <= ShaderStage.Fragment; stage++)
{
ref var buffers = ref bindings[(int)stage - 1];
for (int index = 0; index < buffers.Count; index++)
{
ref var binding = ref buffers.Bindings[index];
BufferBounds bounds = buffers.Buffers[binding.Slot];
if (bounds.IsUnmapped)
{
continue;
}
_channel.MemoryManager.Physical.BufferCache.SynchronizeBufferRange(bounds.Range);
}
}
}
///
/// Sets the buffer storage of a buffer texture. This will be bound when the buffer manager commits bindings.
///
/// Shader stage accessing the texture
/// Buffer texture
/// Physical ranges of memory where the buffer texture data is located
/// Binding info for the buffer texture
/// Format of the buffer texture
/// Whether the binding is for an image or a sampler
public void SetBufferTextureStorage(
ShaderStage stage,
ITexture texture,
MultiRange range,
TextureBindingInfo bindingInfo,
Format format,
bool isImage)
{
_channel.MemoryManager.Physical.BufferCache.CreateBuffer(range);
_bufferTextures.Add(new BufferTextureBinding(stage, texture, range, bindingInfo, format, isImage));
}
///
/// Force all bound textures and images to be rebound the next time CommitBindings is called.
///
public void Rebind()
{
_rebind = true;
}
}
}