using Ryujinx.Common.Memory;
using Ryujinx.Cpu;
using Ryujinx.Graphics.Device;
using Ryujinx.Graphics.Gpu.Image;
using Ryujinx.Graphics.Gpu.Shader;
using Ryujinx.Memory;
using Ryujinx.Memory.Range;
using Ryujinx.Memory.Tracking;
using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
namespace Ryujinx.Graphics.Gpu.Memory
{
///
/// Represents physical memory, accessible from the GPU.
/// This is actually working CPU virtual addresses, of memory mapped on the application process.
///
class PhysicalMemory : IDisposable
{
private readonly GpuContext _context;
private readonly IVirtualMemoryManagerTracked _cpuMemory;
private int _referenceCount;
///
/// In-memory shader cache.
///
public ShaderCache ShaderCache { get; }
///
/// GPU buffer manager.
///
public BufferCache BufferCache { get; }
///
/// GPU texture manager.
///
public TextureCache TextureCache { get; }
///
/// Creates a new instance of the physical memory.
///
/// GPU context that the physical memory belongs to
/// CPU memory manager of the application process
public PhysicalMemory(GpuContext context, IVirtualMemoryManagerTracked cpuMemory)
{
_context = context;
_cpuMemory = cpuMemory;
ShaderCache = new ShaderCache(context);
BufferCache = new BufferCache(context, this);
TextureCache = new TextureCache(context, this);
if (cpuMemory is IRefCounted rc)
{
rc.IncrementReferenceCount();
}
_referenceCount = 1;
}
///
/// Increments the memory reference count.
///
public void IncrementReferenceCount()
{
Interlocked.Increment(ref _referenceCount);
}
///
/// Decrements the memory reference count.
///
public void DecrementReferenceCount()
{
if (Interlocked.Decrement(ref _referenceCount) == 0 && _cpuMemory is IRefCounted rc)
{
rc.DecrementReferenceCount();
}
}
///
/// Creates a new device memory manager.
///
/// The memory manager
public DeviceMemoryManager CreateDeviceMemoryManager()
{
return new DeviceMemoryManager(_cpuMemory);
}
///
/// Gets a host pointer for a given range of application memory.
/// If the memory region is not a single contiguous block, this method returns 0.
///
///
/// Getting a host pointer is unsafe. It should be considered invalid immediately if the GPU memory is unmapped.
///
/// Ranges of physical memory where the target data is located
/// Pointer to the range of memory
public nint GetHostPointer(MultiRange range)
{
if (range.Count == 1)
{
var singleRange = range.GetSubRange(0);
if (singleRange.Address != MemoryManager.PteUnmapped)
{
var regions = _cpuMemory.GetHostRegions(singleRange.Address, singleRange.Size);
if (regions != null && regions.Count() == 1)
{
return (nint)regions.First().Address;
}
}
}
return 0;
}
///
/// Gets a span of data from the application process.
///
/// Start address of the range
/// Size in bytes to be range
/// True if read tracking is triggered on the span
/// A read only span of the data at the specified memory location
public ReadOnlySpan GetSpan(ulong address, int size, bool tracked = false)
{
return _cpuMemory.GetSpan(address, size, tracked);
}
///
/// Gets a span of data from the application process.
///
/// Ranges of physical memory where the data is located
/// True if read tracking is triggered on the span
/// A read only span of the data at the specified memory location
public ReadOnlySpan GetSpan(MultiRange range, bool tracked = false)
{
if (range.Count == 1)
{
var singleRange = range.GetSubRange(0);
if (singleRange.Address != MemoryManager.PteUnmapped)
{
return _cpuMemory.GetSpan(singleRange.Address, (int)singleRange.Size, tracked);
}
}
Span data = new byte[range.GetSize()];
int offset = 0;
for (int i = 0; i < range.Count; i++)
{
var currentRange = range.GetSubRange(i);
int size = (int)currentRange.Size;
if (currentRange.Address != MemoryManager.PteUnmapped)
{
_cpuMemory.GetSpan(currentRange.Address, size, tracked).CopyTo(data.Slice(offset, size));
}
offset += size;
}
return data;
}
///
/// Gets a writable region from the application process.
///
/// Start address of the range
/// Size in bytes to be range
/// True if write tracking is triggered on the span
/// A writable region with the data at the specified memory location
public WritableRegion GetWritableRegion(ulong address, int size, bool tracked = false)
{
return _cpuMemory.GetWritableRegion(address, size, tracked);
}
///
/// Gets a writable region from GPU mapped memory.
///
/// Range
/// True if write tracking is triggered on the span
/// A writable region with the data at the specified memory location
public WritableRegion GetWritableRegion(MultiRange range, bool tracked = false)
{
if (range.Count == 1)
{
MemoryRange subrange = range.GetSubRange(0);
return GetWritableRegion(subrange.Address, (int)subrange.Size, tracked);
}
else
{
MemoryOwner memoryOwner = MemoryOwner.Rent(checked((int)range.GetSize()));
Span memorySpan = memoryOwner.Span;
int offset = 0;
for (int i = 0; i < range.Count; i++)
{
var currentRange = range.GetSubRange(i);
int size = (int)currentRange.Size;
if (currentRange.Address != MemoryManager.PteUnmapped)
{
GetSpan(currentRange.Address, size).CopyTo(memorySpan.Slice(offset, size));
}
offset += size;
}
return new WritableRegion(new MultiRangeWritableBlock(range, this), 0, memoryOwner, tracked);
}
}
///
/// Reads data from the application process.
///
/// Type of the structure
/// Address to read from
/// The data at the specified memory location
public T Read(ulong address) where T : unmanaged
{
return _cpuMemory.Read(address);
}
///
/// Reads data from the application process, with write tracking.
///
/// Type of the structure
/// Address to read from
/// The data at the specified memory location
public T ReadTracked(ulong address) where T : unmanaged
{
return _cpuMemory.ReadTracked(address);
}
///
/// Writes data to the application process, triggering a precise memory tracking event.
///
/// Address to write into
/// Data to be written
public void WriteTrackedResource(ulong address, ReadOnlySpan data)
{
_cpuMemory.SignalMemoryTracking(address, (ulong)data.Length, true, precise: true);
_cpuMemory.WriteUntracked(address, data);
}
///
/// Writes data to the application process, triggering a precise memory tracking event.
///
/// Address to write into
/// Data to be written
/// Kind of the resource being written, which will not be signalled as CPU modified
public void WriteTrackedResource(ulong address, ReadOnlySpan data, ResourceKind kind)
{
_cpuMemory.SignalMemoryTracking(address, (ulong)data.Length, true, precise: true, exemptId: (int)kind);
_cpuMemory.WriteUntracked(address, data);
}
///
/// Writes data to the application process.
///
/// Address to write into
/// Data to be written
public void Write(ulong address, ReadOnlySpan data)
{
_cpuMemory.Write(address, data);
}
///
/// Writes data to the application process.
///
/// Ranges of physical memory where the data is located
/// Data to be written
public void Write(MultiRange range, ReadOnlySpan data)
{
WriteImpl(range, data, _cpuMemory.Write);
}
///
/// Writes data to the application process, without any tracking.
///
/// Address to write into
/// Data to be written
public void WriteUntracked(ulong address, ReadOnlySpan data)
{
_cpuMemory.WriteUntracked(address, data);
}
///
/// Writes data to the application process, without any tracking.
///
/// Ranges of physical memory where the data is located
/// Data to be written
public void WriteUntracked(MultiRange range, ReadOnlySpan data)
{
WriteImpl(range, data, _cpuMemory.WriteUntracked);
}
///
/// Writes data to the application process, returning false if the data was not changed.
/// This triggers read memory tracking, as a redundancy check would be useless if the data is not up to date.
///
/// The memory manager can return that memory has changed when it hasn't to avoid expensive data copies.
/// Address to write into
/// Data to be written
/// True if the data was changed, false otherwise
public bool WriteWithRedundancyCheck(ulong address, ReadOnlySpan data)
{
return _cpuMemory.WriteWithRedundancyCheck(address, data);
}
private delegate void WriteCallback(ulong address, ReadOnlySpan data);
///
/// Writes data to the application process, using the supplied callback method.
///
/// Ranges of physical memory where the data is located
/// Data to be written
/// Callback method that will perform the write
private static void WriteImpl(MultiRange range, ReadOnlySpan data, WriteCallback writeCallback)
{
if (range.Count == 1)
{
var singleRange = range.GetSubRange(0);
if (singleRange.Address != MemoryManager.PteUnmapped)
{
writeCallback(singleRange.Address, data);
}
}
else
{
int offset = 0;
for (int i = 0; i < range.Count; i++)
{
var currentRange = range.GetSubRange(i);
int size = (int)currentRange.Size;
if (currentRange.Address != MemoryManager.PteUnmapped)
{
writeCallback(currentRange.Address, data.Slice(offset, size));
}
offset += size;
}
}
}
///
/// Fills the specified memory region with a 32-bit integer value.
///
/// CPU virtual address of the region
/// Size of the region
/// Value to fill the region with
/// Kind of the resource being filled, which will not be signalled as CPU modified
public void FillTrackedResource(ulong address, ulong size, uint value, ResourceKind kind)
{
_cpuMemory.SignalMemoryTracking(address, size, write: true, precise: true, (int)kind);
using WritableRegion region = _cpuMemory.GetWritableRegion(address, (int)size);
MemoryMarshal.Cast(region.Memory.Span).Fill(value);
}
///
/// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with.
///
/// CPU virtual address of the region
/// Size of the region
/// Kind of the resource being tracked
/// Region flags
/// The memory tracking handle
public RegionHandle BeginTracking(ulong address, ulong size, ResourceKind kind, RegionFlags flags = RegionFlags.None)
{
return _cpuMemory.BeginTracking(address, size, (int)kind, flags);
}
///
/// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with.
///
/// Ranges of physical memory where the data is located
/// Kind of the resource being tracked
/// The memory tracking handle
public GpuRegionHandle BeginTracking(MultiRange range, ResourceKind kind)
{
var cpuRegionHandles = new RegionHandle[range.Count];
int count = 0;
for (int i = 0; i < range.Count; i++)
{
var currentRange = range.GetSubRange(i);
if (currentRange.Address != MemoryManager.PteUnmapped)
{
cpuRegionHandles[count++] = _cpuMemory.BeginTracking(currentRange.Address, currentRange.Size, (int)kind);
}
}
if (count != range.Count)
{
Array.Resize(ref cpuRegionHandles, count);
}
return new GpuRegionHandle(cpuRegionHandles);
}
///
/// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
///
/// CPU virtual address of the region
/// Size of the region
/// Kind of the resource being tracked
/// Region flags
/// Handles to inherit state from or reuse
/// Desired granularity of write tracking
/// The memory tracking handle
public MultiRegionHandle BeginGranularTracking(
ulong address,
ulong size,
ResourceKind kind,
RegionFlags flags = RegionFlags.None,
IEnumerable handles = null,
ulong granularity = 4096)
{
return _cpuMemory.BeginGranularTracking(address, size, handles, granularity, (int)kind, flags);
}
///
/// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
///
/// CPU virtual address of the region
/// Size of the region
/// Kind of the resource being tracked
/// Desired granularity of write tracking
/// The memory tracking handle
public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ResourceKind kind, ulong granularity = 4096)
{
return _cpuMemory.BeginSmartGranularTracking(address, size, granularity, (int)kind);
}
///
/// Checks if a given memory page is mapped.
///
/// CPU virtual address of the page
/// True if mapped, false otherwise
public bool IsMapped(ulong address)
{
return _cpuMemory.IsMapped(address);
}
///
/// Release our reference to the CPU memory manager.
///
public void Dispose()
{
_context.DeferredActions.Enqueue(Destroy);
}
///
/// Performs disposal of the host GPU caches with resources mapped on this physical memory.
/// This must only be called from the render thread.
///
private void Destroy()
{
ShaderCache.Dispose();
BufferCache.Dispose();
TextureCache.Dispose();
DecrementReferenceCount();
}
}
}