using Ryujinx.Common.Memory; using Ryujinx.Cpu; using Ryujinx.Graphics.Device; using Ryujinx.Graphics.Gpu.Image; using Ryujinx.Graphics.Gpu.Shader; using Ryujinx.Memory; using Ryujinx.Memory.Range; using Ryujinx.Memory.Tracking; using System; using System.Buffers; using System.Collections.Generic; using System.Linq; using System.Runtime.InteropServices; using System.Threading; namespace Ryujinx.Graphics.Gpu.Memory { /// /// Represents physical memory, accessible from the GPU. /// This is actually working CPU virtual addresses, of memory mapped on the application process. /// class PhysicalMemory : IDisposable { private readonly GpuContext _context; private readonly IVirtualMemoryManagerTracked _cpuMemory; private int _referenceCount; /// /// In-memory shader cache. /// public ShaderCache ShaderCache { get; } /// /// GPU buffer manager. /// public BufferCache BufferCache { get; } /// /// GPU texture manager. /// public TextureCache TextureCache { get; } /// /// Creates a new instance of the physical memory. /// /// GPU context that the physical memory belongs to /// CPU memory manager of the application process public PhysicalMemory(GpuContext context, IVirtualMemoryManagerTracked cpuMemory) { _context = context; _cpuMemory = cpuMemory; ShaderCache = new ShaderCache(context); BufferCache = new BufferCache(context, this); TextureCache = new TextureCache(context, this); if (cpuMemory is IRefCounted rc) { rc.IncrementReferenceCount(); } _referenceCount = 1; } /// /// Increments the memory reference count. /// public void IncrementReferenceCount() { Interlocked.Increment(ref _referenceCount); } /// /// Decrements the memory reference count. /// public void DecrementReferenceCount() { if (Interlocked.Decrement(ref _referenceCount) == 0 && _cpuMemory is IRefCounted rc) { rc.DecrementReferenceCount(); } } /// /// Creates a new device memory manager. /// /// The memory manager public DeviceMemoryManager CreateDeviceMemoryManager() { return new DeviceMemoryManager(_cpuMemory); } /// /// Gets a host pointer for a given range of application memory. /// If the memory region is not a single contiguous block, this method returns 0. /// /// /// Getting a host pointer is unsafe. It should be considered invalid immediately if the GPU memory is unmapped. /// /// Ranges of physical memory where the target data is located /// Pointer to the range of memory public nint GetHostPointer(MultiRange range) { if (range.Count == 1) { var singleRange = range.GetSubRange(0); if (singleRange.Address != MemoryManager.PteUnmapped) { var regions = _cpuMemory.GetHostRegions(singleRange.Address, singleRange.Size); if (regions != null && regions.Count() == 1) { return (nint)regions.First().Address; } } } return 0; } /// /// Gets a span of data from the application process. /// /// Start address of the range /// Size in bytes to be range /// True if read tracking is triggered on the span /// A read only span of the data at the specified memory location public ReadOnlySpan GetSpan(ulong address, int size, bool tracked = false) { return _cpuMemory.GetSpan(address, size, tracked); } /// /// Gets a span of data from the application process. /// /// Ranges of physical memory where the data is located /// True if read tracking is triggered on the span /// A read only span of the data at the specified memory location public ReadOnlySpan GetSpan(MultiRange range, bool tracked = false) { if (range.Count == 1) { var singleRange = range.GetSubRange(0); if (singleRange.Address != MemoryManager.PteUnmapped) { return _cpuMemory.GetSpan(singleRange.Address, (int)singleRange.Size, tracked); } } Span data = new byte[range.GetSize()]; int offset = 0; for (int i = 0; i < range.Count; i++) { var currentRange = range.GetSubRange(i); int size = (int)currentRange.Size; if (currentRange.Address != MemoryManager.PteUnmapped) { _cpuMemory.GetSpan(currentRange.Address, size, tracked).CopyTo(data.Slice(offset, size)); } offset += size; } return data; } /// /// Gets a writable region from the application process. /// /// Start address of the range /// Size in bytes to be range /// True if write tracking is triggered on the span /// A writable region with the data at the specified memory location public WritableRegion GetWritableRegion(ulong address, int size, bool tracked = false) { return _cpuMemory.GetWritableRegion(address, size, tracked); } /// /// Gets a writable region from GPU mapped memory. /// /// Range /// True if write tracking is triggered on the span /// A writable region with the data at the specified memory location public WritableRegion GetWritableRegion(MultiRange range, bool tracked = false) { if (range.Count == 1) { MemoryRange subrange = range.GetSubRange(0); return GetWritableRegion(subrange.Address, (int)subrange.Size, tracked); } else { MemoryOwner memoryOwner = MemoryOwner.Rent(checked((int)range.GetSize())); Span memorySpan = memoryOwner.Span; int offset = 0; for (int i = 0; i < range.Count; i++) { var currentRange = range.GetSubRange(i); int size = (int)currentRange.Size; if (currentRange.Address != MemoryManager.PteUnmapped) { GetSpan(currentRange.Address, size).CopyTo(memorySpan.Slice(offset, size)); } offset += size; } return new WritableRegion(new MultiRangeWritableBlock(range, this), 0, memoryOwner, tracked); } } /// /// Reads data from the application process. /// /// Type of the structure /// Address to read from /// The data at the specified memory location public T Read(ulong address) where T : unmanaged { return _cpuMemory.Read(address); } /// /// Reads data from the application process, with write tracking. /// /// Type of the structure /// Address to read from /// The data at the specified memory location public T ReadTracked(ulong address) where T : unmanaged { return _cpuMemory.ReadTracked(address); } /// /// Writes data to the application process, triggering a precise memory tracking event. /// /// Address to write into /// Data to be written public void WriteTrackedResource(ulong address, ReadOnlySpan data) { _cpuMemory.SignalMemoryTracking(address, (ulong)data.Length, true, precise: true); _cpuMemory.WriteUntracked(address, data); } /// /// Writes data to the application process, triggering a precise memory tracking event. /// /// Address to write into /// Data to be written /// Kind of the resource being written, which will not be signalled as CPU modified public void WriteTrackedResource(ulong address, ReadOnlySpan data, ResourceKind kind) { _cpuMemory.SignalMemoryTracking(address, (ulong)data.Length, true, precise: true, exemptId: (int)kind); _cpuMemory.WriteUntracked(address, data); } /// /// Writes data to the application process. /// /// Address to write into /// Data to be written public void Write(ulong address, ReadOnlySpan data) { _cpuMemory.Write(address, data); } /// /// Writes data to the application process. /// /// Ranges of physical memory where the data is located /// Data to be written public void Write(MultiRange range, ReadOnlySpan data) { WriteImpl(range, data, _cpuMemory.Write); } /// /// Writes data to the application process, without any tracking. /// /// Address to write into /// Data to be written public void WriteUntracked(ulong address, ReadOnlySpan data) { _cpuMemory.WriteUntracked(address, data); } /// /// Writes data to the application process, without any tracking. /// /// Ranges of physical memory where the data is located /// Data to be written public void WriteUntracked(MultiRange range, ReadOnlySpan data) { WriteImpl(range, data, _cpuMemory.WriteUntracked); } /// /// Writes data to the application process, returning false if the data was not changed. /// This triggers read memory tracking, as a redundancy check would be useless if the data is not up to date. /// /// The memory manager can return that memory has changed when it hasn't to avoid expensive data copies. /// Address to write into /// Data to be written /// True if the data was changed, false otherwise public bool WriteWithRedundancyCheck(ulong address, ReadOnlySpan data) { return _cpuMemory.WriteWithRedundancyCheck(address, data); } private delegate void WriteCallback(ulong address, ReadOnlySpan data); /// /// Writes data to the application process, using the supplied callback method. /// /// Ranges of physical memory where the data is located /// Data to be written /// Callback method that will perform the write private static void WriteImpl(MultiRange range, ReadOnlySpan data, WriteCallback writeCallback) { if (range.Count == 1) { var singleRange = range.GetSubRange(0); if (singleRange.Address != MemoryManager.PteUnmapped) { writeCallback(singleRange.Address, data); } } else { int offset = 0; for (int i = 0; i < range.Count; i++) { var currentRange = range.GetSubRange(i); int size = (int)currentRange.Size; if (currentRange.Address != MemoryManager.PteUnmapped) { writeCallback(currentRange.Address, data.Slice(offset, size)); } offset += size; } } } /// /// Fills the specified memory region with a 32-bit integer value. /// /// CPU virtual address of the region /// Size of the region /// Value to fill the region with /// Kind of the resource being filled, which will not be signalled as CPU modified public void FillTrackedResource(ulong address, ulong size, uint value, ResourceKind kind) { _cpuMemory.SignalMemoryTracking(address, size, write: true, precise: true, (int)kind); using WritableRegion region = _cpuMemory.GetWritableRegion(address, (int)size); MemoryMarshal.Cast(region.Memory.Span).Fill(value); } /// /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. /// /// CPU virtual address of the region /// Size of the region /// Kind of the resource being tracked /// Region flags /// The memory tracking handle public RegionHandle BeginTracking(ulong address, ulong size, ResourceKind kind, RegionFlags flags = RegionFlags.None) { return _cpuMemory.BeginTracking(address, size, (int)kind, flags); } /// /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. /// /// Ranges of physical memory where the data is located /// Kind of the resource being tracked /// The memory tracking handle public GpuRegionHandle BeginTracking(MultiRange range, ResourceKind kind) { var cpuRegionHandles = new RegionHandle[range.Count]; int count = 0; for (int i = 0; i < range.Count; i++) { var currentRange = range.GetSubRange(i); if (currentRange.Address != MemoryManager.PteUnmapped) { cpuRegionHandles[count++] = _cpuMemory.BeginTracking(currentRange.Address, currentRange.Size, (int)kind); } } if (count != range.Count) { Array.Resize(ref cpuRegionHandles, count); } return new GpuRegionHandle(cpuRegionHandles); } /// /// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. /// /// CPU virtual address of the region /// Size of the region /// Kind of the resource being tracked /// Region flags /// Handles to inherit state from or reuse /// Desired granularity of write tracking /// The memory tracking handle public MultiRegionHandle BeginGranularTracking( ulong address, ulong size, ResourceKind kind, RegionFlags flags = RegionFlags.None, IEnumerable handles = null, ulong granularity = 4096) { return _cpuMemory.BeginGranularTracking(address, size, handles, granularity, (int)kind, flags); } /// /// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. /// /// CPU virtual address of the region /// Size of the region /// Kind of the resource being tracked /// Desired granularity of write tracking /// The memory tracking handle public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ResourceKind kind, ulong granularity = 4096) { return _cpuMemory.BeginSmartGranularTracking(address, size, granularity, (int)kind); } /// /// Checks if a given memory page is mapped. /// /// CPU virtual address of the page /// True if mapped, false otherwise public bool IsMapped(ulong address) { return _cpuMemory.IsMapped(address); } /// /// Release our reference to the CPU memory manager. /// public void Dispose() { _context.DeferredActions.Enqueue(Destroy); } /// /// Performs disposal of the host GPU caches with resources mapped on this physical memory. /// This must only be called from the render thread. /// private void Destroy() { ShaderCache.Dispose(); BufferCache.Dispose(); TextureCache.Dispose(); DecrementReferenceCount(); } } }