using Ryujinx.Common.Memory;
using Ryujinx.Memory;
using System;
using System.Buffers;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
namespace Ryujinx.Graphics.Device
{
///
/// Device memory manager.
///
public class DeviceMemoryManager : IWritableBlock
{
private const int PtLvl0Bits = 10;
private const int PtLvl1Bits = 10;
public const int PtPageBits = 12;
private const ulong PtLvl0Size = 1UL << PtLvl0Bits;
private const ulong PtLvl1Size = 1UL << PtLvl1Bits;
public const ulong PageSize = 1UL << PtPageBits;
private const ulong PtLvl0Mask = PtLvl0Size - 1;
private const ulong PtLvl1Mask = PtLvl1Size - 1;
public const ulong PageMask = PageSize - 1;
private const int PtLvl0Bit = PtPageBits + PtLvl1Bits;
private const int PtLvl1Bit = PtPageBits;
private const int AddressSpaceBits = PtPageBits + PtLvl1Bits + PtLvl0Bits;
public const ulong PteUnmapped = ulong.MaxValue;
private readonly ulong[][] _pageTable;
private readonly IVirtualMemoryManager _physical;
///
/// Creates a new instance of the GPU memory manager.
///
/// Physical memory that this memory manager will map into
public DeviceMemoryManager(IVirtualMemoryManager physicalMemory)
{
_physical = physicalMemory;
_pageTable = new ulong[PtLvl0Size][];
}
///
/// Reads data from GPU mapped memory.
///
/// Type of the data
/// GPU virtual address where the data is located
/// The data at the specified memory location
public T Read(ulong va) where T : unmanaged
{
int size = Unsafe.SizeOf();
if (IsContiguous(va, size))
{
return _physical.Read(Translate(va));
}
else
{
Span data = new byte[size];
ReadImpl(va, data);
return MemoryMarshal.Cast(data)[0];
}
}
///
/// Gets a read-only span of data from GPU mapped memory.
///
/// GPU virtual address where the data is located
/// Size of the data
/// The span of the data at the specified memory location
public ReadOnlySpan GetSpan(ulong va, int size)
{
if (IsContiguous(va, size))
{
return _physical.GetSpan(Translate(va), size);
}
else
{
Span data = new byte[size];
ReadImpl(va, data);
return data;
}
}
///
/// Reads data from a possibly non-contiguous region of GPU mapped memory.
///
/// GPU virtual address of the data
/// Span to write the read data into
private void ReadImpl(ulong va, Span data)
{
if (data.Length == 0)
{
return;
}
int offset = 0, size;
if ((va & PageMask) != 0)
{
ulong pa = Translate(va);
size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask));
if (pa != PteUnmapped && _physical.IsMapped(pa))
{
_physical.GetSpan(pa, size).CopyTo(data[..size]);
}
offset += size;
}
for (; offset < data.Length; offset += size)
{
ulong pa = Translate(va + (ulong)offset);
size = Math.Min(data.Length - offset, (int)PageSize);
if (pa != PteUnmapped && _physical.IsMapped(pa))
{
_physical.GetSpan(pa, size).CopyTo(data.Slice(offset, size));
}
}
}
///
/// Gets a writable region from GPU mapped memory.
///
/// Start address of the range
/// Size in bytes to be range
/// A writable region with the data at the specified memory location
public WritableRegion GetWritableRegion(ulong va, int size)
{
if (IsContiguous(va, size))
{
return _physical.GetWritableRegion(Translate(va), size, tracked: true);
}
else
{
IMemoryOwner memoryOwner = ByteMemoryPool.Rent(size);
GetSpan(va, size).CopyTo(memoryOwner.Memory.Span);
return new WritableRegion(this, va, memoryOwner, tracked: true);
}
}
///
/// Writes data to GPU mapped memory.
///
/// Type of the data
/// GPU virtual address to write the value into
/// The value to be written
public void Write(ulong va, T value) where T : unmanaged
{
Write(va, MemoryMarshal.Cast(MemoryMarshal.CreateSpan(ref value, 1)));
}
///
/// Writes data to GPU mapped memory.
///
/// GPU virtual address to write the data into
/// The data to be written
public void Write(ulong va, ReadOnlySpan data)
{
if (IsContiguous(va, data.Length))
{
_physical.Write(Translate(va), data);
}
else
{
int offset = 0, size;
if ((va & PageMask) != 0)
{
ulong pa = Translate(va);
size = Math.Min(data.Length, (int)PageSize - (int)(va & PageMask));
if (pa != PteUnmapped && _physical.IsMapped(pa))
{
_physical.Write(pa, data[..size]);
}
offset += size;
}
for (; offset < data.Length; offset += size)
{
ulong pa = Translate(va + (ulong)offset);
size = Math.Min(data.Length - offset, (int)PageSize);
if (pa != PteUnmapped && _physical.IsMapped(pa))
{
_physical.Write(pa, data.Slice(offset, size));
}
}
}
}
///
/// Writes data to GPU mapped memory without write tracking.
///
/// GPU virtual address to write the data into
/// The data to be written
public void WriteUntracked(ulong va, ReadOnlySpan data)
{
throw new NotSupportedException();
}
///
/// Maps a given range of pages to the specified CPU virtual address.
///
///
/// All addresses and sizes must be page aligned.
///
/// CPU virtual address to map into
/// GPU virtual address to be mapped
/// Kind of the resource located at the mapping
public void Map(ulong pa, ulong va, ulong size)
{
lock (_pageTable)
{
for (ulong offset = 0; offset < size; offset += PageSize)
{
SetPte(va + offset, PackPte(pa + offset));
}
}
}
///
/// Unmaps a given range of pages at the specified GPU virtual memory region.
///
/// GPU virtual address to unmap
/// Size in bytes of the region being unmapped
public void Unmap(ulong va, ulong size)
{
lock (_pageTable)
{
for (ulong offset = 0; offset < size; offset += PageSize)
{
SetPte(va + offset, PteUnmapped);
}
}
}
///
/// Checks if a region of GPU mapped memory is contiguous.
///
/// GPU virtual address of the region
/// Size of the region
/// True if the region is contiguous, false otherwise
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private bool IsContiguous(ulong va, int size)
{
if (!ValidateAddress(va) || GetPte(va) == PteUnmapped)
{
return false;
}
ulong endVa = (va + (ulong)size + PageMask) & ~PageMask;
va &= ~PageMask;
int pages = (int)((endVa - va) / PageSize);
for (int page = 0; page < pages - 1; page++)
{
if (!ValidateAddress(va + PageSize) || GetPte(va + PageSize) == PteUnmapped)
{
return false;
}
if (Translate(va) + PageSize != Translate(va + PageSize))
{
return false;
}
va += PageSize;
}
return true;
}
///
/// Validates a GPU virtual address.
///
/// Address to validate
/// True if the address is valid, false otherwise
private static bool ValidateAddress(ulong va)
{
return va < (1UL << AddressSpaceBits);
}
///
/// Checks if a given page is mapped.
///
/// GPU virtual address of the page to check
/// True if the page is mapped, false otherwise
public bool IsMapped(ulong va)
{
return Translate(va) != PteUnmapped;
}
///
/// Translates a GPU virtual address to a CPU virtual address.
///
/// GPU virtual address to be translated
/// CPU virtual address, or if unmapped
public ulong Translate(ulong va)
{
if (!ValidateAddress(va))
{
return PteUnmapped;
}
ulong pte = GetPte(va);
if (pte == PteUnmapped)
{
return PteUnmapped;
}
return UnpackPaFromPte(pte) + (va & PageMask);
}
///
/// Gets the Page Table entry for a given GPU virtual address.
///
/// GPU virtual address
/// Page table entry (CPU virtual address)
private ulong GetPte(ulong va)
{
ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask;
ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask;
if (_pageTable[l0] == null)
{
return PteUnmapped;
}
return _pageTable[l0][l1];
}
///
/// Sets a Page Table entry at a given GPU virtual address.
///
/// GPU virtual address
/// Page table entry (CPU virtual address)
private void SetPte(ulong va, ulong pte)
{
ulong l0 = (va >> PtLvl0Bit) & PtLvl0Mask;
ulong l1 = (va >> PtLvl1Bit) & PtLvl1Mask;
if (_pageTable[l0] == null)
{
_pageTable[l0] = new ulong[PtLvl1Size];
for (ulong index = 0; index < PtLvl1Size; index++)
{
_pageTable[l0][index] = PteUnmapped;
}
}
_pageTable[l0][l1] = pte;
}
///
/// Creates a page table entry from a physical address and kind.
///
/// Physical address
/// Page table entry
private static ulong PackPte(ulong pa)
{
return pa;
}
///
/// Unpacks physical address from a page table entry.
///
/// Page table entry
/// Physical address
private static ulong UnpackPaFromPte(ulong pte)
{
return pte;
}
}
}