using Ryujinx.Common.Pools;
using Ryujinx.Memory.Range;
using System.Collections.Generic;
namespace Ryujinx.Memory.Tracking
{
///
/// Manages memory tracking for a given virutal/physical memory block.
///
public class MemoryTracking
{
private readonly IVirtualMemoryManager _memoryManager;
private readonly InvalidAccessHandler _invalidAccessHandler;
// Only use these from within the lock.
private readonly NonOverlappingRangeList _virtualRegions;
// Guest virtual regions are a subset of the normal virtual regions, with potentially different protection
// and expanded area of effect on platforms that don't support misaligned page protection.
private readonly NonOverlappingRangeList _guestVirtualRegions;
private readonly int _pageSize;
private readonly bool _singleByteGuestTracking;
///
/// This lock must be obtained when traversing or updating the region-handle hierarchy.
/// It is not required when reading dirty flags.
///
internal object TrackingLock = new();
///
/// Create a new tracking structure for the given "physical" memory block,
/// with a given "virtual" memory manager that will provide mappings and virtual memory protection.
///
///
/// If is true, the memory manager must also support protection on partially
/// unmapped regions without throwing exceptions or dropping protection on the mapped portion.
///
/// Virtual memory manager
/// Page size of the virtual memory space
/// Method to call for invalid memory accesses
/// True if the guest only signals writes for the first byte
public MemoryTracking(
IVirtualMemoryManager memoryManager,
int pageSize,
InvalidAccessHandler invalidAccessHandler = null,
bool singleByteGuestTracking = false)
{
_memoryManager = memoryManager;
_pageSize = pageSize;
_invalidAccessHandler = invalidAccessHandler;
_singleByteGuestTracking = singleByteGuestTracking;
_virtualRegions = new NonOverlappingRangeList();
_guestVirtualRegions = new NonOverlappingRangeList();
}
private (ulong address, ulong size) PageAlign(ulong address, ulong size)
{
ulong pageMask = (ulong)_pageSize - 1;
ulong rA = address & ~pageMask;
ulong rS = ((address + size + pageMask) & ~pageMask) - rA;
return (rA, rS);
}
///
/// Indicate that a virtual region has been mapped, and which physical region it has been mapped to.
/// Should be called after the mapping is complete.
///
/// Virtual memory address
/// Size to be mapped
public void Map(ulong va, ulong size)
{
// A mapping may mean we need to re-evaluate each VirtualRegion's affected area.
// Find all handles that overlap with the range, we need to recalculate their physical regions
lock (TrackingLock)
{
ref var overlaps = ref ThreadStaticArray.Get();
for (int type = 0; type < 2; type++)
{
NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps);
for (int i = 0; i < count; i++)
{
VirtualRegion region = overlaps[i];
// If the region has been fully remapped, signal that it has been mapped again.
bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size);
if (remapped)
{
region.SignalMappingChanged(true);
}
region.UpdateProtection();
}
}
}
}
///
/// Indicate that a virtual region has been unmapped.
/// Should be called before the unmapping is complete.
///
/// Virtual memory address
/// Size to be unmapped
public void Unmap(ulong va, ulong size)
{
// An unmapping may mean we need to re-evaluate each VirtualRegion's affected area.
// Find all handles that overlap with the range, we need to notify them that the region was unmapped.
lock (TrackingLock)
{
ref var overlaps = ref ThreadStaticArray.Get();
for (int type = 0; type < 2; type++)
{
NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions;
int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps);
for (int i = 0; i < count; i++)
{
VirtualRegion region = overlaps[i];
region.SignalMappingChanged(false);
}
}
}
}
///
/// Alter a tracked memory region to properly capture unaligned accesses.
/// For most memory manager modes, this does nothing.
///
/// Original region address
/// Original region size
/// A new address and size for tracking unaligned accesses
internal (ulong newAddress, ulong newSize) GetUnalignedSafeRegion(ulong address, ulong size)
{
if (_singleByteGuestTracking)
{
// The guest only signals the first byte of each memory access with the current memory manager.
// To catch unaligned access properly, we need to also protect the page before the address.
// Assume that the address and size are already aligned.
return (address - (ulong)_pageSize, size + (ulong)_pageSize);
}
else
{
return (address, size);
}
}
///
/// Get a list of virtual regions that a handle covers.
///
/// Starting virtual memory address of the handle
/// Size of the handle's memory region
/// True if getting handles for guest protection, false otherwise
/// A list of virtual regions within the given range
internal List GetVirtualRegionsForHandle(ulong va, ulong size, bool guest)
{
List result = new();
NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions;
regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest));
return result;
}
///
/// Remove a virtual region from the range list. This assumes that the lock has been acquired.
///
/// Region to remove
internal void RemoveVirtual(VirtualRegion region)
{
if (region.Guest)
{
_guestVirtualRegions.Remove(region);
}
else
{
_virtualRegions.Remove(region);
}
}
///
/// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
///
/// CPU virtual address of the region
/// Size of the region
/// Handles to inherit state from or reuse. When none are present, provide null
/// Desired granularity of write tracking
/// Handle ID
/// Region flags
/// The memory tracking handle
public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None)
{
return new MultiRegionHandle(this, address, size, handles, granularity, id, flags);
}
///
/// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with.
///
/// CPU virtual address of the region
/// Size of the region
/// Desired granularity of write tracking
/// Handle ID
/// The memory tracking handle
public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id)
{
(address, size) = PageAlign(address, size);
return new SmartMultiRegionHandle(this, address, size, granularity, id);
}
///
/// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with.
///
/// CPU virtual address of the region
/// Size of the region
/// Handle ID
/// Region flags
/// The memory tracking handle
public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None)
{
var (paAddress, paSize) = PageAlign(address, size);
lock (TrackingLock)
{
bool mapped = _memoryManager.IsRangeMapped(address, size);
RegionHandle handle = new(this, paAddress, paSize, address, size, id, flags, mapped);
return handle;
}
}
///
/// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with.
///
/// CPU virtual address of the region
/// Size of the region
/// The bitmap owning the dirty flag for this handle
/// The bit of this handle within the dirty flag
/// Handle ID
/// Region flags
/// The memory tracking handle
internal RegionHandle BeginTrackingBitmap(ulong address, ulong size, ConcurrentBitmap bitmap, int bit, int id, RegionFlags flags = RegionFlags.None)
{
var (paAddress, paSize) = PageAlign(address, size);
lock (TrackingLock)
{
bool mapped = _memoryManager.IsRangeMapped(address, size);
RegionHandle handle = new(this, paAddress, paSize, address, size, bitmap, bit, id, flags, mapped);
return handle;
}
}
///
/// Signal that a virtual memory event happened at the given location.
/// The memory event is assumed to be triggered by guest code.
///
/// Virtual address accessed
/// Size of the region affected in bytes
/// Whether the region was written to or read
/// True if the event triggered any tracking regions, false otherwise
public bool VirtualMemoryEvent(ulong address, ulong size, bool write)
{
return VirtualMemoryEvent(address, size, write, precise: false, exemptId: null, guest: true);
}
///
/// Signal that a virtual memory event happened at the given location.
/// This can be flagged as a precise event, which will avoid reprotection and call special handlers if possible.
/// A precise event has an exact address and size, rather than triggering on page granularity.
///
/// Virtual address accessed
/// Size of the region affected in bytes
/// Whether the region was written to or read
/// True if the access is precise, false otherwise
/// Optional ID that of the handles that should not be signalled
/// True if the access is from the guest, false otherwise
/// True if the event triggered any tracking regions, false otherwise
public bool VirtualMemoryEvent(ulong address, ulong size, bool write, bool precise, int? exemptId = null, bool guest = false)
{
// Look up the virtual region using the region list.
// Signal up the chain to relevant handles.
bool shouldThrow = false;
lock (TrackingLock)
{
ref var overlaps = ref ThreadStaticArray.Get();
NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions;
int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps);
if (count == 0 && !precise)
{
if (_memoryManager.IsRangeMapped(address, size))
{
// TODO: There is currently the possibility that a page can be protected after its virtual region is removed.
// This code handles that case when it happens, but it would be better to find out how this happens.
_memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest);
return true; // This memory _should_ be mapped, so we need to try again.
}
else
{
shouldThrow = true;
}
}
else
{
if (guest && _singleByteGuestTracking)
{
// Increase the access size to trigger handles with misaligned accesses.
size += (ulong)_pageSize;
}
for (int i = 0; i < count; i++)
{
VirtualRegion region = overlaps[i];
if (precise)
{
region.SignalPrecise(address, size, write, exemptId);
}
else
{
region.Signal(address, size, write, exemptId);
}
}
}
}
if (shouldThrow)
{
_invalidAccessHandler?.Invoke(address);
// We can't continue - it's impossible to remove protection from the page.
// Even if the access handler wants us to continue, we wouldn't be able to.
throw new InvalidMemoryRegionException();
}
return true;
}
///
/// Reprotect a given virtual region. The virtual memory manager will handle this.
///
/// Region to reprotect
/// Memory permission to protect with
/// True if the protection is for guest access, false otherwise
internal void ProtectVirtualRegion(VirtualRegion region, MemoryPermission permission, bool guest)
{
_memoryManager.TrackingReprotect(region.Address, region.Size, permission, guest);
}
///
/// Returns the number of virtual regions currently being tracked.
/// Useful for tests and metrics.
///
/// The number of virtual regions
public int GetRegionCount()
{
lock (TrackingLock)
{
return _virtualRegions.Count;
}
}
}
}