using Ryujinx.Common.Pools; using Ryujinx.Memory.Range; using System.Collections.Generic; namespace Ryujinx.Memory.Tracking { /// /// Manages memory tracking for a given virutal/physical memory block. /// public class MemoryTracking { private readonly IVirtualMemoryManager _memoryManager; private readonly InvalidAccessHandler _invalidAccessHandler; // Only use these from within the lock. private readonly NonOverlappingRangeList _virtualRegions; // Guest virtual regions are a subset of the normal virtual regions, with potentially different protection // and expanded area of effect on platforms that don't support misaligned page protection. private readonly NonOverlappingRangeList _guestVirtualRegions; private readonly int _pageSize; private readonly bool _singleByteGuestTracking; /// /// This lock must be obtained when traversing or updating the region-handle hierarchy. /// It is not required when reading dirty flags. /// internal object TrackingLock = new(); /// /// Create a new tracking structure for the given "physical" memory block, /// with a given "virtual" memory manager that will provide mappings and virtual memory protection. /// /// /// If is true, the memory manager must also support protection on partially /// unmapped regions without throwing exceptions or dropping protection on the mapped portion. /// /// Virtual memory manager /// Page size of the virtual memory space /// Method to call for invalid memory accesses /// True if the guest only signals writes for the first byte public MemoryTracking( IVirtualMemoryManager memoryManager, int pageSize, InvalidAccessHandler invalidAccessHandler = null, bool singleByteGuestTracking = false) { _memoryManager = memoryManager; _pageSize = pageSize; _invalidAccessHandler = invalidAccessHandler; _singleByteGuestTracking = singleByteGuestTracking; _virtualRegions = new NonOverlappingRangeList(); _guestVirtualRegions = new NonOverlappingRangeList(); } private (ulong address, ulong size) PageAlign(ulong address, ulong size) { ulong pageMask = (ulong)_pageSize - 1; ulong rA = address & ~pageMask; ulong rS = ((address + size + pageMask) & ~pageMask) - rA; return (rA, rS); } /// /// Indicate that a virtual region has been mapped, and which physical region it has been mapped to. /// Should be called after the mapping is complete. /// /// Virtual memory address /// Size to be mapped public void Map(ulong va, ulong size) { // A mapping may mean we need to re-evaluate each VirtualRegion's affected area. // Find all handles that overlap with the range, we need to recalculate their physical regions lock (TrackingLock) { ref var overlaps = ref ThreadStaticArray.Get(); for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); for (int i = 0; i < count; i++) { VirtualRegion region = overlaps[i]; // If the region has been fully remapped, signal that it has been mapped again. bool remapped = _memoryManager.IsRangeMapped(region.Address, region.Size); if (remapped) { region.SignalMappingChanged(true); } region.UpdateProtection(); } } } } /// /// Indicate that a virtual region has been unmapped. /// Should be called before the unmapping is complete. /// /// Virtual memory address /// Size to be unmapped public void Unmap(ulong va, ulong size) { // An unmapping may mean we need to re-evaluate each VirtualRegion's affected area. // Find all handles that overlap with the range, we need to notify them that the region was unmapped. lock (TrackingLock) { ref var overlaps = ref ThreadStaticArray.Get(); for (int type = 0; type < 2; type++) { NonOverlappingRangeList regions = type == 0 ? _virtualRegions : _guestVirtualRegions; int count = regions.FindOverlapsNonOverlapping(va, size, ref overlaps); for (int i = 0; i < count; i++) { VirtualRegion region = overlaps[i]; region.SignalMappingChanged(false); } } } } /// /// Alter a tracked memory region to properly capture unaligned accesses. /// For most memory manager modes, this does nothing. /// /// Original region address /// Original region size /// A new address and size for tracking unaligned accesses internal (ulong newAddress, ulong newSize) GetUnalignedSafeRegion(ulong address, ulong size) { if (_singleByteGuestTracking) { // The guest only signals the first byte of each memory access with the current memory manager. // To catch unaligned access properly, we need to also protect the page before the address. // Assume that the address and size are already aligned. return (address - (ulong)_pageSize, size + (ulong)_pageSize); } else { return (address, size); } } /// /// Get a list of virtual regions that a handle covers. /// /// Starting virtual memory address of the handle /// Size of the handle's memory region /// True if getting handles for guest protection, false otherwise /// A list of virtual regions within the given range internal List GetVirtualRegionsForHandle(ulong va, ulong size, bool guest) { List result = new(); NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; regions.GetOrAddRegions(result, va, size, (va, size) => new VirtualRegion(this, va, size, guest)); return result; } /// /// Remove a virtual region from the range list. This assumes that the lock has been acquired. /// /// Region to remove internal void RemoveVirtual(VirtualRegion region) { if (region.Guest) { _guestVirtualRegions.Remove(region); } else { _virtualRegions.Remove(region); } } /// /// Obtains a memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. /// /// CPU virtual address of the region /// Size of the region /// Handles to inherit state from or reuse. When none are present, provide null /// Desired granularity of write tracking /// Handle ID /// Region flags /// The memory tracking handle public MultiRegionHandle BeginGranularTracking(ulong address, ulong size, IEnumerable handles, ulong granularity, int id, RegionFlags flags = RegionFlags.None) { return new MultiRegionHandle(this, address, size, handles, granularity, id, flags); } /// /// Obtains a smart memory tracking handle for the given virtual region, with a specified granularity. This should be disposed when finished with. /// /// CPU virtual address of the region /// Size of the region /// Desired granularity of write tracking /// Handle ID /// The memory tracking handle public SmartMultiRegionHandle BeginSmartGranularTracking(ulong address, ulong size, ulong granularity, int id) { (address, size) = PageAlign(address, size); return new SmartMultiRegionHandle(this, address, size, granularity, id); } /// /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. /// /// CPU virtual address of the region /// Size of the region /// Handle ID /// Region flags /// The memory tracking handle public RegionHandle BeginTracking(ulong address, ulong size, int id, RegionFlags flags = RegionFlags.None) { var (paAddress, paSize) = PageAlign(address, size); lock (TrackingLock) { bool mapped = _memoryManager.IsRangeMapped(address, size); RegionHandle handle = new(this, paAddress, paSize, address, size, id, flags, mapped); return handle; } } /// /// Obtains a memory tracking handle for the given virtual region. This should be disposed when finished with. /// /// CPU virtual address of the region /// Size of the region /// The bitmap owning the dirty flag for this handle /// The bit of this handle within the dirty flag /// Handle ID /// Region flags /// The memory tracking handle internal RegionHandle BeginTrackingBitmap(ulong address, ulong size, ConcurrentBitmap bitmap, int bit, int id, RegionFlags flags = RegionFlags.None) { var (paAddress, paSize) = PageAlign(address, size); lock (TrackingLock) { bool mapped = _memoryManager.IsRangeMapped(address, size); RegionHandle handle = new(this, paAddress, paSize, address, size, bitmap, bit, id, flags, mapped); return handle; } } /// /// Signal that a virtual memory event happened at the given location. /// The memory event is assumed to be triggered by guest code. /// /// Virtual address accessed /// Size of the region affected in bytes /// Whether the region was written to or read /// True if the event triggered any tracking regions, false otherwise public bool VirtualMemoryEvent(ulong address, ulong size, bool write) { return VirtualMemoryEvent(address, size, write, precise: false, exemptId: null, guest: true); } /// /// Signal that a virtual memory event happened at the given location. /// This can be flagged as a precise event, which will avoid reprotection and call special handlers if possible. /// A precise event has an exact address and size, rather than triggering on page granularity. /// /// Virtual address accessed /// Size of the region affected in bytes /// Whether the region was written to or read /// True if the access is precise, false otherwise /// Optional ID that of the handles that should not be signalled /// True if the access is from the guest, false otherwise /// True if the event triggered any tracking regions, false otherwise public bool VirtualMemoryEvent(ulong address, ulong size, bool write, bool precise, int? exemptId = null, bool guest = false) { // Look up the virtual region using the region list. // Signal up the chain to relevant handles. bool shouldThrow = false; lock (TrackingLock) { ref var overlaps = ref ThreadStaticArray.Get(); NonOverlappingRangeList regions = guest ? _guestVirtualRegions : _virtualRegions; int count = regions.FindOverlapsNonOverlapping(address, size, ref overlaps); if (count == 0 && !precise) { if (_memoryManager.IsRangeMapped(address, size)) { // TODO: There is currently the possibility that a page can be protected after its virtual region is removed. // This code handles that case when it happens, but it would be better to find out how this happens. _memoryManager.TrackingReprotect(address & ~(ulong)(_pageSize - 1), (ulong)_pageSize, MemoryPermission.ReadAndWrite, guest); return true; // This memory _should_ be mapped, so we need to try again. } else { shouldThrow = true; } } else { if (guest && _singleByteGuestTracking) { // Increase the access size to trigger handles with misaligned accesses. size += (ulong)_pageSize; } for (int i = 0; i < count; i++) { VirtualRegion region = overlaps[i]; if (precise) { region.SignalPrecise(address, size, write, exemptId); } else { region.Signal(address, size, write, exemptId); } } } } if (shouldThrow) { _invalidAccessHandler?.Invoke(address); // We can't continue - it's impossible to remove protection from the page. // Even if the access handler wants us to continue, we wouldn't be able to. throw new InvalidMemoryRegionException(); } return true; } /// /// Reprotect a given virtual region. The virtual memory manager will handle this. /// /// Region to reprotect /// Memory permission to protect with /// True if the protection is for guest access, false otherwise internal void ProtectVirtualRegion(VirtualRegion region, MemoryPermission permission, bool guest) { _memoryManager.TrackingReprotect(region.Address, region.Size, permission, guest); } /// /// Returns the number of virtual regions currently being tracked. /// Useful for tests and metrics. /// /// The number of virtual regions public int GetRegionCount() { lock (TrackingLock) { return _virtualRegions.Count; } } } }