Implement a new physical memory manager and replace DeviceMemory (#856)

* Implement a new physical memory manager and replace DeviceMemory * Proper generic constraints * Fix debug build * Add memory tests * New CPU memory manager and general code cleanup * Remove host memory management from CPU project, use Ryujinx.Memory instead * Fix tests * Document exceptions on MemoryBlock * Fix leak on unix memory allocation * Proper disposal of some objects on tests * Fix JitCache not being set as initialized * GetRef without checks for 8-bits and 16-bits CAS * Add MemoryBlock destructor * Throw in separate method to improve codegen * Address PR feedback * QueryModified improvements * Fix memory write tracking not marking all pages as modified in some cases * Simplify MarkRegionAsModified * Remove XML doc for ghost param * Add back optimization to avoid useless buffer updates * Add Ryujinx.Cpu project, move MemoryManager there and remove MemoryBlockWrapper * Some nits * Do not perform address translation when size is 0 * Address PR feedback and format NativeInterface class * Remove ghost parameter description * Update Ryujinx.Cpu to .NET Core 3.1 * Address PR feedback * Fix build * Return a well defined value for GetPhysicalAddress with invalid VA, and do not return unmapped ranges as modified * Typo
author: gdkchan <gab.dark.100@gmail.com> 2020-05-03 19:54:50 -0300
committer: GitHub <noreply@github.com> 2020-05-04 08:54:50 +1000
commit: f77694e4f774c9391aad5344e70a7c8721cfedc6 (patch)
tree: 36bccaeb153bd5d921c751966b8a734a7b4b6ae1 /Ryujinx.Cpu
parent: 1758424208335d1f4ff7c27c554e517c81bf72f6 (diff)
7 files changed, 632 insertions, 0 deletions
diff --git a/Ryujinx.Cpu/CpuContext.cs b/Ryujinx.Cpu/CpuContext.cs
new file mode 100644
index 00000000..275fcc68
--- /dev/null
+++ b/Ryujinx.Cpu/CpuContext.cs
@@ -0,0 +1,19 @@
+using ARMeilleure.State;
+using ARMeilleure.Translation;
+
+namespace Ryujinx.Cpu
+{
+    public class CpuContext
+    {
+        private readonly Translator _translator;
+
+        public CpuContext(MemoryManager memory)
+        {
+            _translator = new Translator(new JitMemoryAllocator(), memory);
+        }
+
+        public static ExecutionContext CreateExecutionContext() => new ExecutionContext(new JitMemoryAllocator());
+
+        public void Execute(ExecutionContext context, ulong address) => _translator.Execute(context, address);
+    }
+}
diff --git a/Ryujinx.Cpu/JitMemoryAllocator.cs b/Ryujinx.Cpu/JitMemoryAllocator.cs
new file mode 100644
index 00000000..26ccd732
--- /dev/null
+++ b/Ryujinx.Cpu/JitMemoryAllocator.cs
@@ -0,0 +1,11 @@
+using ARMeilleure.Memory;
+using Ryujinx.Memory;
+
+namespace Ryujinx.Cpu
+{
+    class JitMemoryAllocator : IJitMemoryAllocator
+    {
+        public IJitMemoryBlock Allocate(ulong size) => new JitMemoryBlock(size, MemoryAllocationFlags.None);
+        public IJitMemoryBlock Reserve(ulong size) => new JitMemoryBlock(size, MemoryAllocationFlags.Reserve);
+    }
+}
diff --git a/Ryujinx.Cpu/JitMemoryBlock.cs b/Ryujinx.Cpu/JitMemoryBlock.cs
new file mode 100644
index 00000000..3ad62d71
--- /dev/null
+++ b/Ryujinx.Cpu/JitMemoryBlock.cs
@@ -0,0 +1,24 @@
+using ARMeilleure.Memory;
+using Ryujinx.Memory;
+using System;
+
+namespace Ryujinx.Cpu
+{
+    class JitMemoryBlock : IJitMemoryBlock
+    {
+        private readonly MemoryBlock _impl;
+
+        public IntPtr Pointer => _impl.Pointer;
+
+        public JitMemoryBlock(ulong size, MemoryAllocationFlags flags)
+        {
+            _impl = new MemoryBlock(size, flags);
+        }
+
+        public bool Commit(ulong offset, ulong size) => _impl.Commit(offset, size);
+        public void MapAsRx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadAndExecute);
+        public void MapAsRwx(ulong offset, ulong size) => _impl.Reprotect(offset, size, MemoryPermission.ReadWriteExecute);
+
+        public void Dispose() => _impl.Dispose();
+    }
+}
diff --git a/Ryujinx.Cpu/MemoryHelper.cs b/Ryujinx.Cpu/MemoryHelper.cs
new file mode 100644
index 00000000..5ba5ef38
--- /dev/null
+++ b/Ryujinx.Cpu/MemoryHelper.cs
@@ -0,0 +1,73 @@
+using System;
+using System.IO;
+using System.Runtime.InteropServices;
+using System.Text;
+
+namespace Ryujinx.Cpu
+{
+    public static class MemoryHelper
+    {
+        public static void FillWithZeros(MemoryManager memory, long position, int size)
+        {
+            int size8 = size & ~(8 - 1);
+
+            for (int offs = 0; offs < size8; offs += 8)
+            {
+                memory.Write<long>((ulong)(position + offs), 0);
+            }
+
+            for (int offs = size8; offs < (size - size8); offs++)
+            {
+                memory.Write<byte>((ulong)(position + offs), 0);
+            }
+        }
+
+        public unsafe static T Read<T>(MemoryManager memory, long position) where T : struct
+        {
+            long size = Marshal.SizeOf<T>();
+
+            byte[] data = new byte[size];
+
+            memory.Read((ulong)position, data);
+
+            fixed (byte* ptr = data)
+            {
+                return Marshal.PtrToStructure<T>((IntPtr)ptr);
+            }
+        }
+
+        public unsafe static void Write<T>(MemoryManager memory, long position, T value) where T : struct
+        {
+            long size = Marshal.SizeOf<T>();
+
+            byte[] data = new byte[size];
+
+            fixed (byte* ptr = data)
+            {
+                Marshal.StructureToPtr<T>(value, (IntPtr)ptr, false);
+            }
+
+            memory.Write((ulong)position, data);
+        }
+
+        public static string ReadAsciiString(MemoryManager memory, long position, long maxSize = -1)
+        {
+            using (MemoryStream ms = new MemoryStream())
+            {
+                for (long offs = 0; offs < maxSize || maxSize == -1; offs++)
+                {
+                    byte value = memory.Read<byte>((ulong)(position + offs));
+
+                    if (value == 0)
+                    {
+                        break;
+                    }
+
+                    ms.WriteByte(value);
+                }
+
+                return Encoding.ASCII.GetString(ms.ToArray());
+            }
+        }
+    }
+}
+\ No newline at end of file
diff --git a/Ryujinx.Cpu/MemoryManager.cs b/Ryujinx.Cpu/MemoryManager.cs
new file mode 100644
index 00000000..001c41f8
--- /dev/null
+++ b/Ryujinx.Cpu/MemoryManager.cs
@@ -0,0 +1,466 @@
+using ARMeilleure.Memory;
+using Ryujinx.Memory;
+using System;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Threading;
+
+namespace Ryujinx.Cpu
+{
+    /// <summary>
+    /// Represents a CPU memory manager.
+    /// </summary>
+    public sealed class MemoryManager : IMemoryManager, IDisposable
+    {
+        public const int PageBits = 12;
+        public const int PageSize = 1 << PageBits;
+        public const int PageMask = PageSize - 1;
+
+        private const int PteSize = 8;
+
+        public int AddressSpaceBits { get; }
+
+        private readonly ulong _addressSpaceSize;
+
+        private readonly MemoryBlock _backingMemory;
+        private readonly MemoryBlock _pageTable;
+
+        public IntPtr PageTablePointer => _pageTable.Pointer;
+
+        /// <summary>
+        /// Creates a new instance of the memory manager.
+        /// </summary>
+        /// <param name="backingMemory">Physical backing memory where virtual memory will be mapped to</param>
+        /// <param name="addressSpaceSize">Size of the address space</param>
+        public MemoryManager(MemoryBlock backingMemory, ulong addressSpaceSize)
+        {
+            ulong asSize = PageSize;
+            int asBits = PageBits;
+
+            while (asSize < addressSpaceSize)
+            {
+                asSize <<= 1;
+                asBits++;
+            }
+
+            AddressSpaceBits = asBits;
+            _addressSpaceSize = asSize;
+            _backingMemory = backingMemory;
+            _pageTable = new MemoryBlock((asSize / PageSize) * PteSize);
+        }
+
+        /// <summary>
+        /// Maps a virtual memory range into a physical memory range.
+        /// </summary>
+        /// <remarks>
+        /// Addresses and size must be page aligned.
+        /// </remarks>
+        /// <param name="va">Virtual memory address</param>
+        /// <param name="pa">Physical memory address</param>
+        /// <param name="size">Size to be mapped</param>
+        public void Map(ulong va, ulong pa, ulong size)
+        {
+            while (size != 0)
+            {
+                _pageTable.Write((va / PageSize) * PteSize, PaToPte(pa));
+
+                va += PageSize;
+                pa += PageSize;
+                size -= PageSize;
+            }
+        }
+
+        /// <summary>
+        /// Unmaps a previously mapped range of virtual memory.
+        /// </summary>
+        /// <param name="va">Virtual address of the range to be unmapped</param>
+        /// <param name="size">Size of the range to be unmapped</param>
+        public void Unmap(ulong va, ulong size)
+        {
+            while (size != 0)
+            {
+                _pageTable.Write((va / PageSize) * PteSize, 0UL);
+
+                va += PageSize;
+                size -= PageSize;
+            }
+        }
+
+        /// <summary>
+        /// Reads data from CPU mapped memory.
+        /// </summary>
+        /// <typeparam name="T">Type of the data being read</typeparam>
+        /// <param name="va">Virtual address of the data in memory</param>
+        /// <returns>The data</returns>
+        public T Read<T>(ulong va) where T : unmanaged
+        {
+            return MemoryMarshal.Cast<byte, T>(GetSpan(va, Unsafe.SizeOf<T>()))[0];
+        }
+
+        /// <summary>
+        /// Reads data from CPU mapped memory.
+        /// </summary>
+        /// <param name="va">Virtual address of the data in memory</param>
+        /// <param name="data">Span to store the data being read into</param>
+        public void Read(ulong va, Span<byte> data)
+        {
+            ReadImpl(va, data);
+        }
+
+        /// <summary>
+        /// Writes data to CPU mapped memory.
+        /// </summary>
+        /// <typeparam name="T">Type of the data being written</typeparam>
+        /// <param name="va">Virtual address to write the data into</param>
+        /// <param name="value">Data to be written</param>
+        public void Write<T>(ulong va, T value) where T : unmanaged
+        {
+            Write(va, MemoryMarshal.Cast<T, byte>(MemoryMarshal.CreateSpan(ref value, 1)));
+        }
+
+        /// <summary>
+        /// Writes data to CPU mapped memory.
+        /// </summary>
+        /// <param name="va">Virtual address to write the data into</param>
+        /// <param name="data">Data to be written</param>
+        public void Write(ulong va, ReadOnlySpan<byte> data)
+        {
+            if (data.Length == 0)
+            {
+                return;
+            }
+
+            MarkRegionAsModified(va, (ulong)data.Length);
+
+            if (IsContiguous(va, data.Length))
+            {
+                data.CopyTo(_backingMemory.GetSpan(GetPhysicalAddressInternal(va), data.Length));
+            }
+            else
+            {
+                int offset = 0, size;
+
+                if ((va & PageMask) != 0)
+                {
+                    ulong pa = GetPhysicalAddressInternal(va);
+
+                    size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
+
+                    data.Slice(0, size).CopyTo(_backingMemory.GetSpan(pa, size));
+
+                    offset += size;
+                }
+
+                for (; offset < data.Length; offset += size)
+                {
+                    ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
+
+                    size = Math.Min(data.Length - offset, PageSize);
+
+                    data.Slice(offset, size).CopyTo(_backingMemory.GetSpan(pa, size));
+                }
+            }
+        }
+
+        /// <summary>
+        /// Gets a read-only span of data from CPU mapped memory.
+        /// </summary>
+        /// <remarks>
+        /// This may perform a allocation if the data is not contiguous in memory.
+        /// For this reason, the span is read-only, you can't modify the data.
+        /// </remarks>
+        /// <param name="va">Virtual address of the data</param>
+        /// <param name="size">Size of the data</param>
+        /// <returns>A read-only span of the data</returns>
+        public ReadOnlySpan<byte> GetSpan(ulong va, int size)
+        {
+            if (size == 0)
+            {
+                return ReadOnlySpan<byte>.Empty;
+            }
+
+            if (IsContiguous(va, size))
+            {
+                return _backingMemory.GetSpan(GetPhysicalAddressInternal(va), size);
+            }
+            else
+            {
+                Span<byte> data = new byte[size];
+
+                ReadImpl(va, data);
+
+                return data;
+            }
+        }
+
+        /// <summary>
+        /// Gets a reference for the given type at the specified virtual memory address.
+        /// </summary>
+        /// <remarks>
+        /// The data must be located at a contiguous memory region.
+        /// </remarks>
+        /// <typeparam name="T">Type of the data to get the reference</typeparam>
+        /// <param name="va">Virtual address of the data</param>
+        /// <returns>A reference to the data in memory</returns>
+        public ref T GetRef<T>(ulong va) where T : unmanaged
+        {
+            if (!IsContiguous(va, Unsafe.SizeOf<T>()))
+            {
+                ThrowMemoryNotContiguous();
+            }
+
+            MarkRegionAsModified(va, (ulong)Unsafe.SizeOf<T>());
+
+            return ref _backingMemory.GetRef<T>(GetPhysicalAddressInternal(va));
+        }
+
+        private void ThrowMemoryNotContiguous() => throw new MemoryNotContiguousException();
+
+        // TODO: Remove that once we have proper 8-bits and 16-bits CAS.
+        public ref T GetRefNoChecks<T>(ulong va) where T : unmanaged
+        {
+            MarkRegionAsModified(va, (ulong)Unsafe.SizeOf<T>());
+
+            return ref _backingMemory.GetRef<T>(GetPhysicalAddressInternal(va));
+        }
+
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        private bool IsContiguous(ulong va, int size)
+        {
+            if (!ValidateAddress(va))
+            {
+                return false;
+            }
+
+            ulong endVa = (va + (ulong)size + PageMask) & ~(ulong)PageMask;
+
+            va &= ~(ulong)PageMask;
+
+            int pages = (int)((endVa - va) / PageSize);
+
+            for (int page = 0; page < pages - 1; page++)
+            {
+                if (!ValidateAddress(va + PageSize))
+                {
+                    return false;
+                }
+
+                if (GetPhysicalAddressInternal(va) + PageSize != GetPhysicalAddressInternal(va + PageSize))
+                {
+                    return false;
+                }
+
+                va += PageSize;
+            }
+
+            return true;
+        }
+
+        private void ReadImpl(ulong va, Span<byte> data)
+        {
+            if (data.Length == 0)
+            {
+                return;
+            }
+
+            int offset = 0, size;
+
+            if ((va & PageMask) != 0)
+            {
+                ulong pa = GetPhysicalAddressInternal(va);
+
+                size = Math.Min(data.Length, PageSize - (int)(va & PageMask));
+
+                _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(0, size));
+
+                offset += size;
+            }
+
+            for (; offset < data.Length; offset += size)
+            {
+                ulong pa = GetPhysicalAddressInternal(va + (ulong)offset);
+
+                size = Math.Min(data.Length - offset, PageSize);
+
+                _backingMemory.GetSpan(pa, size).CopyTo(data.Slice(offset, size));
+            }
+        }
+
+        /// <summary>
+        /// Checks if a specified virtual memory region has been modified by the CPU since the last call.
+        /// </summary>
+        /// <param name="va">Virtual address of the region</param>
+        /// <param name="size">Size of the region</param>
+        /// <param name="id">Resource identifier number (maximum is 15)</param>
+        /// <param name="modifiedRanges">Optional array where the modified ranges should be written</param>
+        /// <returns>The number of modified ranges</returns>
+        [MethodImpl(MethodImplOptions.AggressiveInlining)]
+        public int QueryModified(ulong va, ulong size, int id, (ulong, ulong)[] modifiedRanges = null)
+        {
+            if (!ValidateAddress(va))
+            {
+                return 0;
+            }
+
+            ulong maxSize = _addressSpaceSize - va;
+
+            if (size > maxSize)
+            {
+                size = maxSize;
+            }
+
+            // We need to ensure that the tagged pointer value is negative,
+            // JIT generated code checks that to take the slow paths and call the MemoryManager Read/Write methods.
+            long tag = (0x8000L | (1L << id)) << 48;
+
+            ulong endVa = (va + size + PageMask) & ~(ulong)PageMask;
+
+            va &= ~(ulong)PageMask;
+
+            ulong rgStart = va;
+            ulong rgSize = 0;
+
+            int rangeIndex = 0;
+
+            for (; va < endVa; va += PageSize)
+            {
+                while (true)
+                {
+                    ref long pte = ref _pageTable.GetRef<long>((va >> PageBits) * PteSize);
+
+                    long pteValue = pte;
+
+                    // If the PTE value is 0, that means that the page is unmapped.
+                    // We behave as if the page was not modified, since modifying a page
+                    // that is not even mapped is impossible.
+                    if ((pteValue & tag) == tag || pteValue == 0)
+                    {
+                        if (rgSize != 0)
+                        {
+                            if (modifiedRanges != null && rangeIndex < modifiedRanges.Length)
+                            {
+                                modifiedRanges[rangeIndex] = (rgStart, rgSize);
+                            }
+
+                            rangeIndex++;
+
+                            rgSize = 0;
+                        }
+
+                        break;
+                    }
+                    else
+                    {
+                        if (Interlocked.CompareExchange(ref pte, pteValue | tag, pteValue) == pteValue)
+                        {
+                            if (rgSize == 0)
+                            {
+                                rgStart = va;
+                            }
+
+                            rgSize += PageSize;
+
+                            break;
+                        }
+                    }
+                }
+            }
+
+            if (rgSize != 0)
+            {
+                if (modifiedRanges != null && rangeIndex < modifiedRanges.Length)
+                {
+                    modifiedRanges[rangeIndex] = (rgStart, rgSize);
+                }
+
+                rangeIndex++;
+            }
+
+            return rangeIndex;
+        }
+
+        /// <summary>
+        /// Checks if the page at a given CPU virtual address.
+        /// </summary>
+        /// <param name="va">Virtual address to check</param>
+        /// <returns>True if the address is mapped, false otherwise</returns>
+        public bool IsMapped(ulong va)
+        {
+            if (!ValidateAddress(va))
+            {
+                return false;
+            }
+
+            return _pageTable.Read<ulong>((va / PageSize) * PteSize) != 0;
+        }
+
+        private bool ValidateAddress(ulong va)
+        {
+            return va < _addressSpaceSize;
+        }
+
+        /// <summary>
+        /// Performs address translation of the address inside a CPU mapped memory range.
+        /// </summary>
+        /// <remarks>
+        /// If the address is invalid or unmapped, -1 will be returned.
+        /// </remarks>
+        /// <param name="va">Virtual address to be translated</param>
+        /// <returns>The physical address</returns>
+        public ulong GetPhysicalAddress(ulong va)
+        {
+            // We return -1L if the virtual address is invalid or unmapped.
+            if (!ValidateAddress(va) || !IsMapped(va))
+            {
+                return ulong.MaxValue;
+            }
+
+            return GetPhysicalAddressInternal(va);
+        }
+
+        private ulong GetPhysicalAddressInternal(ulong va)
+        {
+            return PteToPa(_pageTable.Read<ulong>((va / PageSize) * PteSize) & ~(0xffffUL << 48)) + (va & PageMask);
+        }
+
+        private void MarkRegionAsModified(ulong va, ulong size)
+        {
+            ulong endVa = (va + size + PageMask) & ~(ulong)PageMask;
+
+            while (va < endVa)
+            {
+                ref long pageRef = ref _pageTable.GetRef<long>((va >> PageBits) * PteSize);
+
+                long pte;
+
+                do
+                {
+                    pte = Volatile.Read(ref pageRef);
+
+                    if (pte >= 0)
+                    {
+                        break;
+                    }
+                }
+                while (Interlocked.CompareExchange(ref pageRef, pte & ~(0xffffL << 48), pte) != pte);
+
+                va += PageSize;
+            }
+        }
+
+        private ulong PaToPte(ulong pa)
+        {
+            return (ulong)_backingMemory.GetPointer(pa, PageSize).ToInt64();
+        }
+
+        private ulong PteToPa(ulong pte)
+        {
+            return (ulong)((long)pte - _backingMemory.Pointer.ToInt64());
+        }
+
+        public void Dispose()
+        {
+            _pageTable.Dispose();
+        }
+    }
+}
diff --git a/Ryujinx.Cpu/MemoryNotContiguousException.cs b/Ryujinx.Cpu/MemoryNotContiguousException.cs
new file mode 100644
index 00000000..d7c827cc
--- /dev/null
+++ b/Ryujinx.Cpu/MemoryNotContiguousException.cs
@@ -0,0 +1,19 @@
+using System;
+
+namespace Ryujinx.Cpu
+{
+    class MemoryNotContiguousException : Exception
+    {
+        public MemoryNotContiguousException() : base("The specified memory region is not contiguous.")
+        {
+        }
+
+        public MemoryNotContiguousException(string message) : base(message)
+        {
+        }
+
+        public MemoryNotContiguousException(string message, Exception innerException) : base(message, innerException)
+        {
+        }
+    }
+}
+\ No newline at end of file
diff --git a/Ryujinx.Cpu/Ryujinx.Cpu.csproj b/Ryujinx.Cpu/Ryujinx.Cpu.csproj
new file mode 100644
index 00000000..3fd647eb
--- /dev/null
+++ b/Ryujinx.Cpu/Ryujinx.Cpu.csproj
@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Debug|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <PropertyGroup Condition="'$(Configuration)|$(Platform)'=='Release|AnyCPU'">
+    <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ARMeilleure\ARMeilleure.csproj" />
+    <ProjectReference Include="..\Ryujinx.Memory\Ryujinx.Memory.csproj" />
+  </ItemGroup>
+
+</Project>
author	gdkchan <gab.dark.100@gmail.com>	2020-05-03 19:54:50 -0300
committer	GitHub <noreply@github.com>	2020-05-04 08:54:50 +1000
commit	f77694e4f774c9391aad5344e70a7c8721cfedc6 (patch)
tree	36bccaeb153bd5d921c751966b8a734a7b4b6ae1 /Ryujinx.Cpu
parent	1758424208335d1f4ff7c27c554e517c81bf72f6 (diff)