1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
|
using Ryujinx.Cpu.Signal;
using Ryujinx.Memory;
using System;
using System.Diagnostics;
using System.Numerics;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.Jit.HostTracked
{
sealed class NativePageTable : IDisposable
{
private delegate ulong TrackingEventDelegate(ulong address, ulong size, bool write);
private const int PageBits = 12;
private const int PageSize = 1 << PageBits;
private const int PageMask = PageSize - 1;
private const int PteSize = 8;
private readonly int _bitsPerPtPage;
private readonly int _entriesPerPtPage;
private readonly int _pageCommitmentBits;
private readonly PageTable<ulong> _pageTable;
private readonly MemoryBlock _nativePageTable;
private readonly ulong[] _pageCommitmentBitmap;
private readonly ulong _hostPageSize;
private readonly TrackingEventDelegate _trackingEvent;
private bool _disposed;
public IntPtr PageTablePointer => _nativePageTable.Pointer;
public NativePageTable(ulong asSize)
{
ulong hostPageSize = MemoryBlock.GetPageSize();
_entriesPerPtPage = (int)(hostPageSize / sizeof(ulong));
_bitsPerPtPage = BitOperations.Log2((uint)_entriesPerPtPage);
_pageCommitmentBits = PageBits + _bitsPerPtPage;
_hostPageSize = hostPageSize;
_pageTable = new PageTable<ulong>();
_nativePageTable = new MemoryBlock((asSize / PageSize) * PteSize + _hostPageSize, MemoryAllocationFlags.Reserve);
_pageCommitmentBitmap = new ulong[(asSize >> _pageCommitmentBits) / (sizeof(ulong) * 8)];
ulong ptStart = (ulong)_nativePageTable.Pointer;
ulong ptEnd = ptStart + _nativePageTable.Size;
_trackingEvent = VirtualMemoryEvent;
bool added = NativeSignalHandler.AddTrackedRegion((nuint)ptStart, (nuint)ptEnd, Marshal.GetFunctionPointerForDelegate(_trackingEvent));
if (!added)
{
throw new InvalidOperationException("Number of allowed tracked regions exceeded.");
}
}
public void Map(ulong va, ulong pa, ulong size, AddressSpacePartitioned addressSpace, MemoryBlock backingMemory, bool privateMap)
{
while (size != 0)
{
_pageTable.Map(va, pa);
EnsureCommitment(va);
if (privateMap)
{
_nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, addressSpace.GetPointer(va, PageSize)));
}
else
{
_nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, backingMemory.GetPointer(pa, PageSize)));
}
va += PageSize;
pa += PageSize;
size -= PageSize;
}
}
public void Unmap(ulong va, ulong size)
{
IntPtr guardPagePtr = GetGuardPagePointer();
while (size != 0)
{
_pageTable.Unmap(va);
_nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, guardPagePtr));
va += PageSize;
size -= PageSize;
}
}
public ulong Read(ulong va)
{
ulong pte = _nativePageTable.Read<ulong>((va / PageSize) * PteSize);
pte += va & ~(ulong)PageMask;
return pte + (va & PageMask);
}
public void Update(ulong va, IntPtr ptr, ulong size)
{
ulong remainingSize = size;
while (remainingSize != 0)
{
EnsureCommitment(va);
_nativePageTable.Write((va / PageSize) * PteSize, GetPte(va, ptr));
va += PageSize;
ptr += PageSize;
remainingSize -= PageSize;
}
}
private void EnsureCommitment(ulong va)
{
ulong bit = va >> _pageCommitmentBits;
int index = (int)(bit / (sizeof(ulong) * 8));
int shift = (int)(bit % (sizeof(ulong) * 8));
ulong mask = 1UL << shift;
ulong oldMask = _pageCommitmentBitmap[index];
if ((oldMask & mask) == 0)
{
lock (_pageCommitmentBitmap)
{
oldMask = _pageCommitmentBitmap[index];
if ((oldMask & mask) != 0)
{
return;
}
_nativePageTable.Commit(bit * _hostPageSize, _hostPageSize);
Span<ulong> pageSpan = MemoryMarshal.Cast<byte, ulong>(_nativePageTable.GetSpan(bit * _hostPageSize, (int)_hostPageSize));
Debug.Assert(pageSpan.Length == _entriesPerPtPage);
IntPtr guardPagePtr = GetGuardPagePointer();
for (int i = 0; i < pageSpan.Length; i++)
{
pageSpan[i] = GetPte((bit << _pageCommitmentBits) | ((ulong)i * PageSize), guardPagePtr);
}
_pageCommitmentBitmap[index] = oldMask | mask;
}
}
}
private IntPtr GetGuardPagePointer()
{
return _nativePageTable.GetPointer(_nativePageTable.Size - _hostPageSize, _hostPageSize);
}
private static ulong GetPte(ulong va, IntPtr ptr)
{
Debug.Assert((va & PageMask) == 0);
return (ulong)ptr - va;
}
public ulong GetPhysicalAddress(ulong va)
{
return _pageTable.Read(va) + (va & PageMask);
}
private ulong VirtualMemoryEvent(ulong address, ulong size, bool write)
{
if (address < _nativePageTable.Size - _hostPageSize)
{
// Some prefetch instructions do not cause faults with invalid addresses.
// Retry if we are hitting a case where the page table is unmapped, the next
// run will execute the actual instruction.
// The address loaded from the page table will be invalid, and it should hit the else case
// if the instruction faults on unmapped or protected memory.
ulong va = address * (PageSize / sizeof(ulong));
EnsureCommitment(va);
return (ulong)_nativePageTable.Pointer + address;
}
else
{
throw new InvalidMemoryRegionException();
}
}
private void Dispose(bool disposing)
{
if (!_disposed)
{
if (disposing)
{
NativeSignalHandler.RemoveTrackedRegion((nuint)_nativePageTable.Pointer);
_nativePageTable.Dispose();
}
_disposed = true;
}
}
public void Dispose()
{
Dispose(disposing: true);
GC.SuppressFinalize(this);
}
}
}
|
