using ARMeilleure.Common;
using Ryujinx.Cpu.LightningJit.Cache;
using Ryujinx.Cpu.LightningJit.CodeGen;
using Ryujinx.Cpu.LightningJit.CodeGen.Arm64;
using Ryujinx.Cpu.LightningJit.State;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Runtime.InteropServices;
namespace Ryujinx.Cpu.LightningJit
{
delegate void DispatcherFunction(IntPtr nativeContext, ulong startAddress);
///
/// Represents a stub manager.
///
class TranslatorStubs : IDisposable
{
private delegate ulong GetFunctionAddressDelegate(IntPtr framePointer, ulong address);
private readonly Lazy _slowDispatchStub;
private bool _disposed;
private readonly AddressTable _functionTable;
private readonly NoWxCache _noWxCache;
private readonly GetFunctionAddressDelegate _getFunctionAddressRef;
private readonly IntPtr _getFunctionAddress;
private readonly Lazy _dispatchStub;
private readonly Lazy _dispatchLoop;
///
/// Gets the dispatch stub.
///
/// instance was disposed
public IntPtr DispatchStub
{
get
{
ObjectDisposedException.ThrowIf(_disposed, this);
return _dispatchStub.Value;
}
}
///
/// Gets the slow dispatch stub.
///
/// instance was disposed
public IntPtr SlowDispatchStub
{
get
{
ObjectDisposedException.ThrowIf(_disposed, this);
return _slowDispatchStub.Value;
}
}
///
/// Gets the dispatch loop function.
///
/// instance was disposed
public DispatcherFunction DispatchLoop
{
get
{
ObjectDisposedException.ThrowIf(_disposed, this);
return _dispatchLoop.Value;
}
}
///
/// Initializes a new instance of the class with the specified
/// instance.
///
/// Function table used to store pointers to the functions that the guest code will call
/// Cache used on platforms that enforce W^X, otherwise should be null
/// is null
public TranslatorStubs(AddressTable functionTable, NoWxCache noWxCache)
{
ArgumentNullException.ThrowIfNull(functionTable);
_functionTable = functionTable;
_noWxCache = noWxCache;
_getFunctionAddressRef = NativeInterface.GetFunctionAddress;
_getFunctionAddress = Marshal.GetFunctionPointerForDelegate(_getFunctionAddressRef);
_slowDispatchStub = new(GenerateSlowDispatchStub, isThreadSafe: true);
_dispatchStub = new(GenerateDispatchStub, isThreadSafe: true);
_dispatchLoop = new(GenerateDispatchLoop, isThreadSafe: true);
}
///
/// Releases all resources used by the instance.
///
public void Dispose()
{
Dispose(true);
GC.SuppressFinalize(this);
}
///
/// Releases all unmanaged and optionally managed resources used by the instance.
///
/// to dispose managed resources also; otherwise just unmanaged resouces
protected virtual void Dispose(bool disposing)
{
if (!_disposed)
{
if (_noWxCache == null)
{
if (_dispatchStub.IsValueCreated)
{
JitCache.Unmap(_dispatchStub.Value);
}
if (_dispatchLoop.IsValueCreated)
{
JitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value));
}
}
_disposed = true;
}
}
///
/// Frees resources used by the instance.
///
~TranslatorStubs()
{
Dispose(false);
}
///
/// Generates a .
///
/// Generated
private IntPtr GenerateDispatchStub()
{
List branchToFallbackOffsets = new();
CodeWriter writer = new();
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
Assembler asm = new(writer);
RegisterSaveRestore rsr = new((1u << 19) | (1u << 21) | (1u << 22), hasCall: true);
rsr.WritePrologue(ref asm);
Operand context = Register(19);
asm.Mov(context, Register(0));
// Load the target guest address from the native context.
Operand guestAddress = Register(16);
asm.LdrRiUn(guestAddress, context, NativeContext.GetDispatchAddressOffset());
// Check if guest address is within range of the AddressTable.
asm.And(Register(17), guestAddress, Const(~_functionTable.Mask));
branchToFallbackOffsets.Add(writer.InstructionPointer);
asm.Cbnz(Register(17), 0);
Operand page = Register(17);
Operand index = Register(21);
Operand mask = Register(22);
asm.Mov(page, (ulong)_functionTable.Base);
for (int i = 0; i < _functionTable.Levels.Length; i++)
{
ref var level = ref _functionTable.Levels[i];
asm.Mov(mask, level.Mask >> level.Index);
asm.And(index, mask, guestAddress, ArmShiftType.Lsr, level.Index);
if (i < _functionTable.Levels.Length - 1)
{
asm.LdrRr(page, page, index, ArmExtensionType.Uxtx, true);
branchToFallbackOffsets.Add(writer.InstructionPointer);
asm.Cbz(page, 0);
}
}
asm.LdrRr(page, page, index, ArmExtensionType.Uxtx, true);
rsr.WriteEpilogue(ref asm);
asm.Br(page);
foreach (int branchOffset in branchToFallbackOffsets)
{
uint branchInst = writer.ReadInstructionAt(branchOffset);
Debug.Assert(writer.InstructionPointer > branchOffset);
writer.WriteInstructionAt(branchOffset, branchInst | ((uint)(writer.InstructionPointer - branchOffset) << 5));
}
// Fallback.
asm.Mov(Register(0), Register(29));
asm.Mov(Register(1), guestAddress);
asm.Mov(Register(16), (ulong)_getFunctionAddress);
asm.Blr(Register(16));
asm.Mov(Register(16), Register(0));
asm.Mov(Register(0), Register(19));
rsr.WriteEpilogue(ref asm);
asm.Br(Register(16));
}
else
{
throw new PlatformNotSupportedException();
}
return Map(writer.AsByteSpan());
}
///
/// Generates a .
///
/// Generated
private IntPtr GenerateSlowDispatchStub()
{
CodeWriter writer = new();
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
Assembler asm = new(writer);
RegisterSaveRestore rsr = new(1u << 19, hasCall: true);
rsr.WritePrologue(ref asm);
Operand context = Register(19);
asm.Mov(context, Register(0));
// Load the target guest address from the native context.
asm.Mov(Register(0), Register(29));
asm.LdrRiUn(Register(1), context, NativeContext.GetDispatchAddressOffset());
asm.Mov(Register(16), (ulong)_getFunctionAddress);
asm.Blr(Register(16));
asm.Mov(Register(16), Register(0));
asm.Mov(Register(0), Register(19));
rsr.WriteEpilogue(ref asm);
asm.Br(Register(16));
}
else
{
throw new PlatformNotSupportedException();
}
return Map(writer.AsByteSpan());
}
///
/// Emits code that syncs FP state before executing guest code, or returns it to normal.
///
/// Assembler
/// Pointer to the native context
/// First temporary register
/// Second temporary register
/// True if entering guest code, false otherwise
private static void EmitSyncFpContext(ref Assembler asm, Operand context, Operand tempRegister, Operand tempRegister2, bool enter)
{
if (enter)
{
EmitSwapFpFlags(ref asm, context, tempRegister, tempRegister2, NativeContext.GetFpFlagsOffset(), NativeContext.GetHostFpFlagsOffset());
}
else
{
EmitSwapFpFlags(ref asm, context, tempRegister, tempRegister2, NativeContext.GetHostFpFlagsOffset(), NativeContext.GetFpFlagsOffset());
}
}
///
/// Swaps the FPCR and FPSR values with values stored in the native context.
///
/// Assembler
/// Pointer to the native context
/// First temporary register
/// Second temporary register
/// Offset of the new flags that will be loaded
/// Offset where the current flags should be saved
private static void EmitSwapFpFlags(ref Assembler asm, Operand context, Operand tempRegister, Operand tempRegister2, int loadOffset, int storeOffset)
{
asm.MrsFpcr(tempRegister);
asm.MrsFpsr(tempRegister2);
asm.Orr(tempRegister, tempRegister, tempRegister2);
asm.StrRiUn(tempRegister, context, storeOffset);
asm.LdrRiUn(tempRegister, context, loadOffset);
asm.MsrFpcr(tempRegister);
asm.MsrFpsr(tempRegister2);
}
///
/// Generates a function.
///
/// function
private DispatcherFunction GenerateDispatchLoop()
{
CodeWriter writer = new();
if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
{
Assembler asm = new(writer);
RegisterSaveRestore rsr = new(1u << 19, hasCall: true);
rsr.WritePrologue(ref asm);
Operand context = Register(19);
asm.Mov(context, Register(0));
EmitSyncFpContext(ref asm, context, Register(16, OperandType.I32), Register(17, OperandType.I32), true);
// Load the target guest address from the native context.
Operand guestAddress = Register(16);
asm.Mov(guestAddress, Register(1));
int loopStartIndex = writer.InstructionPointer;
asm.StrRiUn(guestAddress, context, NativeContext.GetDispatchAddressOffset());
asm.Mov(Register(0), context);
asm.Mov(Register(17), (ulong)DispatchStub);
asm.Blr(Register(17));
asm.Mov(guestAddress, Register(0));
asm.Cbz(guestAddress, 16);
asm.LdrRiUn(Register(17), context, NativeContext.GetRunningOffset());
asm.Cbz(Register(17), 8);
asm.B((loopStartIndex - writer.InstructionPointer) * 4);
EmitSyncFpContext(ref asm, context, Register(16, OperandType.I32), Register(17, OperandType.I32), false);
rsr.WriteEpilogue(ref asm);
asm.Ret();
}
else
{
throw new PlatformNotSupportedException();
}
IntPtr pointer = Map(writer.AsByteSpan());
return Marshal.GetDelegateForFunctionPointer(pointer);
}
private IntPtr Map(ReadOnlySpan code)
{
if (_noWxCache != null)
{
return _noWxCache.MapPageAligned(code);
}
else
{
return JitCache.Map(code);
}
}
private static Operand Register(int register, OperandType type = OperandType.I64)
{
return new Operand(register, RegisterType.Integer, type);
}
private static Operand Const(ulong value)
{
return new(OperandKind.Constant, OperandType.I64, value);
}
}
}