using ARMeilleure.Common; using Ryujinx.Cpu.LightningJit.Cache; using Ryujinx.Cpu.LightningJit.CodeGen; using Ryujinx.Cpu.LightningJit.CodeGen.Arm64; using Ryujinx.Cpu.LightningJit.State; using System; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; namespace Ryujinx.Cpu.LightningJit { delegate void DispatcherFunction(IntPtr nativeContext, ulong startAddress); /// <summary> /// Represents a stub manager. /// </summary> class TranslatorStubs : IDisposable { private delegate ulong GetFunctionAddressDelegate(IntPtr framePointer, ulong address); private readonly Lazy<IntPtr> _slowDispatchStub; private bool _disposed; private readonly AddressTable<ulong> _functionTable; private readonly NoWxCache _noWxCache; private readonly GetFunctionAddressDelegate _getFunctionAddressRef; private readonly IntPtr _getFunctionAddress; private readonly Lazy<IntPtr> _dispatchStub; private readonly Lazy<DispatcherFunction> _dispatchLoop; /// <summary> /// Gets the dispatch stub. /// </summary> /// <exception cref="ObjectDisposedException"><see cref="TranslatorStubs"/> instance was disposed</exception> public IntPtr DispatchStub { get { ObjectDisposedException.ThrowIf(_disposed, this); return _dispatchStub.Value; } } /// <summary> /// Gets the slow dispatch stub. /// </summary> /// <exception cref="ObjectDisposedException"><see cref="TranslatorStubs"/> instance was disposed</exception> public IntPtr SlowDispatchStub { get { ObjectDisposedException.ThrowIf(_disposed, this); return _slowDispatchStub.Value; } } /// <summary> /// Gets the dispatch loop function. /// </summary> /// <exception cref="ObjectDisposedException"><see cref="TranslatorStubs"/> instance was disposed</exception> public DispatcherFunction DispatchLoop { get { ObjectDisposedException.ThrowIf(_disposed, this); return _dispatchLoop.Value; } } /// <summary> /// Initializes a new instance of the <see cref="TranslatorStubs"/> class with the specified /// <see cref="Translator"/> instance. /// </summary> /// <param name="functionTable">Function table used to store pointers to the functions that the guest code will call</param> /// <param name="noWxCache">Cache used on platforms that enforce W^X, otherwise should be null</param> /// <exception cref="ArgumentNullException"><paramref name="translator"/> is null</exception> public TranslatorStubs(AddressTable<ulong> functionTable, NoWxCache noWxCache) { ArgumentNullException.ThrowIfNull(functionTable); _functionTable = functionTable; _noWxCache = noWxCache; _getFunctionAddressRef = NativeInterface.GetFunctionAddress; _getFunctionAddress = Marshal.GetFunctionPointerForDelegate(_getFunctionAddressRef); _slowDispatchStub = new(GenerateSlowDispatchStub, isThreadSafe: true); _dispatchStub = new(GenerateDispatchStub, isThreadSafe: true); _dispatchLoop = new(GenerateDispatchLoop, isThreadSafe: true); } /// <summary> /// Releases all resources used by the <see cref="TranslatorStubs"/> instance. /// </summary> public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } /// <summary> /// Releases all unmanaged and optionally managed resources used by the <see cref="TranslatorStubs"/> instance. /// </summary> /// <param name="disposing"><see langword="true"/> to dispose managed resources also; otherwise just unmanaged resouces</param> protected virtual void Dispose(bool disposing) { if (!_disposed) { if (_noWxCache == null) { if (_dispatchStub.IsValueCreated) { JitCache.Unmap(_dispatchStub.Value); } if (_dispatchLoop.IsValueCreated) { JitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value)); } } _disposed = true; } } /// <summary> /// Frees resources used by the <see cref="TranslatorStubs"/> instance. /// </summary> ~TranslatorStubs() { Dispose(false); } /// <summary> /// Generates a <see cref="DispatchStub"/>. /// </summary> /// <returns>Generated <see cref="DispatchStub"/></returns> private IntPtr GenerateDispatchStub() { List<int> branchToFallbackOffsets = new(); CodeWriter writer = new(); if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { Assembler asm = new(writer); RegisterSaveRestore rsr = new((1u << 19) | (1u << 21) | (1u << 22), hasCall: true); rsr.WritePrologue(ref asm); Operand context = Register(19); asm.Mov(context, Register(0)); // Load the target guest address from the native context. Operand guestAddress = Register(16); asm.LdrRiUn(guestAddress, context, NativeContext.GetDispatchAddressOffset()); // Check if guest address is within range of the AddressTable. asm.And(Register(17), guestAddress, Const(~_functionTable.Mask)); branchToFallbackOffsets.Add(writer.InstructionPointer); asm.Cbnz(Register(17), 0); Operand page = Register(17); Operand index = Register(21); Operand mask = Register(22); asm.Mov(page, (ulong)_functionTable.Base); for (int i = 0; i < _functionTable.Levels.Length; i++) { ref var level = ref _functionTable.Levels[i]; asm.Mov(mask, level.Mask >> level.Index); asm.And(index, mask, guestAddress, ArmShiftType.Lsr, level.Index); if (i < _functionTable.Levels.Length - 1) { asm.LdrRr(page, page, index, ArmExtensionType.Uxtx, true); branchToFallbackOffsets.Add(writer.InstructionPointer); asm.Cbz(page, 0); } } asm.LdrRr(page, page, index, ArmExtensionType.Uxtx, true); rsr.WriteEpilogue(ref asm); asm.Br(page); foreach (int branchOffset in branchToFallbackOffsets) { uint branchInst = writer.ReadInstructionAt(branchOffset); Debug.Assert(writer.InstructionPointer > branchOffset); writer.WriteInstructionAt(branchOffset, branchInst | ((uint)(writer.InstructionPointer - branchOffset) << 5)); } // Fallback. asm.Mov(Register(0), Register(29)); asm.Mov(Register(1), guestAddress); asm.Mov(Register(16), (ulong)_getFunctionAddress); asm.Blr(Register(16)); asm.Mov(Register(16), Register(0)); asm.Mov(Register(0), Register(19)); rsr.WriteEpilogue(ref asm); asm.Br(Register(16)); } else { throw new PlatformNotSupportedException(); } return Map(writer.AsByteSpan()); } /// <summary> /// Generates a <see cref="SlowDispatchStub"/>. /// </summary> /// <returns>Generated <see cref="SlowDispatchStub"/></returns> private IntPtr GenerateSlowDispatchStub() { CodeWriter writer = new(); if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { Assembler asm = new(writer); RegisterSaveRestore rsr = new(1u << 19, hasCall: true); rsr.WritePrologue(ref asm); Operand context = Register(19); asm.Mov(context, Register(0)); // Load the target guest address from the native context. asm.Mov(Register(0), Register(29)); asm.LdrRiUn(Register(1), context, NativeContext.GetDispatchAddressOffset()); asm.Mov(Register(16), (ulong)_getFunctionAddress); asm.Blr(Register(16)); asm.Mov(Register(16), Register(0)); asm.Mov(Register(0), Register(19)); rsr.WriteEpilogue(ref asm); asm.Br(Register(16)); } else { throw new PlatformNotSupportedException(); } return Map(writer.AsByteSpan()); } /// <summary> /// Emits code that syncs FP state before executing guest code, or returns it to normal. /// </summary> /// <param name="asm">Assembler</param> /// <param name="context">Pointer to the native context</param> /// <param name="tempRegister">First temporary register</param> /// <param name="tempRegister2">Second temporary register</param> /// <param name="enter">True if entering guest code, false otherwise</param> private static void EmitSyncFpContext(ref Assembler asm, Operand context, Operand tempRegister, Operand tempRegister2, bool enter) { if (enter) { EmitSwapFpFlags(ref asm, context, tempRegister, tempRegister2, NativeContext.GetFpFlagsOffset(), NativeContext.GetHostFpFlagsOffset()); } else { EmitSwapFpFlags(ref asm, context, tempRegister, tempRegister2, NativeContext.GetHostFpFlagsOffset(), NativeContext.GetFpFlagsOffset()); } } /// <summary> /// Swaps the FPCR and FPSR values with values stored in the native context. /// </summary> /// <param name="asm">Assembler</param> /// <param name="context">Pointer to the native context</param> /// <param name="tempRegister">First temporary register</param> /// <param name="tempRegister2">Second temporary register</param> /// <param name="loadOffset">Offset of the new flags that will be loaded</param> /// <param name="storeOffset">Offset where the current flags should be saved</param> private static void EmitSwapFpFlags(ref Assembler asm, Operand context, Operand tempRegister, Operand tempRegister2, int loadOffset, int storeOffset) { asm.MrsFpcr(tempRegister); asm.MrsFpsr(tempRegister2); asm.Orr(tempRegister, tempRegister, tempRegister2); asm.StrRiUn(tempRegister, context, storeOffset); asm.LdrRiUn(tempRegister, context, loadOffset); asm.MsrFpcr(tempRegister); asm.MsrFpsr(tempRegister2); } /// <summary> /// Generates a <see cref="DispatchLoop"/> function. /// </summary> /// <returns><see cref="DispatchLoop"/> function</returns> private DispatcherFunction GenerateDispatchLoop() { CodeWriter writer = new(); if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { Assembler asm = new(writer); RegisterSaveRestore rsr = new(1u << 19, hasCall: true); rsr.WritePrologue(ref asm); Operand context = Register(19); asm.Mov(context, Register(0)); EmitSyncFpContext(ref asm, context, Register(16, OperandType.I32), Register(17, OperandType.I32), true); // Load the target guest address from the native context. Operand guestAddress = Register(16); asm.Mov(guestAddress, Register(1)); int loopStartIndex = writer.InstructionPointer; asm.StrRiUn(guestAddress, context, NativeContext.GetDispatchAddressOffset()); asm.Mov(Register(0), context); asm.Mov(Register(17), (ulong)DispatchStub); asm.Blr(Register(17)); asm.Mov(guestAddress, Register(0)); asm.Cbz(guestAddress, 16); asm.LdrRiUn(Register(17), context, NativeContext.GetRunningOffset()); asm.Cbz(Register(17), 8); asm.B((loopStartIndex - writer.InstructionPointer) * 4); EmitSyncFpContext(ref asm, context, Register(16, OperandType.I32), Register(17, OperandType.I32), false); rsr.WriteEpilogue(ref asm); asm.Ret(); } else { throw new PlatformNotSupportedException(); } IntPtr pointer = Map(writer.AsByteSpan()); return Marshal.GetDelegateForFunctionPointer<DispatcherFunction>(pointer); } private IntPtr Map(ReadOnlySpan<byte> code) { if (_noWxCache != null) { return _noWxCache.MapPageAligned(code); } else { return JitCache.Map(code); } } private static Operand Register(int register, OperandType type = OperandType.I64) { return new Operand(register, RegisterType.Integer, type); } private static Operand Const(ulong value) { return new(OperandKind.Constant, OperandType.I64, value); } } }