using ARMeilleure.Common; using Ryujinx.Cpu.LightningJit.Cache; using Ryujinx.Cpu.LightningJit.CodeGen; using Ryujinx.Cpu.LightningJit.CodeGen.Arm64; using Ryujinx.Cpu.LightningJit.State; using System; using System.Collections.Generic; using System.Diagnostics; using System.Runtime.InteropServices; namespace Ryujinx.Cpu.LightningJit { delegate void DispatcherFunction(IntPtr nativeContext, ulong startAddress); /// /// Represents a stub manager. /// class TranslatorStubs : IDisposable { private delegate ulong GetFunctionAddressDelegate(IntPtr framePointer, ulong address); private readonly Lazy _slowDispatchStub; private bool _disposed; private readonly AddressTable _functionTable; private readonly NoWxCache _noWxCache; private readonly GetFunctionAddressDelegate _getFunctionAddressRef; private readonly IntPtr _getFunctionAddress; private readonly Lazy _dispatchStub; private readonly Lazy _dispatchLoop; /// /// Gets the dispatch stub. /// /// instance was disposed public IntPtr DispatchStub { get { ObjectDisposedException.ThrowIf(_disposed, this); return _dispatchStub.Value; } } /// /// Gets the slow dispatch stub. /// /// instance was disposed public IntPtr SlowDispatchStub { get { ObjectDisposedException.ThrowIf(_disposed, this); return _slowDispatchStub.Value; } } /// /// Gets the dispatch loop function. /// /// instance was disposed public DispatcherFunction DispatchLoop { get { ObjectDisposedException.ThrowIf(_disposed, this); return _dispatchLoop.Value; } } /// /// Initializes a new instance of the class with the specified /// instance. /// /// Function table used to store pointers to the functions that the guest code will call /// Cache used on platforms that enforce W^X, otherwise should be null /// is null public TranslatorStubs(AddressTable functionTable, NoWxCache noWxCache) { ArgumentNullException.ThrowIfNull(functionTable); _functionTable = functionTable; _noWxCache = noWxCache; _getFunctionAddressRef = NativeInterface.GetFunctionAddress; _getFunctionAddress = Marshal.GetFunctionPointerForDelegate(_getFunctionAddressRef); _slowDispatchStub = new(GenerateSlowDispatchStub, isThreadSafe: true); _dispatchStub = new(GenerateDispatchStub, isThreadSafe: true); _dispatchLoop = new(GenerateDispatchLoop, isThreadSafe: true); } /// /// Releases all resources used by the instance. /// public void Dispose() { Dispose(true); GC.SuppressFinalize(this); } /// /// Releases all unmanaged and optionally managed resources used by the instance. /// /// to dispose managed resources also; otherwise just unmanaged resouces protected virtual void Dispose(bool disposing) { if (!_disposed) { if (_noWxCache == null) { if (_dispatchStub.IsValueCreated) { JitCache.Unmap(_dispatchStub.Value); } if (_dispatchLoop.IsValueCreated) { JitCache.Unmap(Marshal.GetFunctionPointerForDelegate(_dispatchLoop.Value)); } } _disposed = true; } } /// /// Frees resources used by the instance. /// ~TranslatorStubs() { Dispose(false); } /// /// Generates a . /// /// Generated private IntPtr GenerateDispatchStub() { List branchToFallbackOffsets = new(); CodeWriter writer = new(); if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { Assembler asm = new(writer); RegisterSaveRestore rsr = new((1u << 19) | (1u << 21) | (1u << 22), hasCall: true); rsr.WritePrologue(ref asm); Operand context = Register(19); asm.Mov(context, Register(0)); // Load the target guest address from the native context. Operand guestAddress = Register(16); asm.LdrRiUn(guestAddress, context, NativeContext.GetDispatchAddressOffset()); // Check if guest address is within range of the AddressTable. asm.And(Register(17), guestAddress, Const(~_functionTable.Mask)); branchToFallbackOffsets.Add(writer.InstructionPointer); asm.Cbnz(Register(17), 0); Operand page = Register(17); Operand index = Register(21); Operand mask = Register(22); asm.Mov(page, (ulong)_functionTable.Base); for (int i = 0; i < _functionTable.Levels.Length; i++) { ref var level = ref _functionTable.Levels[i]; asm.Mov(mask, level.Mask >> level.Index); asm.And(index, mask, guestAddress, ArmShiftType.Lsr, level.Index); if (i < _functionTable.Levels.Length - 1) { asm.LdrRr(page, page, index, ArmExtensionType.Uxtx, true); branchToFallbackOffsets.Add(writer.InstructionPointer); asm.Cbz(page, 0); } } asm.LdrRr(page, page, index, ArmExtensionType.Uxtx, true); rsr.WriteEpilogue(ref asm); asm.Br(page); foreach (int branchOffset in branchToFallbackOffsets) { uint branchInst = writer.ReadInstructionAt(branchOffset); Debug.Assert(writer.InstructionPointer > branchOffset); writer.WriteInstructionAt(branchOffset, branchInst | ((uint)(writer.InstructionPointer - branchOffset) << 5)); } // Fallback. asm.Mov(Register(0), Register(29)); asm.Mov(Register(1), guestAddress); asm.Mov(Register(16), (ulong)_getFunctionAddress); asm.Blr(Register(16)); asm.Mov(Register(16), Register(0)); asm.Mov(Register(0), Register(19)); rsr.WriteEpilogue(ref asm); asm.Br(Register(16)); } else { throw new PlatformNotSupportedException(); } return Map(writer.AsByteSpan()); } /// /// Generates a . /// /// Generated private IntPtr GenerateSlowDispatchStub() { CodeWriter writer = new(); if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { Assembler asm = new(writer); RegisterSaveRestore rsr = new(1u << 19, hasCall: true); rsr.WritePrologue(ref asm); Operand context = Register(19); asm.Mov(context, Register(0)); // Load the target guest address from the native context. asm.Mov(Register(0), Register(29)); asm.LdrRiUn(Register(1), context, NativeContext.GetDispatchAddressOffset()); asm.Mov(Register(16), (ulong)_getFunctionAddress); asm.Blr(Register(16)); asm.Mov(Register(16), Register(0)); asm.Mov(Register(0), Register(19)); rsr.WriteEpilogue(ref asm); asm.Br(Register(16)); } else { throw new PlatformNotSupportedException(); } return Map(writer.AsByteSpan()); } /// /// Emits code that syncs FP state before executing guest code, or returns it to normal. /// /// Assembler /// Pointer to the native context /// First temporary register /// Second temporary register /// True if entering guest code, false otherwise private static void EmitSyncFpContext(ref Assembler asm, Operand context, Operand tempRegister, Operand tempRegister2, bool enter) { if (enter) { EmitSwapFpFlags(ref asm, context, tempRegister, tempRegister2, NativeContext.GetFpFlagsOffset(), NativeContext.GetHostFpFlagsOffset()); } else { EmitSwapFpFlags(ref asm, context, tempRegister, tempRegister2, NativeContext.GetHostFpFlagsOffset(), NativeContext.GetFpFlagsOffset()); } } /// /// Swaps the FPCR and FPSR values with values stored in the native context. /// /// Assembler /// Pointer to the native context /// First temporary register /// Second temporary register /// Offset of the new flags that will be loaded /// Offset where the current flags should be saved private static void EmitSwapFpFlags(ref Assembler asm, Operand context, Operand tempRegister, Operand tempRegister2, int loadOffset, int storeOffset) { asm.MrsFpcr(tempRegister); asm.MrsFpsr(tempRegister2); asm.Orr(tempRegister, tempRegister, tempRegister2); asm.StrRiUn(tempRegister, context, storeOffset); asm.LdrRiUn(tempRegister, context, loadOffset); asm.MsrFpcr(tempRegister); asm.MsrFpsr(tempRegister2); } /// /// Generates a function. /// /// function private DispatcherFunction GenerateDispatchLoop() { CodeWriter writer = new(); if (RuntimeInformation.ProcessArchitecture == Architecture.Arm64) { Assembler asm = new(writer); RegisterSaveRestore rsr = new(1u << 19, hasCall: true); rsr.WritePrologue(ref asm); Operand context = Register(19); asm.Mov(context, Register(0)); EmitSyncFpContext(ref asm, context, Register(16, OperandType.I32), Register(17, OperandType.I32), true); // Load the target guest address from the native context. Operand guestAddress = Register(16); asm.Mov(guestAddress, Register(1)); int loopStartIndex = writer.InstructionPointer; asm.StrRiUn(guestAddress, context, NativeContext.GetDispatchAddressOffset()); asm.Mov(Register(0), context); asm.Mov(Register(17), (ulong)DispatchStub); asm.Blr(Register(17)); asm.Mov(guestAddress, Register(0)); asm.Cbz(guestAddress, 16); asm.LdrRiUn(Register(17), context, NativeContext.GetRunningOffset()); asm.Cbz(Register(17), 8); asm.B((loopStartIndex - writer.InstructionPointer) * 4); EmitSyncFpContext(ref asm, context, Register(16, OperandType.I32), Register(17, OperandType.I32), false); rsr.WriteEpilogue(ref asm); asm.Ret(); } else { throw new PlatformNotSupportedException(); } IntPtr pointer = Map(writer.AsByteSpan()); return Marshal.GetDelegateForFunctionPointer(pointer); } private IntPtr Map(ReadOnlySpan code) { if (_noWxCache != null) { return _noWxCache.MapPageAligned(code); } else { return JitCache.Map(code); } } private static Operand Register(int register, OperandType type = OperandType.I64) { return new Operand(register, RegisterType.Integer, type); } private static Operand Const(ulong value) { return new(OperandKind.Constant, OperandType.I64, value); } } }