using Ryujinx.Cpu.LightningJit.CodeGen; using Ryujinx.Cpu.LightningJit.CodeGen.Arm64; using System; using System.Diagnostics; using System.Numerics; using System.Runtime.InteropServices; namespace Ryujinx.Cpu.LightningJit.Arm32.Target.Arm64 { static class InstEmitSystem { private delegate void SoftwareInterruptHandler(ulong address, int imm); private delegate ulong Get64(); private delegate bool GetBool(); private const int SpIndex = 31; public static void Bkpt(CodeGenContext context, uint imm) { context.AddPendingBkpt(imm); context.Arm64Assembler.B(0); } public static void Cps(CodeGenContext context, uint imod, uint m, uint a, uint i, uint f, uint mode) { // NOP in user mode. } public static void Dbg(CodeGenContext context, uint option) { // NOP in ARMv8. } public static void Hlt(CodeGenContext context, uint imm) { } public static void Mcr(CodeGenContext context, uint encoding, uint coproc, uint opc1, uint rt, uint crn, uint crm, uint opc2) { if (coproc != 15 || opc1 != 0) { Udf(context, encoding, 0); return; } Operand ctx = Register(context.RegisterAllocator.FixedContextRegister); Operand rtOperand = InstEmitCommon.GetInputGpr(context, rt); switch (crn) { case 13: // Process and Thread Info. if (crm == 0) { switch (opc2) { case 2: context.Arm64Assembler.StrRiUn(rtOperand, ctx, NativeContextOffsets.TpidrEl0Offset); return; } } break; } } public static void Mcrr(CodeGenContext context, uint encoding, uint coproc, uint opc1, uint rt, uint crm) { if (coproc != 15 || opc1 != 0) { Udf(context, encoding, 0); return; } // We don't have any system register that needs to be modified using a 64-bit value. } public static void Mrc(CodeGenContext context, uint encoding, uint coproc, uint opc1, uint rt, uint crn, uint crm, uint opc2) { if (coproc != 15 || opc1 != 0) { Udf(context, encoding, 0); return; } Operand ctx = Register(context.RegisterAllocator.FixedContextRegister); Operand rtOperand = InstEmitCommon.GetInputGpr(context, rt); bool hasValue = false; using ScopedRegister tempRegister = context.RegisterAllocator.AllocateTempGprRegisterScoped(); Operand dest = rt == RegisterUtils.PcRegister ? tempRegister.Operand : rtOperand; switch (crn) { case 13: // Process and Thread Info. if (crm == 0) { switch (opc2) { case 2: context.Arm64Assembler.LdrRiUn(dest, ctx, NativeContextOffsets.TpidrEl0Offset); hasValue = true; break; case 3: context.Arm64Assembler.LdrRiUn(dest, ctx, NativeContextOffsets.TpidrroEl0Offset); hasValue = true; break; } } break; } if (rt == RegisterUtils.PcRegister) { context.Arm64Assembler.MsrNzcv(dest); context.SetNzcvModified(); } else if (!hasValue) { context.Arm64Assembler.Mov(dest, 0u); } } public static void Mrrc(CodeGenContext context, uint encoding, uint coproc, uint opc1, uint rt, uint rt2, uint crm) { if (coproc != 15) { Udf(context, encoding, 0); return; } switch (crm) { case 14: switch (opc1) { case 0: context.AddPendingReadCntpct(rt, rt2); context.Arm64Assembler.B(0); return; } break; } // Unsupported system register. context.Arm64Assembler.Mov(InstEmitCommon.GetOutputGpr(context, rt), 0u); context.Arm64Assembler.Mov(InstEmitCommon.GetOutputGpr(context, rt2), 0u); } public static void Mrs(CodeGenContext context, uint rd, bool r) { Operand rdOperand = InstEmitCommon.GetOutputGpr(context, rd); if (r) { // Reads SPSR, unpredictable in user mode. context.Arm64Assembler.Mov(rdOperand, 0u); } else { Operand ctx = Register(context.RegisterAllocator.FixedContextRegister); using ScopedRegister tempRegister = context.RegisterAllocator.AllocateTempGprRegisterScoped(); context.Arm64Assembler.LdrRiUn(tempRegister.Operand, ctx, NativeContextOffsets.FlagsBaseOffset); // Copy GE flags to destination register. context.Arm64Assembler.Ubfx(rdOperand, tempRegister.Operand, 16, 4); // Insert Q flag. context.Arm64Assembler.And(tempRegister.Operand, tempRegister.Operand, InstEmitCommon.Const(1 << 27)); context.Arm64Assembler.Orr(rdOperand, rdOperand, tempRegister.Operand); // Insert NZCV flags. context.Arm64Assembler.MrsNzcv(tempRegister.Operand); context.Arm64Assembler.Orr(rdOperand, rdOperand, tempRegister.Operand); // All other flags can't be accessed in user mode or have "unknown" values. } } public static void MrsBr(CodeGenContext context, uint rd, uint m1, bool r) { Operand rdOperand = InstEmitCommon.GetOutputGpr(context, rd); // Reads banked register, unpredictable in user mode. context.Arm64Assembler.Mov(rdOperand, 0u); } public static void MsrBr(CodeGenContext context, uint rn, uint m1, bool r) { // Writes banked register, unpredictable in user mode. } public static void MsrI(CodeGenContext context, uint imm, uint mask, bool r) { if (r) { // Writes SPSR, unpredictable in user mode. } else { Operand ctx = Register(context.RegisterAllocator.FixedContextRegister); using ScopedRegister tempRegister = context.RegisterAllocator.AllocateTempGprRegisterScoped(); using ScopedRegister tempRegister2 = context.RegisterAllocator.AllocateTempGprRegisterScoped(); context.Arm64Assembler.LdrRiUn(tempRegister.Operand, ctx, NativeContextOffsets.FlagsBaseOffset); if ((mask & 2) != 0) { // Endian flag. context.Arm64Assembler.Mov(tempRegister2.Operand, (imm >> 9) & 1); context.Arm64Assembler.Bfi(tempRegister.Operand, tempRegister2.Operand, 9, 1); } if ((mask & 4) != 0) { // GE flags. context.Arm64Assembler.Mov(tempRegister2.Operand, (imm >> 16) & 0xf); context.Arm64Assembler.Bfi(tempRegister.Operand, tempRegister2.Operand, 16, 4); } if ((mask & 8) != 0) { // NZCVQ flags. context.Arm64Assembler.Mov(tempRegister2.Operand, (imm >> 27) & 0x1f); context.Arm64Assembler.Bfi(tempRegister.Operand, tempRegister2.Operand, 27, 5); context.Arm64Assembler.Mov(tempRegister2.Operand, (imm >> 28) & 0xf); InstEmitCommon.RestoreNzcvFlags(context, tempRegister2.Operand); context.SetNzcvModified(); } } } public static void MsrR(CodeGenContext context, uint rn, uint mask, bool r) { Operand rnOperand = InstEmitCommon.GetInputGpr(context, rn); if (r) { // Writes SPSR, unpredictable in user mode. } else { Operand ctx = Register(context.RegisterAllocator.FixedContextRegister); using ScopedRegister tempRegister = context.RegisterAllocator.AllocateTempGprRegisterScoped(); using ScopedRegister tempRegister2 = context.RegisterAllocator.AllocateTempGprRegisterScoped(); context.Arm64Assembler.LdrRiUn(tempRegister.Operand, ctx, NativeContextOffsets.FlagsBaseOffset); if ((mask & 2) != 0) { // Endian flag. context.Arm64Assembler.Lsr(tempRegister2.Operand, rnOperand, InstEmitCommon.Const(9)); context.Arm64Assembler.Bfi(tempRegister.Operand, tempRegister2.Operand, 9, 1); } if ((mask & 4) != 0) { // GE flags. context.Arm64Assembler.Lsr(tempRegister2.Operand, rnOperand, InstEmitCommon.Const(16)); context.Arm64Assembler.Bfi(tempRegister.Operand, tempRegister2.Operand, 16, 4); } if ((mask & 8) != 0) { // NZCVQ flags. context.Arm64Assembler.Lsr(tempRegister2.Operand, rnOperand, InstEmitCommon.Const(27)); context.Arm64Assembler.Bfi(tempRegister.Operand, tempRegister2.Operand, 27, 5); context.Arm64Assembler.Lsr(tempRegister2.Operand, rnOperand, InstEmitCommon.Const(28)); InstEmitCommon.RestoreNzcvFlags(context, tempRegister2.Operand); context.SetNzcvModified(); } } } public static void Setend(CodeGenContext context, bool e) { Operand ctx = Register(context.RegisterAllocator.FixedContextRegister); using ScopedRegister tempRegister = context.RegisterAllocator.AllocateTempGprRegisterScoped(); context.Arm64Assembler.LdrRiUn(tempRegister.Operand, ctx, NativeContextOffsets.FlagsBaseOffset); if (e) { context.Arm64Assembler.Orr(tempRegister.Operand, tempRegister.Operand, InstEmitCommon.Const(1 << 9)); } else { context.Arm64Assembler.Bfc(tempRegister.Operand, 9, 1); } context.Arm64Assembler.StrRiUn(tempRegister.Operand, ctx, NativeContextOffsets.FlagsBaseOffset); } public static void Svc(CodeGenContext context, uint imm) { context.AddPendingSvc(imm); context.Arm64Assembler.B(0); } public static void Udf(CodeGenContext context, uint encoding, uint imm) { context.AddPendingUdf(encoding); context.Arm64Assembler.B(0); } public static void PrivilegedInstruction(CodeGenContext context, uint encoding) { Udf(context, encoding, 0); } private static IntPtr GetBkptHandlerPtr() { return Marshal.GetFunctionPointerForDelegate(NativeInterface.Break); } private static IntPtr GetSvcHandlerPtr() { return Marshal.GetFunctionPointerForDelegate(NativeInterface.SupervisorCall); } private static IntPtr GetUdfHandlerPtr() { return Marshal.GetFunctionPointerForDelegate(NativeInterface.Undefined); } private static IntPtr GetCntpctEl0Ptr() { return Marshal.GetFunctionPointerForDelegate(NativeInterface.GetCntpctEl0); } private static IntPtr CheckSynchronizationPtr() { return Marshal.GetFunctionPointerForDelegate(NativeInterface.CheckSynchronization); } public static bool NeedsCall(InstName name) { // All instructions that might do a host call should be included here. // That is required to reserve space on the stack for caller saved registers. return name == InstName.Mrrc; } public static bool NeedsCallSkipContext(InstName name) { // All instructions that might do a host call should be included here. // That is required to reserve space on the stack for caller saved registers. switch (name) { case InstName.Mcr: case InstName.Mrc: case InstName.Svc: case InstName.Udf: return true; } return false; } public static void WriteBkpt(CodeWriter writer, RegisterAllocator regAlloc, TailMerger tailMerger, int spillBaseOffset, uint pc, uint imm) { Assembler asm = new(writer); WriteCall(ref asm, regAlloc, GetBkptHandlerPtr(), skipContext: true, spillBaseOffset, null, pc, imm); WriteSyncPoint(writer, ref asm, regAlloc, tailMerger, spillBaseOffset); } public static void WriteSvc(CodeWriter writer, RegisterAllocator regAlloc, TailMerger tailMerger, int spillBaseOffset, uint pc, uint svcId) { Assembler asm = new(writer); WriteCall(ref asm, regAlloc, GetSvcHandlerPtr(), skipContext: true, spillBaseOffset, null, pc, svcId); WriteSyncPoint(writer, ref asm, regAlloc, tailMerger, spillBaseOffset); } public static void WriteUdf(CodeWriter writer, RegisterAllocator regAlloc, TailMerger tailMerger, int spillBaseOffset, uint pc, uint imm) { Assembler asm = new(writer); WriteCall(ref asm, regAlloc, GetUdfHandlerPtr(), skipContext: true, spillBaseOffset, null, pc, imm); WriteSyncPoint(writer, ref asm, regAlloc, tailMerger, spillBaseOffset); } public static void WriteReadCntpct(CodeWriter writer, RegisterAllocator regAlloc, int spillBaseOffset, int rt, int rt2) { Assembler asm = new(writer); uint resultMask = (1u << rt) | (1u << rt2); int tempRegister = 0; while ((resultMask & (1u << tempRegister)) != 0 && tempRegister < 32) { tempRegister++; } Debug.Assert(tempRegister < 32); WriteSpill(ref asm, regAlloc, resultMask, skipContext: false, spillBaseOffset, tempRegister); Operand rn = Register(tempRegister); asm.Mov(rn, (ulong)GetCntpctEl0Ptr()); asm.Blr(rn); if (rt != rt2) { asm.Lsr(Register(rt2), Register(0), InstEmitCommon.Const(32)); } asm.Mov(Register(rt, OperandType.I32), Register(0, OperandType.I32)); // Zero-extend. WriteFill(ref asm, regAlloc, resultMask, skipContext: false, spillBaseOffset, tempRegister); } public static void WriteSyncPoint( CodeWriter writer, ref Assembler asm, RegisterAllocator regAlloc, TailMerger tailMerger, int spillBaseOffset, Action storeToContext = null, Action loadFromContext = null) { int tempRegister = regAlloc.AllocateTempGprRegister(); Operand rt = Register(tempRegister, OperandType.I32); asm.LdrRiUn(rt, Register(regAlloc.FixedContextRegister), NativeContextOffsets.CounterOffset); int branchIndex = writer.InstructionPointer; asm.Cbnz(rt, 0); storeToContext?.Invoke(); WriteSpill(ref asm, regAlloc, 1u << tempRegister, skipContext: true, spillBaseOffset, tempRegister); Operand rn = Register(tempRegister == 0 ? 1 : 0); asm.Mov(rn, (ulong)CheckSynchronizationPtr()); asm.Blr(rn); tailMerger.AddConditionalZeroReturn(writer, asm, Register(0, OperandType.I32)); WriteFill(ref asm, regAlloc, 1u << tempRegister, skipContext: true, spillBaseOffset, tempRegister); loadFromContext?.Invoke(); asm.LdrRiUn(rt, Register(regAlloc.FixedContextRegister), NativeContextOffsets.CounterOffset); uint branchInst = writer.ReadInstructionAt(branchIndex); writer.WriteInstructionAt(branchIndex, branchInst | (((uint)(writer.InstructionPointer - branchIndex) & 0x7ffff) << 5)); asm.Sub(rt, rt, new Operand(OperandKind.Constant, OperandType.I32, 1)); asm.StrRiUn(rt, Register(regAlloc.FixedContextRegister), NativeContextOffsets.CounterOffset); regAlloc.FreeTempGprRegister(tempRegister); } private static void WriteCall( ref Assembler asm, RegisterAllocator regAlloc, IntPtr funcPtr, bool skipContext, int spillBaseOffset, int? resultRegister, params ulong[] callArgs) { uint resultMask = 0u; if (resultRegister.HasValue) { resultMask = 1u << resultRegister.Value; } int tempRegister = callArgs.Length; if (resultRegister.HasValue && tempRegister == resultRegister.Value) { tempRegister++; } WriteSpill(ref asm, regAlloc, resultMask, skipContext, spillBaseOffset, tempRegister); // We only support up to 7 arguments right now. // ABI defines the first 8 integer arguments to be passed on registers X0-X7. // We need at least one register to put the function address on, so that reduces the number of // registers we can use for that by one. Debug.Assert(callArgs.Length < 8); for (int index = 0; index < callArgs.Length; index++) { asm.Mov(Register(index), callArgs[index]); } Operand rn = Register(tempRegister); asm.Mov(rn, (ulong)funcPtr); asm.Blr(rn); if (resultRegister.HasValue && resultRegister.Value != 0) { asm.Mov(Register(resultRegister.Value), Register(0)); } WriteFill(ref asm, regAlloc, resultMask, skipContext, spillBaseOffset, tempRegister); } private static void WriteSpill(ref Assembler asm, RegisterAllocator regAlloc, uint exceptMask, bool skipContext, int spillOffset, int tempRegister) { if (skipContext) { InstEmitFlow.WriteSpillSkipContext(ref asm, regAlloc, spillOffset); } else { WriteSpillOrFill(ref asm, regAlloc, exceptMask, spillOffset, tempRegister, spill: true); } } private static void WriteFill(ref Assembler asm, RegisterAllocator regAlloc, uint exceptMask, bool skipContext, int spillOffset, int tempRegister) { if (skipContext) { InstEmitFlow.WriteFillSkipContext(ref asm, regAlloc, spillOffset); } else { WriteSpillOrFill(ref asm, regAlloc, exceptMask, spillOffset, tempRegister, spill: false); } } private static void WriteSpillOrFill( ref Assembler asm, RegisterAllocator regAlloc, uint exceptMask, int spillOffset, int tempRegister, bool spill) { uint gprMask = regAlloc.UsedGprsMask & ~(AbiConstants.GprCalleeSavedRegsMask | exceptMask); if (!spill) { // We must reload the status register before reloading the GPRs, // since we might otherwise trash one of them by using it as temp register. Operand rt = Register(tempRegister, OperandType.I32); asm.LdrRiUn(rt, Register(SpIndex), spillOffset + BitOperations.PopCount(gprMask) * 8); asm.MsrNzcv(rt); } while (gprMask != 0) { int reg = BitOperations.TrailingZeroCount(gprMask); if (reg < 31 && (gprMask & (2u << reg)) != 0 && spillOffset < RegisterSaveRestore.Encodable9BitsOffsetLimit) { if (spill) { asm.StpRiUn(Register(reg), Register(reg + 1), Register(SpIndex), spillOffset); } else { asm.LdpRiUn(Register(reg), Register(reg + 1), Register(SpIndex), spillOffset); } gprMask &= ~(3u << reg); spillOffset += 16; } else { if (spill) { asm.StrRiUn(Register(reg), Register(SpIndex), spillOffset); } else { asm.LdrRiUn(Register(reg), Register(SpIndex), spillOffset); } gprMask &= ~(1u << reg); spillOffset += 8; } } if (spill) { Operand rt = Register(tempRegister, OperandType.I32); asm.MrsNzcv(rt); asm.StrRiUn(rt, Register(SpIndex), spillOffset); } spillOffset += 8; if ((spillOffset & 8) != 0) { spillOffset += 8; } uint fpSimdMask = regAlloc.UsedFpSimdMask; while (fpSimdMask != 0) { int reg = BitOperations.TrailingZeroCount(fpSimdMask); if (reg < 31 && (fpSimdMask & (2u << reg)) != 0 && spillOffset < RegisterSaveRestore.Encodable9BitsOffsetLimit) { if (spill) { asm.StpRiUn(Register(reg, OperandType.V128), Register(reg + 1, OperandType.V128), Register(SpIndex), spillOffset); } else { asm.LdpRiUn(Register(reg, OperandType.V128), Register(reg + 1, OperandType.V128), Register(SpIndex), spillOffset); } fpSimdMask &= ~(3u << reg); spillOffset += 32; } else { if (spill) { asm.StrRiUn(Register(reg, OperandType.V128), Register(SpIndex), spillOffset); } else { asm.LdrRiUn(Register(reg, OperandType.V128), Register(SpIndex), spillOffset); } fpSimdMask &= ~(1u << reg); spillOffset += 16; } } } public static Operand Register(int register, OperandType type = OperandType.I64) { return new Operand(register, RegisterType.Integer, type); } } }