diff options
Diffstat (limited to 'ARMeilleure/CodeGen/Arm64/CodeGenerator.cs')
| -rw-r--r-- | ARMeilleure/CodeGen/Arm64/CodeGenerator.cs | 1576 |
1 files changed, 1576 insertions, 0 deletions
diff --git a/ARMeilleure/CodeGen/Arm64/CodeGenerator.cs b/ARMeilleure/CodeGen/Arm64/CodeGenerator.cs new file mode 100644 index 00000000..704aa45a --- /dev/null +++ b/ARMeilleure/CodeGen/Arm64/CodeGenerator.cs @@ -0,0 +1,1576 @@ +using ARMeilleure.CodeGen.Linking; +using ARMeilleure.CodeGen.Optimizations; +using ARMeilleure.CodeGen.RegisterAllocators; +using ARMeilleure.CodeGen.Unwinding; +using ARMeilleure.Common; +using ARMeilleure.Diagnostics; +using ARMeilleure.IntermediateRepresentation; +using ARMeilleure.Translation; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Numerics; + +using static ARMeilleure.IntermediateRepresentation.Operand; +using static ARMeilleure.IntermediateRepresentation.Operand.Factory; + +namespace ARMeilleure.CodeGen.Arm64 +{ + static class CodeGenerator + { + private const int DWordScale = 3; + + private const int RegistersCount = 32; + + private const int FpRegister = 29; + private const int LrRegister = 30; + private const int SpRegister = 31; + private const int ZrRegister = 31; + + private enum AccessSize + { + Byte, + Hword, + Auto + } + + private static Action<CodeGenContext, Operation>[] _instTable; + + static CodeGenerator() + { + _instTable = new Action<CodeGenContext, Operation>[EnumUtils.GetCount(typeof(Instruction))]; + + Add(Instruction.Add, GenerateAdd); + Add(Instruction.BitwiseAnd, GenerateBitwiseAnd); + Add(Instruction.BitwiseExclusiveOr, GenerateBitwiseExclusiveOr); + Add(Instruction.BitwiseNot, GenerateBitwiseNot); + Add(Instruction.BitwiseOr, GenerateBitwiseOr); + Add(Instruction.BranchIf, GenerateBranchIf); + Add(Instruction.ByteSwap, GenerateByteSwap); + Add(Instruction.Call, GenerateCall); + //Add(Instruction.Clobber, GenerateClobber); + Add(Instruction.Compare, GenerateCompare); + Add(Instruction.CompareAndSwap, GenerateCompareAndSwap); + Add(Instruction.CompareAndSwap16, GenerateCompareAndSwap16); + Add(Instruction.CompareAndSwap8, GenerateCompareAndSwap8); + Add(Instruction.ConditionalSelect, GenerateConditionalSelect); + Add(Instruction.ConvertI64ToI32, GenerateConvertI64ToI32); + Add(Instruction.ConvertToFP, GenerateConvertToFP); + Add(Instruction.ConvertToFPUI, GenerateConvertToFPUI); + Add(Instruction.Copy, GenerateCopy); + Add(Instruction.CountLeadingZeros, GenerateCountLeadingZeros); + Add(Instruction.Divide, GenerateDivide); + Add(Instruction.DivideUI, GenerateDivideUI); + Add(Instruction.Fill, GenerateFill); + Add(Instruction.Load, GenerateLoad); + Add(Instruction.Load16, GenerateLoad16); + Add(Instruction.Load8, GenerateLoad8); + Add(Instruction.MemoryBarrier, GenerateMemoryBarrier); + Add(Instruction.Multiply, GenerateMultiply); + Add(Instruction.Multiply64HighSI, GenerateMultiply64HighSI); + Add(Instruction.Multiply64HighUI, GenerateMultiply64HighUI); + Add(Instruction.Negate, GenerateNegate); + Add(Instruction.Return, GenerateReturn); + Add(Instruction.RotateRight, GenerateRotateRight); + Add(Instruction.ShiftLeft, GenerateShiftLeft); + Add(Instruction.ShiftRightSI, GenerateShiftRightSI); + Add(Instruction.ShiftRightUI, GenerateShiftRightUI); + Add(Instruction.SignExtend16, GenerateSignExtend16); + Add(Instruction.SignExtend32, GenerateSignExtend32); + Add(Instruction.SignExtend8, GenerateSignExtend8); + Add(Instruction.Spill, GenerateSpill); + Add(Instruction.SpillArg, GenerateSpillArg); + Add(Instruction.StackAlloc, GenerateStackAlloc); + Add(Instruction.Store, GenerateStore); + Add(Instruction.Store16, GenerateStore16); + Add(Instruction.Store8, GenerateStore8); + Add(Instruction.Subtract, GenerateSubtract); + Add(Instruction.Tailcall, GenerateTailcall); + Add(Instruction.VectorCreateScalar, GenerateVectorCreateScalar); + Add(Instruction.VectorExtract, GenerateVectorExtract); + Add(Instruction.VectorExtract16, GenerateVectorExtract16); + Add(Instruction.VectorExtract8, GenerateVectorExtract8); + Add(Instruction.VectorInsert, GenerateVectorInsert); + Add(Instruction.VectorInsert16, GenerateVectorInsert16); + Add(Instruction.VectorInsert8, GenerateVectorInsert8); + Add(Instruction.VectorOne, GenerateVectorOne); + Add(Instruction.VectorZero, GenerateVectorZero); + Add(Instruction.VectorZeroUpper64, GenerateVectorZeroUpper64); + Add(Instruction.VectorZeroUpper96, GenerateVectorZeroUpper96); + Add(Instruction.ZeroExtend16, GenerateZeroExtend16); + Add(Instruction.ZeroExtend32, GenerateZeroExtend32); + Add(Instruction.ZeroExtend8, GenerateZeroExtend8); + + static void Add(Instruction inst, Action<CodeGenContext, Operation> func) + { + _instTable[(int)inst] = func; + } + } + + public static CompiledFunction Generate(CompilerContext cctx) + { + ControlFlowGraph cfg = cctx.Cfg; + + Logger.StartPass(PassName.Optimization); + + if (cctx.Options.HasFlag(CompilerOptions.Optimize)) + { + if (cctx.Options.HasFlag(CompilerOptions.SsaForm)) + { + Optimizer.RunPass(cfg); + } + + BlockPlacement.RunPass(cfg); + } + + Arm64Optimizer.RunPass(cfg); + + Logger.EndPass(PassName.Optimization, cfg); + + Logger.StartPass(PassName.PreAllocation); + + StackAllocator stackAlloc = new(); + + PreAllocator.RunPass(cctx, stackAlloc, out int maxCallArgs); + + Logger.EndPass(PassName.PreAllocation, cfg); + + Logger.StartPass(PassName.RegisterAllocation); + + if (cctx.Options.HasFlag(CompilerOptions.SsaForm)) + { + Ssa.Deconstruct(cfg); + } + + IRegisterAllocator regAlloc; + + if (cctx.Options.HasFlag(CompilerOptions.Lsra)) + { + regAlloc = new LinearScanAllocator(); + } + else + { + regAlloc = new HybridAllocator(); + } + + RegisterMasks regMasks = new( + CallingConvention.GetIntAvailableRegisters(), + CallingConvention.GetVecAvailableRegisters(), + CallingConvention.GetIntCallerSavedRegisters(), + CallingConvention.GetVecCallerSavedRegisters(), + CallingConvention.GetIntCalleeSavedRegisters(), + CallingConvention.GetVecCalleeSavedRegisters(), + RegistersCount); + + AllocationResult allocResult = regAlloc.RunPass(cfg, stackAlloc, regMasks); + + Logger.EndPass(PassName.RegisterAllocation, cfg); + + Logger.StartPass(PassName.CodeGeneration); + + //Console.Error.WriteLine(IRDumper.GetDump(cfg)); + + bool relocatable = (cctx.Options & CompilerOptions.Relocatable) != 0; + + CodeGenContext context = new(allocResult, maxCallArgs, cfg.Blocks.Count, relocatable); + + UnwindInfo unwindInfo = WritePrologue(context); + + for (BasicBlock block = cfg.Blocks.First; block != null; block = block.ListNext) + { + context.EnterBlock(block); + + for (Operation node = block.Operations.First; node != default;) + { + node = GenerateOperation(context, node); + } + + if (block.SuccessorsCount == 0) + { + // The only blocks which can have 0 successors are exit blocks. + Operation last = block.Operations.Last; + + Debug.Assert(last.Instruction == Instruction.Tailcall || + last.Instruction == Instruction.Return); + } + else + { + BasicBlock succ = block.GetSuccessor(0); + + if (succ != block.ListNext) + { + context.JumpTo(succ); + } + } + } + + (byte[] code, RelocInfo relocInfo) = context.GetCode(); + + Logger.EndPass(PassName.CodeGeneration); + + return new CompiledFunction(code, unwindInfo, relocInfo); + } + + private static Operation GenerateOperation(CodeGenContext context, Operation operation) + { + if (operation.Instruction == Instruction.Extended) + { + CodeGeneratorIntrinsic.GenerateOperation(context, operation); + } + else + { + if (IsLoadOrStore(operation) && + operation.ListNext != default && + operation.ListNext.Instruction == operation.Instruction && + TryPairMemoryOp(context, operation, operation.ListNext)) + { + // Skip next operation if we managed to pair them. + return operation.ListNext.ListNext; + } + + Action<CodeGenContext, Operation> func = _instTable[(int)operation.Instruction]; + + if (func != null) + { + func(context, operation); + } + else + { + throw new ArgumentException($"Invalid instruction \"{operation.Instruction}\"."); + } + } + + return operation.ListNext; + } + + private static void GenerateAdd(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + // ValidateBinOp(dest, src1, src2); + + if (dest.Type.IsInteger()) + { + context.Assembler.Add(dest, src1, src2); + } + else + { + context.Assembler.FaddScalar(dest, src1, src2); + } + } + + private static void GenerateBitwiseAnd(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + ValidateBinOp(dest, src1, src2); + + Debug.Assert(dest.Type.IsInteger()); + + context.Assembler.And(dest, src1, src2); + } + + private static void GenerateBitwiseExclusiveOr(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + ValidateBinOp(dest, src1, src2); + + if (dest.Type.IsInteger()) + { + context.Assembler.Eor(dest, src1, src2); + } + else + { + context.Assembler.EorVector(dest, src1, src2); + } + } + + private static void GenerateBitwiseNot(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + ValidateUnOp(dest, source); + + Debug.Assert(dest.Type.IsInteger()); + + context.Assembler.Mvn(dest, source); + } + + private static void GenerateBitwiseOr(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + ValidateBinOp(dest, src1, src2); + + Debug.Assert(dest.Type.IsInteger()); + + context.Assembler.Orr(dest, src1, src2); + } + + private static void GenerateBranchIf(CodeGenContext context, Operation operation) + { + Operand comp = operation.GetSource(2); + + Debug.Assert(comp.Kind == OperandKind.Constant); + + var cond = ((Comparison)comp.AsInt32()).ToArmCondition(); + + GenerateCompareCommon(context, operation); + + context.JumpTo(cond, context.CurrBlock.GetSuccessor(1)); + } + + private static void GenerateByteSwap(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + ValidateUnOp(dest, source); + + Debug.Assert(dest.Type.IsInteger()); + + context.Assembler.Rev(dest, source); + } + + private static void GenerateCall(CodeGenContext context, Operation operation) + { + context.Assembler.Blr(operation.GetSource(0)); + } + + private static void GenerateCompare(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand comp = operation.GetSource(2); + + Debug.Assert(dest.Type == OperandType.I32); + Debug.Assert(comp.Kind == OperandKind.Constant); + + var cond = ((Comparison)comp.AsInt32()).ToArmCondition(); + + GenerateCompareCommon(context, operation); + + context.Assembler.Cset(dest, cond); + } + + private static void GenerateCompareAndSwap(CodeGenContext context, Operation operation) + { + if (operation.SourcesCount == 5) // CompareAndSwap128 has 5 sources, compared to CompareAndSwap64/32's 3. + { + Operand actualLow = operation.GetDestination(0); + Operand actualHigh = operation.GetDestination(1); + Operand temp0 = operation.GetDestination(2); + Operand temp1 = operation.GetDestination(3); + Operand address = operation.GetSource(0); + Operand expectedLow = operation.GetSource(1); + Operand expectedHigh = operation.GetSource(2); + Operand desiredLow = operation.GetSource(3); + Operand desiredHigh = operation.GetSource(4); + + GenerateAtomicDcas( + context, + address, + expectedLow, + expectedHigh, + desiredLow, + desiredHigh, + actualLow, + actualHigh, + temp0, + temp1); + } + else + { + Operand actual = operation.GetDestination(0); + Operand result = operation.GetDestination(1); + Operand address = operation.GetSource(0); + Operand expected = operation.GetSource(1); + Operand desired = operation.GetSource(2); + + GenerateAtomicCas(context, address, expected, desired, actual, result, AccessSize.Auto); + } + } + + private static void GenerateCompareAndSwap16(CodeGenContext context, Operation operation) + { + Operand actual = operation.GetDestination(0); + Operand result = operation.GetDestination(1); + Operand address = operation.GetSource(0); + Operand expected = operation.GetSource(1); + Operand desired = operation.GetSource(2); + + GenerateAtomicCas(context, address, expected, desired, actual, result, AccessSize.Hword); + } + + private static void GenerateCompareAndSwap8(CodeGenContext context, Operation operation) + { + Operand actual = operation.GetDestination(0); + Operand result = operation.GetDestination(1); + Operand address = operation.GetSource(0); + Operand expected = operation.GetSource(1); + Operand desired = operation.GetSource(2); + + GenerateAtomicCas(context, address, expected, desired, actual, result, AccessSize.Byte); + } + + private static void GenerateCompareCommon(CodeGenContext context, Operation operation) + { + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + EnsureSameType(src1, src2); + + Debug.Assert(src1.Type.IsInteger()); + + context.Assembler.Cmp(src1, src2); + } + + private static void GenerateConditionalSelect(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + Operand src3 = operation.GetSource(2); + + EnsureSameType(dest, src2, src3); + + Debug.Assert(dest.Type.IsInteger()); + Debug.Assert(src1.Type == OperandType.I32); + + context.Assembler.Cmp (src1, Const(src1.Type, 0)); + context.Assembler.Csel(dest, src2, src3, ArmCondition.Ne); + } + + private static void GenerateConvertI64ToI32(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type == OperandType.I32 && source.Type == OperandType.I64); + + context.Assembler.Mov(dest, Register(source, OperandType.I32)); + } + + private static void GenerateConvertToFP(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type == OperandType.FP32 || dest.Type == OperandType.FP64); + Debug.Assert(dest.Type != source.Type); + Debug.Assert(source.Type != OperandType.V128); + + if (source.Type.IsInteger()) + { + context.Assembler.ScvtfScalar(dest, source); + } + else + { + context.Assembler.FcvtScalar(dest, source); + } + } + + private static void GenerateConvertToFPUI(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type == OperandType.FP32 || dest.Type == OperandType.FP64); + Debug.Assert(dest.Type != source.Type); + Debug.Assert(source.Type.IsInteger()); + + context.Assembler.UcvtfScalar(dest, source); + } + + private static void GenerateCopy(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + EnsureSameType(dest, source); + + Debug.Assert(dest.Type.IsInteger() || source.Kind != OperandKind.Constant); + + // Moves to the same register are useless. + if (dest.Kind == source.Kind && dest.Value == source.Value) + { + return; + } + + if (dest.Kind == OperandKind.Register && source.Kind == OperandKind.Constant) + { + if (source.Relocatable) + { + context.ReserveRelocatableConstant(dest, source.Symbol, source.Value); + } + else + { + GenerateConstantCopy(context, dest, source.Value); + } + } + else + { + context.Assembler.Mov(dest, source); + } + } + + private static void GenerateCountLeadingZeros(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + EnsureSameType(dest, source); + + Debug.Assert(dest.Type.IsInteger()); + + context.Assembler.Clz(dest, source); + } + + private static void GenerateDivide(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand dividend = operation.GetSource(0); + Operand divisor = operation.GetSource(1); + + ValidateBinOp(dest, dividend, divisor); + + if (dest.Type.IsInteger()) + { + context.Assembler.Sdiv(dest, dividend, divisor); + } + else + { + context.Assembler.FdivScalar(dest, dividend, divisor); + } + } + + private static void GenerateDivideUI(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand dividend = operation.GetSource(0); + Operand divisor = operation.GetSource(1); + + ValidateBinOp(dest, dividend, divisor); + + context.Assembler.Udiv(dest, dividend, divisor); + } + + private static void GenerateLoad(CodeGenContext context, Operation operation) + { + Operand value = operation.Destination; + Operand address = operation.GetSource(0); + + context.Assembler.Ldr(value, address); + } + + private static void GenerateLoad16(CodeGenContext context, Operation operation) + { + Operand value = operation.Destination; + Operand address = operation.GetSource(0); + + Debug.Assert(value.Type.IsInteger()); + + context.Assembler.LdrhRiUn(value, address, 0); + } + + private static void GenerateLoad8(CodeGenContext context, Operation operation) + { + Operand value = operation.Destination; + Operand address = operation.GetSource(0); + + Debug.Assert(value.Type.IsInteger()); + + context.Assembler.LdrbRiUn(value, address, 0); + } + + private static void GenerateMemoryBarrier(CodeGenContext context, Operation operation) + { + context.Assembler.Dmb(0xf); + } + + private static void GenerateMultiply(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + EnsureSameType(dest, src1, src2); + + if (dest.Type.IsInteger()) + { + context.Assembler.Mul(dest, src1, src2); + } + else + { + context.Assembler.FmulScalar(dest, src1, src2); + } + } + + private static void GenerateMultiply64HighSI(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + EnsureSameType(dest, src1, src2); + + Debug.Assert(dest.Type == OperandType.I64); + + context.Assembler.Smulh(dest, src1, src2); + } + + private static void GenerateMultiply64HighUI(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + EnsureSameType(dest, src1, src2); + + Debug.Assert(dest.Type == OperandType.I64); + + context.Assembler.Umulh(dest, src1, src2); + } + + private static void GenerateNegate(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + ValidateUnOp(dest, source); + + if (dest.Type.IsInteger()) + { + context.Assembler.Neg(dest, source); + } + else + { + context.Assembler.FnegScalar(dest, source); + } + } + + private static void GenerateLoad(CodeGenContext context, Operand value, Operand address, int offset) + { + if (CodeGenCommon.ConstFitsOnUImm12(offset, value.Type)) + { + context.Assembler.LdrRiUn(value, address, offset); + } + else if (CodeGenCommon.ConstFitsOnSImm9(offset)) + { + context.Assembler.Ldur(value, address, offset); + } + else + { + Operand tempAddress = Register(CodeGenCommon.ReservedRegister); + GenerateConstantCopy(context, tempAddress, (ulong)offset); + context.Assembler.Add(tempAddress, address, tempAddress, ArmExtensionType.Uxtx); // Address might be SP and must be the first input. + context.Assembler.LdrRiUn(value, tempAddress, 0); + } + } + + private static void GenerateReturn(CodeGenContext context, Operation operation) + { + WriteEpilogue(context); + + context.Assembler.Ret(Register(LrRegister)); + } + + private static void GenerateRotateRight(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + ValidateShift(dest, src1, src2); + + context.Assembler.Ror(dest, src1, src2); + } + + private static void GenerateShiftLeft(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + ValidateShift(dest, src1, src2); + + context.Assembler.Lsl(dest, src1, src2); + } + + private static void GenerateShiftRightSI(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + ValidateShift(dest, src1, src2); + + context.Assembler.Asr(dest, src1, src2); + } + + private static void GenerateShiftRightUI(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + ValidateShift(dest, src1, src2); + + context.Assembler.Lsr(dest, src1, src2); + } + + private static void GenerateSignExtend16(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); + + context.Assembler.Sxth(dest, source); + } + + private static void GenerateSignExtend32(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); + + context.Assembler.Sxtw(dest, source); + } + + private static void GenerateSignExtend8(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); + + context.Assembler.Sxtb(dest, source); + } + + private static void GenerateFill(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand offset = operation.GetSource(0); + + Debug.Assert(offset.Kind == OperandKind.Constant); + + int offs = offset.AsInt32() + context.CallArgsRegionSize + context.FpLrSaveRegionSize; + + GenerateLoad(context, dest, Register(SpRegister), offs); + } + + private static void GenerateStore(CodeGenContext context, Operand value, Operand address, int offset) + { + if (CodeGenCommon.ConstFitsOnUImm12(offset, value.Type)) + { + context.Assembler.StrRiUn(value, address, offset); + } + else if (CodeGenCommon.ConstFitsOnSImm9(offset)) + { + context.Assembler.Stur(value, address, offset); + } + else + { + Operand tempAddress = Register(CodeGenCommon.ReservedRegister); + GenerateConstantCopy(context, tempAddress, (ulong)offset); + context.Assembler.Add(tempAddress, address, tempAddress, ArmExtensionType.Uxtx); // Address might be SP and must be the first input. + context.Assembler.StrRiUn(value, tempAddress, 0); + } + } + + private static void GenerateSpill(CodeGenContext context, Operation operation) + { + GenerateSpill(context, operation, context.CallArgsRegionSize + context.FpLrSaveRegionSize); + } + + private static void GenerateSpillArg(CodeGenContext context, Operation operation) + { + GenerateSpill(context, operation, 0); + } + + private static void GenerateStackAlloc(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand offset = operation.GetSource(0); + + Debug.Assert(offset.Kind == OperandKind.Constant); + + int offs = offset.AsInt32() + context.CallArgsRegionSize + context.FpLrSaveRegionSize; + + context.Assembler.Add(dest, Register(SpRegister), Const(dest.Type, offs)); + } + + private static void GenerateStore(CodeGenContext context, Operation operation) + { + Operand value = operation.GetSource(1); + Operand address = operation.GetSource(0); + + context.Assembler.Str(value, address); + } + + private static void GenerateStore16(CodeGenContext context, Operation operation) + { + Operand value = operation.GetSource(1); + Operand address = operation.GetSource(0); + + Debug.Assert(value.Type.IsInteger()); + + context.Assembler.StrhRiUn(value, address, 0); + } + + private static void GenerateStore8(CodeGenContext context, Operation operation) + { + Operand value = operation.GetSource(1); + Operand address = operation.GetSource(0); + + Debug.Assert(value.Type.IsInteger()); + + context.Assembler.StrbRiUn(value, address, 0); + } + + private static void GenerateSpill(CodeGenContext context, Operation operation, int baseOffset) + { + Operand offset = operation.GetSource(0); + Operand source = operation.GetSource(1); + + Debug.Assert(offset.Kind == OperandKind.Constant); + + int offs = offset.AsInt32() + baseOffset; + + GenerateStore(context, source, Register(SpRegister), offs); + } + + private static void GenerateSubtract(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); + Operand src2 = operation.GetSource(1); + + // ValidateBinOp(dest, src1, src2); + + if (dest.Type.IsInteger()) + { + context.Assembler.Sub(dest, src1, src2); + } + else + { + context.Assembler.FsubScalar(dest, src1, src2); + } + } + + private static void GenerateTailcall(CodeGenContext context, Operation operation) + { + WriteEpilogue(context); + + context.Assembler.Br(operation.GetSource(0)); + } + + private static void GenerateVectorCreateScalar(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + if (dest != default) + { + Debug.Assert(!dest.Type.IsInteger() && source.Type.IsInteger()); + + OperandType destType = source.Type == OperandType.I64 ? OperandType.FP64 : OperandType.FP32; + + context.Assembler.Fmov(Register(dest, destType), source, topHalf: false); + } + } + + private static void GenerateVectorExtract(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; // Value + Operand src1 = operation.GetSource(0); // Vector + Operand src2 = operation.GetSource(1); // Index + + Debug.Assert(src1.Type == OperandType.V128); + Debug.Assert(src2.Kind == OperandKind.Constant); + + byte index = src2.AsByte(); + + Debug.Assert(index < OperandType.V128.GetSizeInBytes() / dest.Type.GetSizeInBytes()); + + if (dest.Type.IsInteger()) + { + context.Assembler.Umov(dest, src1, index, dest.Type == OperandType.I64 ? 3 : 2); + } + else + { + context.Assembler.DupScalar(dest, src1, index, dest.Type == OperandType.FP64 ? 3 : 2); + } + } + + private static void GenerateVectorExtract16(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; // Value + Operand src1 = operation.GetSource(0); // Vector + Operand src2 = operation.GetSource(1); // Index + + Debug.Assert(src1.Type == OperandType.V128); + Debug.Assert(src2.Kind == OperandKind.Constant); + + byte index = src2.AsByte(); + + Debug.Assert(index < 8); + + context.Assembler.Umov(dest, src1, index, 1); + } + + private static void GenerateVectorExtract8(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; // Value + Operand src1 = operation.GetSource(0); // Vector + Operand src2 = operation.GetSource(1); // Index + + Debug.Assert(src1.Type == OperandType.V128); + Debug.Assert(src2.Kind == OperandKind.Constant); + + byte index = src2.AsByte(); + + Debug.Assert(index < 16); + + context.Assembler.Umov(dest, src1, index, 0); + } + + private static void GenerateVectorInsert(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); // Vector + Operand src2 = operation.GetSource(1); // Value + Operand src3 = operation.GetSource(2); // Index + + EnsureSameReg(dest, src1); + + Debug.Assert(src1.Type == OperandType.V128); + Debug.Assert(src3.Kind == OperandKind.Constant); + + byte index = src3.AsByte(); + + if (src2.Type.IsInteger()) + { + context.Assembler.Ins(dest, src2, index, src2.Type == OperandType.I64 ? 3 : 2); + } + else + { + context.Assembler.Ins(dest, src2, 0, index, src2.Type == OperandType.FP64 ? 3 : 2); + } + } + + private static void GenerateVectorInsert16(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); // Vector + Operand src2 = operation.GetSource(1); // Value + Operand src3 = operation.GetSource(2); // Index + + EnsureSameReg(dest, src1); + + Debug.Assert(src1.Type == OperandType.V128); + Debug.Assert(src3.Kind == OperandKind.Constant); + + byte index = src3.AsByte(); + + context.Assembler.Ins(dest, src2, index, 1); + } + + private static void GenerateVectorInsert8(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand src1 = operation.GetSource(0); // Vector + Operand src2 = operation.GetSource(1); // Value + Operand src3 = operation.GetSource(2); // Index + + EnsureSameReg(dest, src1); + + Debug.Assert(src1.Type == OperandType.V128); + Debug.Assert(src3.Kind == OperandKind.Constant); + + byte index = src3.AsByte(); + + context.Assembler.Ins(dest, src2, index, 0); + } + + private static void GenerateVectorOne(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + + Debug.Assert(!dest.Type.IsInteger()); + + context.Assembler.CmeqVector(dest, dest, dest, 2); + } + + private static void GenerateVectorZero(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + + Debug.Assert(!dest.Type.IsInteger()); + + context.Assembler.EorVector(dest, dest, dest); + } + + private static void GenerateVectorZeroUpper64(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type == OperandType.V128 && source.Type == OperandType.V128); + + context.Assembler.Fmov(Register(dest, OperandType.FP64), Register(source, OperandType.FP64)); + } + + private static void GenerateVectorZeroUpper96(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type == OperandType.V128 && source.Type == OperandType.V128); + + context.Assembler.Fmov(Register(dest, OperandType.FP32), Register(source, OperandType.FP32)); + } + + private static void GenerateZeroExtend16(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); + + context.Assembler.Uxth(dest, source); + } + + private static void GenerateZeroExtend32(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); + + // We can eliminate the move if source is already 32-bit and the registers are the same. + if (dest.Value == source.Value && source.Type == OperandType.I32) + { + return; + } + + context.Assembler.Mov(Register(dest.GetRegister().Index, OperandType.I32), source); + } + + private static void GenerateZeroExtend8(CodeGenContext context, Operation operation) + { + Operand dest = operation.Destination; + Operand source = operation.GetSource(0); + + Debug.Assert(dest.Type.IsInteger() && source.Type.IsInteger()); + + context.Assembler.Uxtb(dest, source); + } + + private static UnwindInfo WritePrologue(CodeGenContext context) + { + List<UnwindPushEntry> pushEntries = new List<UnwindPushEntry>(); + + Operand rsp = Register(SpRegister); + + int intMask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters; + int vecMask = CallingConvention.GetFpCalleeSavedRegisters() & context.AllocResult.VecUsedRegisters; + + int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask); + int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask); + + int calleeSaveRegionSize = Align16(intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * 8); + + int offset = 0; + + WritePrologueCalleeSavesPreIndexed(context, pushEntries, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64); + WritePrologueCalleeSavesPreIndexed(context, pushEntries, ref vecMask, ref offset, calleeSaveRegionSize, OperandType.FP64); + + int localSize = Align16(context.AllocResult.SpillRegionSize + context.FpLrSaveRegionSize); + int outArgsSize = context.CallArgsRegionSize; + + if (CodeGenCommon.ConstFitsOnSImm7(localSize, DWordScale)) + { + if (context.HasCall) + { + context.Assembler.StpRiPre(Register(FpRegister), Register(LrRegister), rsp, -localSize); + context.Assembler.MovSp(Register(FpRegister), rsp); + } + + if (outArgsSize != 0) + { + context.Assembler.Sub(rsp, rsp, Const(OperandType.I64, outArgsSize)); + } + } + else + { + int frameSize = localSize + outArgsSize; + if (frameSize != 0) + { + if (CodeGenCommon.ConstFitsOnUImm12(frameSize)) + { + context.Assembler.Sub(rsp, rsp, Const(OperandType.I64, frameSize)); + } + else + { + Operand tempSize = Register(CodeGenCommon.ReservedRegister); + GenerateConstantCopy(context, tempSize, (ulong)frameSize); + context.Assembler.Sub(rsp, rsp, tempSize, ArmExtensionType.Uxtx); + } + } + + context.Assembler.StpRiUn(Register(FpRegister), Register(LrRegister), rsp, outArgsSize); + + if (outArgsSize != 0) + { + context.Assembler.Add(Register(FpRegister), Register(SpRegister), Const(OperandType.I64, outArgsSize)); + } + else + { + context.Assembler.MovSp(Register(FpRegister), Register(SpRegister)); + } + } + + return new UnwindInfo(pushEntries.ToArray(), context.StreamOffset); + } + + private static void WritePrologueCalleeSavesPreIndexed( + CodeGenContext context, + List<UnwindPushEntry> pushEntries, + ref int mask, + ref int offset, + int calleeSaveRegionSize, + OperandType type) + { + if ((BitOperations.PopCount((uint)mask) & 1) != 0) + { + int reg = BitOperations.TrailingZeroCount(mask); + + pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.PushReg, context.StreamOffset, regIndex: reg)); + + mask &= ~(1 << reg); + + if (offset != 0) + { + context.Assembler.StrRiUn(Register(reg, type), Register(SpRegister), offset); + } + else + { + context.Assembler.StrRiPre(Register(reg, type), Register(SpRegister), -calleeSaveRegionSize); + } + + offset += type.GetSizeInBytes(); + } + + while (mask != 0) + { + int reg = BitOperations.TrailingZeroCount(mask); + + pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.PushReg, context.StreamOffset, regIndex: reg)); + + mask &= ~(1 << reg); + + int reg2 = BitOperations.TrailingZeroCount(mask); + + pushEntries.Add(new UnwindPushEntry(UnwindPseudoOp.PushReg, context.StreamOffset, regIndex: reg2)); + + mask &= ~(1 << reg2); + + if (offset != 0) + { + context.Assembler.StpRiUn(Register(reg, type), Register(reg2, type), Register(SpRegister), offset); + } + else + { + context.Assembler.StpRiPre(Register(reg, type), Register(reg2, type), Register(SpRegister), -calleeSaveRegionSize); + } + + offset += type.GetSizeInBytes() * 2; + } + } + + private static void WriteEpilogue(CodeGenContext context) + { + Operand rsp = Register(SpRegister); + + int localSize = Align16(context.AllocResult.SpillRegionSize + context.FpLrSaveRegionSize); + int outArgsSize = context.CallArgsRegionSize; + + if (CodeGenCommon.ConstFitsOnSImm7(localSize, DWordScale)) + { + if (outArgsSize != 0) + { + context.Assembler.Add(rsp, rsp, Const(OperandType.I64, outArgsSize)); + } + + if (context.HasCall) + { + context.Assembler.LdpRiPost(Register(FpRegister), Register(LrRegister), rsp, localSize); + } + } + else + { + if (context.HasCall) + { + context.Assembler.LdpRiUn(Register(FpRegister), Register(LrRegister), rsp, outArgsSize); + } + + int frameSize = localSize + outArgsSize; + if (frameSize != 0) + { + if (CodeGenCommon.ConstFitsOnUImm12(frameSize)) + { + context.Assembler.Add(rsp, rsp, Const(OperandType.I64, frameSize)); + } + else + { + Operand tempSize = Register(CodeGenCommon.ReservedRegister); + GenerateConstantCopy(context, tempSize, (ulong)frameSize); + context.Assembler.Add(rsp, rsp, tempSize, ArmExtensionType.Uxtx); + } + } + } + + int intMask = CallingConvention.GetIntCalleeSavedRegisters() & context.AllocResult.IntUsedRegisters; + int vecMask = CallingConvention.GetFpCalleeSavedRegisters() & context.AllocResult.VecUsedRegisters; + + int intCalleeSavedRegsCount = BitOperations.PopCount((uint)intMask); + int vecCalleeSavedRegsCount = BitOperations.PopCount((uint)vecMask); + + int offset = intCalleeSavedRegsCount * 8 + vecCalleeSavedRegsCount * 8; + int calleeSaveRegionSize = Align16(offset); + + WriteEpilogueCalleeSavesPostIndexed(context, ref vecMask, ref offset, calleeSaveRegionSize, OperandType.FP64); + WriteEpilogueCalleeSavesPostIndexed(context, ref intMask, ref offset, calleeSaveRegionSize, OperandType.I64); + } + + private static void WriteEpilogueCalleeSavesPostIndexed( + CodeGenContext context, + ref int mask, + ref int offset, + int calleeSaveRegionSize, + OperandType type) + { + while (mask != 0) + { + int reg = BitUtils.HighestBitSet(mask); + + mask &= ~(1 << reg); + + if (mask != 0) + { + int reg2 = BitUtils.HighestBitSet(mask); + + mask &= ~(1 << reg2); + + offset -= type.GetSizeInBytes() * 2; + + if (offset != 0) + { + context.Assembler.LdpRiUn(Register(reg2, type), Register(reg, type), Register(SpRegister), offset); + } + else + { + context.Assembler.LdpRiPost(Register(reg2, type), Register(reg, type), Register(SpRegister), calleeSaveRegionSize); + } + } + else + { + offset -= type.GetSizeInBytes(); + + if (offset != 0) + { + context.Assembler.LdrRiUn(Register(reg, type), Register(SpRegister), offset); + } + else + { + context.Assembler.LdrRiPost(Register(reg, type), Register(SpRegister), calleeSaveRegionSize); + } + } + } + } + + private static void GenerateConstantCopy(CodeGenContext context, Operand dest, ulong value) + { + if (value != 0) + { + int hw = 0; + bool first = true; + + while (value != 0) + { + int valueLow = (ushort)value; + if (valueLow != 0) + { + if (first) + { + context.Assembler.Movz(dest, valueLow, hw); + first = false; + } + else + { + context.Assembler.Movk(dest, valueLow, hw); + } + } + + hw++; + value >>= 16; + } + } + else + { + context.Assembler.Mov(dest, Register(ZrRegister, dest.Type)); + } + } + + private static void GenerateAtomicCas( + CodeGenContext context, + Operand address, + Operand expected, + Operand desired, + Operand actual, + Operand result, + AccessSize accessSize) + { + int startOffset = context.StreamOffset; + + switch (accessSize) + { + case AccessSize.Byte: + context.Assembler.Ldaxrb(actual, address); + break; + case AccessSize.Hword: + context.Assembler.Ldaxrh(actual, address); + break; + default: + context.Assembler.Ldaxr(actual, address); + break; + } + + context.Assembler.Cmp(actual, expected); + + context.JumpToNear(ArmCondition.Ne); + + switch (accessSize) + { + case AccessSize.Byte: + context.Assembler.Stlxrb(desired, address, result); + break; + case AccessSize.Hword: + context.Assembler.Stlxrh(desired, address, result); + break; + default: + context.Assembler.Stlxr(desired, address, result); + break; + } + + context.Assembler.Cbnz(result, startOffset - context.StreamOffset); // Retry if store failed. + + context.JumpHere(); + + context.Assembler.Clrex(); + } + + private static void GenerateAtomicDcas( + CodeGenContext context, + Operand address, + Operand expectedLow, + Operand expectedHigh, + Operand desiredLow, + Operand desiredHigh, + Operand actualLow, + Operand actualHigh, + Operand temp0, + Operand temp1) + { + int startOffset = context.StreamOffset; + + context.Assembler.Ldaxp(actualLow, actualHigh, address); + context.Assembler.Eor(temp0, actualHigh, expectedHigh); + context.Assembler.Eor(temp1, actualLow, expectedLow); + context.Assembler.Orr(temp0, temp1, temp0); + + context.JumpToNearIfNotZero(temp0); + + Operand result = Register(temp0, OperandType.I32); + + context.Assembler.Stlxp(desiredLow, desiredHigh, address, result); + context.Assembler.Cbnz(result, startOffset - context.StreamOffset); // Retry if store failed. + + context.JumpHere(); + + context.Assembler.Clrex(); + } + + private static bool TryPairMemoryOp(CodeGenContext context, Operation currentOp, Operation nextOp) + { + if (!TryGetMemOpBaseAndOffset(currentOp, out Operand op1Base, out int op1Offset)) + { + return false; + } + + if (!TryGetMemOpBaseAndOffset(nextOp, out Operand op2Base, out int op2Offset)) + { + return false; + } + + if (op1Base != op2Base) + { + return false; + } + + OperandType valueType = GetMemOpValueType(currentOp); + + if (valueType != GetMemOpValueType(nextOp) || op1Offset + valueType.GetSizeInBytes() != op2Offset) + { + return false; + } + + if (!CodeGenCommon.ConstFitsOnSImm7(op1Offset, valueType.GetSizeInBytesLog2())) + { + return false; + } + + if (currentOp.Instruction == Instruction.Load) + { + context.Assembler.LdpRiUn(currentOp.Destination, nextOp.Destination, op1Base, op1Offset); + } + else if (currentOp.Instruction == Instruction.Store) + { + context.Assembler.StpRiUn(currentOp.GetSource(1), nextOp.GetSource(1), op1Base, op1Offset); + } + else + { + return false; + } + + return true; + } + + private static bool IsLoadOrStore(Operation operation) + { + return operation.Instruction == Instruction.Load || operation.Instruction == Instruction.Store; + } + + private static OperandType GetMemOpValueType(Operation operation) + { + if (operation.Destination != default) + { + return operation.Destination.Type; + } + + return operation.GetSource(1).Type; + } + + private static bool TryGetMemOpBaseAndOffset(Operation operation, out Operand baseAddress, out int offset) + { + baseAddress = default; + offset = 0; + Operand address = operation.GetSource(0); + + if (address.Kind != OperandKind.Memory) + { + return false; + } + + MemoryOperand memOp = address.GetMemory(); + Operand baseOp = memOp.BaseAddress; + + if (baseOp == default) + { + baseOp = memOp.Index; + + if (baseOp == default || memOp.Scale != Multiplier.x1) + { + return false; + } + } + if (memOp.Index != default) + { + return false; + } + + baseAddress = memOp.BaseAddress; + offset = memOp.Displacement; + + return true; + } + + private static Operand Register(Operand operand, OperandType type = OperandType.I64) + { + return Register(operand.GetRegister().Index, type); + } + + private static Operand Register(int register, OperandType type = OperandType.I64) + { + return Factory.Register(register, RegisterType.Integer, type); + } + + private static int Align16(int value) + { + return (value + 0xf) & ~0xf; + } + + [Conditional("DEBUG")] + private static void ValidateUnOp(Operand dest, Operand source) + { + // Destination and source aren't forced to be equals + // EnsureSameReg (dest, source); + EnsureSameType(dest, source); + } + + [Conditional("DEBUG")] + private static void ValidateBinOp(Operand dest, Operand src1, Operand src2) + { + // Destination and source aren't forced to be equals + // EnsureSameReg (dest, src1); + EnsureSameType(dest, src1, src2); + } + + [Conditional("DEBUG")] + private static void ValidateShift(Operand dest, Operand src1, Operand src2) + { + // Destination and source aren't forced to be equals + // EnsureSameReg (dest, src1); + EnsureSameType(dest, src1); + + Debug.Assert(dest.Type.IsInteger() && src2.Type == OperandType.I32); + } + + private static void EnsureSameReg(Operand op1, Operand op2) + { + Debug.Assert(op1.Kind == OperandKind.Register || op1.Kind == OperandKind.Memory); + Debug.Assert(op1.Kind == op2.Kind); + Debug.Assert(op1.Value == op2.Value); + } + + private static void EnsureSameType(Operand op1, Operand op2) + { + Debug.Assert(op1.Type == op2.Type); + } + + private static void EnsureSameType(Operand op1, Operand op2, Operand op3) + { + Debug.Assert(op1.Type == op2.Type); + Debug.Assert(op1.Type == op3.Type); + } + + private static void EnsureSameType(Operand op1, Operand op2, Operand op3, Operand op4) + { + Debug.Assert(op1.Type == op2.Type); + Debug.Assert(op1.Type == op3.Type); + Debug.Assert(op1.Type == op4.Type); + } + } +}
\ No newline at end of file |