diff options
| author | gdkchan <gab.dark.100@gmail.com> | 2023-01-10 19:16:59 -0300 |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-01-10 19:16:59 -0300 |
| commit | 5e0f8e873857ce3ca3f532aff0936beb28e412c8 (patch) | |
| tree | 576e5110c076b7d1f4d94e608ee21493f5b48879 /ARMeilleure/Instructions/InstEmitSimdArithmetic.cs | |
| parent | d16288a2a87f0979df30ba69d4fe10660177b6ac (diff) | |
Implement JIT Arm64 backend (#4114)1.1.536
* Implement JIT Arm64 backend
* PPTC version bump
* Address some feedback from Arm64 JIT PR
* Address even more PR feedback
* Remove unused IsPageAligned function
* Sync Qc flag before calls
* Fix comment and remove unused enum
* Address riperiperi PR feedback
* Delete Breakpoint IR instruction that was only implemented for Arm64
Diffstat (limited to 'ARMeilleure/Instructions/InstEmitSimdArithmetic.cs')
| -rw-r--r-- | ARMeilleure/Instructions/InstEmitSimdArithmetic.cs | 1492 |
1 files changed, 1222 insertions, 270 deletions
diff --git a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs index b91c522e..3e65db23 100644 --- a/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs +++ b/ARMeilleure/Instructions/InstEmitSimdArithmetic.cs @@ -21,22 +21,47 @@ namespace ARMeilleure.Instructions { public static void Abs_S(ArmEmitterContext context) { - EmitScalarUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOp(context, Intrinsic.Arm64AbsS); + } + else + { + EmitScalarUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + } } public static void Abs_V(ArmEmitterContext context) { - EmitVectorUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64AbsV); + } + else + { + EmitVectorUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + } } public static void Add_S(ArmEmitterContext context) { - EmitScalarBinaryOpZx(context, (op1, op2) => context.Add(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOp(context, Intrinsic.Arm64AddS); + } + else + { + EmitScalarBinaryOpZx(context, (op1, op2) => context.Add(op1, op2)); + } } public static void Add_V(ArmEmitterContext context) { - if (Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64AddV); + } + else if (Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -62,24 +87,42 @@ namespace ARMeilleure.Instructions public static void Addhn_V(ArmEmitterContext context) { - EmitHighNarrow(context, (op1, op2) => context.Add(op1, op2), round: false); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64AddhnV); + } + else + { + EmitHighNarrow(context, (op1, op2) => context.Add(op1, op2), round: false); + } } public static void Addp_S(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOp(context, Intrinsic.Arm64AddpS); + } + else + { + OpCodeSimd op = (OpCodeSimd)context.CurrOp; - Operand ne0 = EmitVectorExtractZx(context, op.Rn, 0, op.Size); - Operand ne1 = EmitVectorExtractZx(context, op.Rn, 1, op.Size); + Operand ne0 = EmitVectorExtractZx(context, op.Rn, 0, op.Size); + Operand ne1 = EmitVectorExtractZx(context, op.Rn, 1, op.Size); - Operand res = context.Add(ne0, ne1); + Operand res = context.Add(ne0, ne1); - context.Copy(GetVec(op.Rd), EmitVectorInsert(context, context.VectorZero(), res, 0, op.Size)); + context.Copy(GetVec(op.Rd), EmitVectorInsert(context, context.VectorZero(), res, 0, op.Size)); + } } public static void Addp_V(ArmEmitterContext context) { - if (Optimizations.UseSsse3) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64AddpV); + } + else if (Optimizations.UseSsse3) { EmitSsse3VectorPairwiseOp(context, X86PaddInstruction); } @@ -91,68 +134,89 @@ namespace ARMeilleure.Instructions public static void Addv_V(ArmEmitterContext context) { - EmitVectorAcrossVectorOpZx(context, (op1, op2) => context.Add(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64AddvV); + } + else + { + EmitVectorAcrossVectorOpZx(context, (op1, op2) => context.Add(op1, op2)); + } } public static void Cls_V(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64ClsV); + } + else + { + OpCodeSimd op = (OpCodeSimd)context.CurrOp; - Operand res = context.VectorZero(); + Operand res = context.VectorZero(); - int elems = op.GetBytesCount() >> op.Size; + int elems = op.GetBytesCount() >> op.Size; - int eSize = 8 << op.Size; + int eSize = 8 << op.Size; - for (int index = 0; index < elems; index++) - { - Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size); + for (int index = 0; index < elems; index++) + { + Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size); - Operand de = context.Call(typeof(SoftFallback).GetMethod(nameof(SoftFallback.CountLeadingSigns)), ne, Const(eSize)); + Operand de = context.Call(typeof(SoftFallback).GetMethod(nameof(SoftFallback.CountLeadingSigns)), ne, Const(eSize)); - res = EmitVectorInsert(context, res, de, index, op.Size); - } + res = EmitVectorInsert(context, res, de, index, op.Size); + } - context.Copy(GetVec(op.Rd), res); + context.Copy(GetVec(op.Rd), res); + } } public static void Clz_V(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; - - int eSize = 8 << op.Size; - - Operand res = eSize switch { - 8 => Clz_V_I8 (context, GetVec(op.Rn)), - 16 => Clz_V_I16(context, GetVec(op.Rn)), - 32 => Clz_V_I32(context, GetVec(op.Rn)), - _ => default - }; - - if (res != default) + if (Optimizations.UseAdvSimd) { - if (op.RegisterSize == RegisterSize.Simd64) - { - res = context.VectorZeroUpper64(res); - } + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64ClzV); } else { - int elems = op.GetBytesCount() >> op.Size; + OpCodeSimd op = (OpCodeSimd)context.CurrOp; - res = context.VectorZero(); + int eSize = 8 << op.Size; - for (int index = 0; index < elems; index++) + Operand res = eSize switch { + 8 => Clz_V_I8 (context, GetVec(op.Rn)), + 16 => Clz_V_I16(context, GetVec(op.Rn)), + 32 => Clz_V_I32(context, GetVec(op.Rn)), + _ => default + }; + + if (res != default) { - Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size); + if (op.RegisterSize == RegisterSize.Simd64) + { + res = context.VectorZeroUpper64(res); + } + } + else + { + int elems = op.GetBytesCount() >> op.Size; + + res = context.VectorZero(); - Operand de = context.Call(typeof(SoftFallback).GetMethod(nameof(SoftFallback.CountLeadingZeros)), ne, Const(eSize)); + for (int index = 0; index < elems; index++) + { + Operand ne = EmitVectorExtractZx(context, op.Rn, index, op.Size); - res = EmitVectorInsert(context, res, de, index, op.Size); + Operand de = context.Call(typeof(SoftFallback).GetMethod(nameof(SoftFallback.CountLeadingZeros)), ne, Const(eSize)); + + res = EmitVectorInsert(context, res, de, index, op.Size); + } } - } - context.Copy(GetVec(op.Rd), res); + context.Copy(GetVec(op.Rd), res); + } } private static Operand Clz_V_I8(ArmEmitterContext context, Operand arg) @@ -271,36 +335,47 @@ namespace ARMeilleure.Instructions public static void Cnt_V(ArmEmitterContext context) { - OpCodeSimd op = (OpCodeSimd)context.CurrOp; + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64CntV); + } + else + { + OpCodeSimd op = (OpCodeSimd)context.CurrOp; - Operand res = context.VectorZero(); + Operand res = context.VectorZero(); - int elems = op.RegisterSize == RegisterSize.Simd128 ? 16 : 8; + int elems = op.RegisterSize == RegisterSize.Simd128 ? 16 : 8; - for (int index = 0; index < elems; index++) - { - Operand ne = EmitVectorExtractZx(context, op.Rn, index, 0); + for (int index = 0; index < elems; index++) + { + Operand ne = EmitVectorExtractZx(context, op.Rn, index, 0); - Operand de; + Operand de; - if (Optimizations.UsePopCnt) - { - de = context.AddIntrinsicLong(Intrinsic.X86Popcnt, ne); - } - else - { - de = EmitCountSetBits8(context, ne); + if (Optimizations.UsePopCnt) + { + de = context.AddIntrinsicLong(Intrinsic.X86Popcnt, ne); + } + else + { + de = EmitCountSetBits8(context, ne); + } + + res = EmitVectorInsert(context, res, de, index, 0); } - res = EmitVectorInsert(context, res, de, index, 0); + context.Copy(GetVec(op.Rd), res); } - - context.Copy(GetVec(op.Rd), res); } public static void Fabd_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FabdS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -336,7 +411,11 @@ namespace ARMeilleure.Instructions public static void Fabd_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FabdV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -377,7 +456,11 @@ namespace ARMeilleure.Instructions public static void Fabs_S(ArmEmitterContext context) { - if (Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FabsS); + } + else if (Optimizations.UseSse2) { OpCodeSimd op = (OpCodeSimd)context.CurrOp; @@ -405,7 +488,11 @@ namespace ARMeilleure.Instructions public static void Fabs_V(ArmEmitterContext context) { - if (Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FabsV); + } + else if (Optimizations.UseSse2) { OpCodeSimd op = (OpCodeSimd)context.CurrOp; @@ -440,7 +527,11 @@ namespace ARMeilleure.Instructions public static void Fadd_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FaddS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitScalarBinaryOpF(context, Intrinsic.X86Addss, Intrinsic.X86Addsd); } @@ -459,7 +550,11 @@ namespace ARMeilleure.Instructions public static void Fadd_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FaddV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitVectorBinaryOpF(context, Intrinsic.X86Addps, Intrinsic.X86Addpd); } @@ -478,7 +573,11 @@ namespace ARMeilleure.Instructions public static void Faddp_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse3) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FaddpS); + } + else if (Optimizations.FastFP && Optimizations.UseSse3) { OpCodeSimd op = (OpCodeSimd)context.CurrOp; @@ -506,7 +605,11 @@ namespace ARMeilleure.Instructions public static void Faddp_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FaddpV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorPairwiseOpF(context, (op1, op2) => { @@ -534,7 +637,11 @@ namespace ARMeilleure.Instructions public static void Fdiv_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FdivS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitScalarBinaryOpF(context, Intrinsic.X86Divss, Intrinsic.X86Divsd); } @@ -553,7 +660,11 @@ namespace ARMeilleure.Instructions public static void Fdiv_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FdivV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitVectorBinaryOpF(context, Intrinsic.X86Divps, Intrinsic.X86Divpd); } @@ -572,7 +683,11 @@ namespace ARMeilleure.Instructions public static void Fmadd_S(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarTernaryOpF(context, Intrinsic.Arm64FmaddS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -607,7 +722,11 @@ namespace ARMeilleure.Instructions public static void Fmax_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FmaxS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41ProcessNaNsOpF(context, (op1, op2) => { @@ -628,7 +747,11 @@ namespace ARMeilleure.Instructions public static void Fmax_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FmaxV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41ProcessNaNsOpF(context, (op1, op2) => { @@ -649,7 +772,11 @@ namespace ARMeilleure.Instructions public static void Fmaxnm_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FmaxnmS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41MaxMinNumOpF(context, isMaxNum: true, scalar: true); } @@ -664,7 +791,11 @@ namespace ARMeilleure.Instructions public static void Fmaxnm_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FmaxnmV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41MaxMinNumOpF(context, isMaxNum: true, scalar: false); } @@ -679,7 +810,11 @@ namespace ARMeilleure.Instructions public static void Fmaxnmp_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FmaxnmpS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2ScalarPairwiseOpF(context, (op1, op2) => { @@ -697,7 +832,11 @@ namespace ARMeilleure.Instructions public static void Fmaxnmp_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FmaxnmpV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorPairwiseOpF(context, (op1, op2) => { @@ -715,7 +854,11 @@ namespace ARMeilleure.Instructions public static void Fmaxnmv_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FmaxnmvV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorAcrossVectorOpF(context, (op1, op2) => { @@ -733,7 +876,11 @@ namespace ARMeilleure.Instructions public static void Fmaxp_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FmaxpV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorPairwiseOpF(context, (op1, op2) => { @@ -757,7 +904,11 @@ namespace ARMeilleure.Instructions public static void Fmaxv_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FmaxvV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorAcrossVectorOpF(context, (op1, op2) => { @@ -781,7 +932,11 @@ namespace ARMeilleure.Instructions public static void Fmin_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FminS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41ProcessNaNsOpF(context, (op1, op2) => { @@ -802,7 +957,11 @@ namespace ARMeilleure.Instructions public static void Fmin_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FminV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41ProcessNaNsOpF(context, (op1, op2) => { @@ -823,7 +982,11 @@ namespace ARMeilleure.Instructions public static void Fminnm_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FminnmS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41MaxMinNumOpF(context, isMaxNum: false, scalar: true); } @@ -838,7 +1001,11 @@ namespace ARMeilleure.Instructions public static void Fminnm_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FminnmV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse41MaxMinNumOpF(context, isMaxNum: false, scalar: false); } @@ -853,7 +1020,11 @@ namespace ARMeilleure.Instructions public static void Fminnmp_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FminnmpS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2ScalarPairwiseOpF(context, (op1, op2) => { @@ -871,7 +1042,11 @@ namespace ARMeilleure.Instructions public static void Fminnmp_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FminnmpV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorPairwiseOpF(context, (op1, op2) => { @@ -889,7 +1064,11 @@ namespace ARMeilleure.Instructions public static void Fminnmv_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FminnmvV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorAcrossVectorOpF(context, (op1, op2) => { @@ -907,7 +1086,11 @@ namespace ARMeilleure.Instructions public static void Fminp_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FminpV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorPairwiseOpF(context, (op1, op2) => { @@ -931,7 +1114,11 @@ namespace ARMeilleure.Instructions public static void Fminv_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FminvV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { EmitSse2VectorAcrossVectorOpF(context, (op1, op2) => { @@ -955,15 +1142,26 @@ namespace ARMeilleure.Instructions public static void Fmla_Se(ArmEmitterContext context) // Fused. { - EmitScalarTernaryOpByElemF(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitScalarTernaryOpFRdByElem(context, Intrinsic.Arm64FmlaSe); + } + else + { + EmitScalarTernaryOpByElemF(context, (op1, op2, op3) => + { + return context.Add(op1, context.Multiply(op2, op3)); + }); + } } public static void Fmla_V(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpFRd(context, Intrinsic.Arm64FmlaV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1006,7 +1204,11 @@ namespace ARMeilleure.Instructions public static void Fmla_Ve(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpFRdByElem(context, Intrinsic.Arm64FmlaVe); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdRegElemF op = (OpCodeSimdRegElemF)context.CurrOp; @@ -1055,15 +1257,26 @@ namespace ARMeilleure.Instructions public static void Fmls_Se(ArmEmitterContext context) // Fused. { - EmitScalarTernaryOpByElemF(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Subtract(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitScalarTernaryOpFRdByElem(context, Intrinsic.Arm64FmlsSe); + } + else + { + EmitScalarTernaryOpByElemF(context, (op1, op2, op3) => + { + return context.Subtract(op1, context.Multiply(op2, op3)); + }); + } } public static void Fmls_V(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpFRd(context, Intrinsic.Arm64FmlsV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1106,7 +1319,11 @@ namespace ARMeilleure.Instructions public static void Fmls_Ve(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpFRdByElem(context, Intrinsic.Arm64FmlsVe); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdRegElemF op = (OpCodeSimdRegElemF)context.CurrOp; @@ -1155,7 +1372,11 @@ namespace ARMeilleure.Instructions public static void Fmsub_S(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarTernaryOpF(context, Intrinsic.Arm64FmsubS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1190,7 +1411,11 @@ namespace ARMeilleure.Instructions public static void Fmul_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FmulS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitScalarBinaryOpF(context, Intrinsic.X86Mulss, Intrinsic.X86Mulsd); } @@ -1209,12 +1434,23 @@ namespace ARMeilleure.Instructions public static void Fmul_Se(ArmEmitterContext context) { - EmitScalarBinaryOpByElemF(context, (op1, op2) => context.Multiply(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpFByElem(context, Intrinsic.Arm64FmulSe); + } + else + { + EmitScalarBinaryOpByElemF(context, (op1, op2) => context.Multiply(op1, op2)); + } } public static void Fmul_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FmulV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitVectorBinaryOpF(context, Intrinsic.X86Mulps, Intrinsic.X86Mulpd); } @@ -1233,7 +1469,11 @@ namespace ARMeilleure.Instructions public static void Fmul_Ve(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpFByElem(context, Intrinsic.Arm64FmulVe); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdRegElemF op = (OpCodeSimdRegElemF)context.CurrOp; @@ -1283,39 +1523,71 @@ namespace ARMeilleure.Instructions public static void Fmulx_S(ArmEmitterContext context) { - EmitScalarBinaryOpF(context, (op1, op2) => + if (Optimizations.UseAdvSimd) { - return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); - }); + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FmulxS); + } + else + { + EmitScalarBinaryOpF(context, (op1, op2) => + { + return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); + }); + } } public static void Fmulx_Se(ArmEmitterContext context) { - EmitScalarBinaryOpByElemF(context, (op1, op2) => + if (Optimizations.UseAdvSimd) { - return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); - }); + InstEmitSimdHelperArm64.EmitScalarBinaryOpFByElem(context, Intrinsic.Arm64FmulxSe); + } + else + { + EmitScalarBinaryOpByElemF(context, (op1, op2) => + { + return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); + }); + } } public static void Fmulx_V(ArmEmitterContext context) { - EmitVectorBinaryOpF(context, (op1, op2) => + if (Optimizations.UseAdvSimd) { - return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); - }); + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FmulxV); + } + else + { + EmitVectorBinaryOpF(context, (op1, op2) => + { + return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); + }); + } } public static void Fmulx_Ve(ArmEmitterContext context) { - EmitVectorBinaryOpByElemF(context, (op1, op2) => + if (Optimizations.UseAdvSimd) { - return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); - }); + InstEmitSimdHelperArm64.EmitVectorBinaryOpFByElem(context, Intrinsic.Arm64FmulxVe); + } + else + { + EmitVectorBinaryOpByElemF(context, (op1, op2) => + { + return EmitSoftFloatCall(context, nameof(SoftFloat32.FPMulX), op1, op2); + }); + } } public static void Fneg_S(ArmEmitterContext context) { - if (Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FnegS); + } + else if (Optimizations.UseSse2) { OpCodeSimd op = (OpCodeSimd)context.CurrOp; @@ -1344,7 +1616,11 @@ namespace ARMeilleure.Instructions public static void Fneg_V(ArmEmitterContext context) { - if (Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FnegV); + } + else if (Optimizations.UseSse2) { OpCodeSimd op = (OpCodeSimd)context.CurrOp; @@ -1380,7 +1656,11 @@ namespace ARMeilleure.Instructions public static void Fnmadd_S(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarTernaryOpF(context, Intrinsic.Arm64FnmaddS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1423,7 +1703,11 @@ namespace ARMeilleure.Instructions public static void Fnmsub_S(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarTernaryOpF(context, Intrinsic.Arm64FnmsubS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1466,7 +1750,14 @@ namespace ARMeilleure.Instructions public static void Fnmul_S(ArmEmitterContext context) { - EmitScalarBinaryOpF(context, (op1, op2) => context.Negate(context.Multiply(op1, op2))); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FnmulS); + } + else + { + EmitScalarBinaryOpF(context, (op1, op2) => context.Negate(context.Multiply(op1, op2))); + } } public static void Frecpe_S(ArmEmitterContext context) @@ -1475,7 +1766,11 @@ namespace ARMeilleure.Instructions int sizeF = op.Size & 1; - if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrecpeS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) { Operand res = EmitSse41Round32Exp8OpF(context, context.AddIntrinsic(Intrinsic.X86Rcpss, GetVec(op.Rn)), scalar: true); @@ -1496,7 +1791,11 @@ namespace ARMeilleure.Instructions int sizeF = op.Size & 1; - if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrecpeV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) { Operand res = EmitSse41Round32Exp8OpF(context, context.AddIntrinsic(Intrinsic.X86Rcpps, GetVec(op.Rn)), scalar: false); @@ -1518,7 +1817,11 @@ namespace ARMeilleure.Instructions public static void Frecps_S(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FrecpsS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1561,7 +1864,11 @@ namespace ARMeilleure.Instructions public static void Frecps_V(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FrecpsV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1609,15 +1916,26 @@ namespace ARMeilleure.Instructions public static void Frecpx_S(ArmEmitterContext context) { - EmitScalarUnaryOpF(context, (op1) => + if (Optimizations.UseAdvSimd) { - return EmitSoftFloatCall(context, nameof(SoftFloat32.FPRecpX), op1); - }); + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FrecpxS); + } + else + { + EmitScalarUnaryOpF(context, (op1) => + { + return EmitSoftFloatCall(context, nameof(SoftFloat32.FPRecpX), op1); + }); + } } public static void Frinta_S(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrintaS); + } + else if (Optimizations.UseSse41) { EmitSse41ScalarRoundOpF(context, FPRoundingMode.ToNearestAway); } @@ -1632,7 +1950,11 @@ namespace ARMeilleure.Instructions public static void Frinta_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrintaV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorRoundOpF(context, FPRoundingMode.ToNearestAway); } @@ -1647,23 +1969,41 @@ namespace ARMeilleure.Instructions public static void Frinti_S(ArmEmitterContext context) { - EmitScalarUnaryOpF(context, (op1) => + if (Optimizations.UseAdvSimd) { - return EmitRoundByRMode(context, op1); - }); + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrintiS); + } + else + { + EmitScalarUnaryOpF(context, (op1) => + { + return EmitRoundByRMode(context, op1); + }); + } } public static void Frinti_V(ArmEmitterContext context) { - EmitVectorUnaryOpF(context, (op1) => + if (Optimizations.UseAdvSimd) { - return EmitRoundByRMode(context, op1); - }); + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrintiV); + } + else + { + EmitVectorUnaryOpF(context, (op1) => + { + return EmitRoundByRMode(context, op1); + }); + } } public static void Frintm_S(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrintmS); + } + else if (Optimizations.UseSse41) { EmitSse41ScalarRoundOpF(context, FPRoundingMode.TowardsMinusInfinity); } @@ -1678,7 +2018,11 @@ namespace ARMeilleure.Instructions public static void Frintm_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrintmV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorRoundOpF(context, FPRoundingMode.TowardsMinusInfinity); } @@ -1693,7 +2037,11 @@ namespace ARMeilleure.Instructions public static void Frintn_S(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrintnS); + } + else if (Optimizations.UseSse41) { EmitSse41ScalarRoundOpF(context, FPRoundingMode.ToNearest); } @@ -1708,7 +2056,11 @@ namespace ARMeilleure.Instructions public static void Frintn_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrintnV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorRoundOpF(context, FPRoundingMode.ToNearest); } @@ -1723,7 +2075,11 @@ namespace ARMeilleure.Instructions public static void Frintp_S(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrintpS); + } + else if (Optimizations.UseSse41) { EmitSse41ScalarRoundOpF(context, FPRoundingMode.TowardsPlusInfinity); } @@ -1738,7 +2094,11 @@ namespace ARMeilleure.Instructions public static void Frintp_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrintpV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorRoundOpF(context, FPRoundingMode.TowardsPlusInfinity); } @@ -1753,6 +2113,7 @@ namespace ARMeilleure.Instructions public static void Frintx_S(ArmEmitterContext context) { + // TODO Arm64: Fast path. Should we set host FPCR? EmitScalarUnaryOpF(context, (op1) => { return EmitRoundByRMode(context, op1); @@ -1761,6 +2122,7 @@ namespace ARMeilleure.Instructions public static void Frintx_V(ArmEmitterContext context) { + // TODO Arm64: Fast path. Should we set host FPCR? EmitVectorUnaryOpF(context, (op1) => { return EmitRoundByRMode(context, op1); @@ -1769,7 +2131,11 @@ namespace ARMeilleure.Instructions public static void Frintz_S(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrintzS); + } + else if (Optimizations.UseSse41) { EmitSse41ScalarRoundOpF(context, FPRoundingMode.TowardsZero); } @@ -1784,7 +2150,11 @@ namespace ARMeilleure.Instructions public static void Frintz_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrintzV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorRoundOpF(context, FPRoundingMode.TowardsZero); } @@ -1803,7 +2173,11 @@ namespace ARMeilleure.Instructions int sizeF = op.Size & 1; - if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FrsqrteS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) { Operand res = EmitSse41Round32Exp8OpF(context, context.AddIntrinsic(Intrinsic.X86Rsqrtss, GetVec(op.Rn)), scalar: true); @@ -1824,7 +2198,11 @@ namespace ARMeilleure.Instructions int sizeF = op.Size & 1; - if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FrsqrteV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41 && sizeF == 0) { Operand res = EmitSse41Round32Exp8OpF(context, context.AddIntrinsic(Intrinsic.X86Rsqrtps, GetVec(op.Rn)), scalar: false); @@ -1846,7 +2224,11 @@ namespace ARMeilleure.Instructions public static void Frsqrts_S(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FrsqrtsS); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1895,7 +2277,11 @@ namespace ARMeilleure.Instructions public static void Frsqrts_V(ArmEmitterContext context) // Fused. { - if (Optimizations.FastFP && Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FrsqrtsV); + } + else if (Optimizations.FastFP && Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -1949,7 +2335,11 @@ namespace ARMeilleure.Instructions public static void Fsqrt_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOpF(context, Intrinsic.Arm64FsqrtS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitScalarUnaryOpF(context, Intrinsic.X86Sqrtss, Intrinsic.X86Sqrtsd); } @@ -1964,7 +2354,11 @@ namespace ARMeilleure.Instructions public static void Fsqrt_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOpF(context, Intrinsic.Arm64FsqrtV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitVectorUnaryOpF(context, Intrinsic.X86Sqrtps, Intrinsic.X86Sqrtpd); } @@ -1979,7 +2373,11 @@ namespace ARMeilleure.Instructions public static void Fsub_S(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOpF(context, Intrinsic.Arm64FsubS); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitScalarBinaryOpF(context, Intrinsic.X86Subss, Intrinsic.X86Subsd); } @@ -1998,7 +2396,11 @@ namespace ARMeilleure.Instructions public static void Fsub_V(ArmEmitterContext context) { - if (Optimizations.FastFP && Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpF(context, Intrinsic.Arm64FsubV); + } + else if (Optimizations.FastFP && Optimizations.UseSse2) { EmitVectorBinaryOpF(context, Intrinsic.X86Subps, Intrinsic.X86Subpd); } @@ -2017,7 +2419,11 @@ namespace ARMeilleure.Instructions public static void Mla_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64MlaV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorMul_AddSub(context, AddSub.Add); } @@ -2032,15 +2438,26 @@ namespace ARMeilleure.Instructions public static void Mla_Ve(ArmEmitterContext context) { - EmitVectorTernaryOpByElemZx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRdByElem(context, Intrinsic.Arm64MlaVe); + } + else + { + EmitVectorTernaryOpByElemZx(context, (op1, op2, op3) => + { + return context.Add(op1, context.Multiply(op2, op3)); + }); + } } public static void Mls_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64MlsV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorMul_AddSub(context, AddSub.Subtract); } @@ -2055,15 +2472,26 @@ namespace ARMeilleure.Instructions public static void Mls_Ve(ArmEmitterContext context) { - EmitVectorTernaryOpByElemZx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Subtract(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRdByElem(context, Intrinsic.Arm64MlsVe); + } + else + { + EmitVectorTernaryOpByElemZx(context, (op1, op2, op3) => + { + return context.Subtract(op1, context.Multiply(op2, op3)); + }); + } } public static void Mul_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64MulV); + } + else if (Optimizations.UseSse41) { EmitSse41VectorMul_AddSub(context, AddSub.None); } @@ -2075,17 +2503,35 @@ namespace ARMeilleure.Instructions public static void Mul_Ve(ArmEmitterContext context) { - EmitVectorBinaryOpByElemZx(context, (op1, op2) => context.Multiply(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpByElem(context, Intrinsic.Arm64MulVe); + } + else + { + EmitVectorBinaryOpByElemZx(context, (op1, op2) => context.Multiply(op1, op2)); + } } public static void Neg_S(ArmEmitterContext context) { - EmitScalarUnaryOpSx(context, (op1) => context.Negate(op1)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarUnaryOp(context, Intrinsic.Arm64NegS); + } + else + { + EmitScalarUnaryOpSx(context, (op1) => context.Negate(op1)); + } } public static void Neg_V(ArmEmitterContext context) { - if (Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64NegV); + } + else if (Optimizations.UseSse2) { OpCodeSimd op = (OpCodeSimd)context.CurrOp; @@ -2110,7 +2556,11 @@ namespace ARMeilleure.Instructions { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UsePclmulqdq && op.Size == 3) + if (Optimizations.UseAdvSimd && false) // Not supported by all Arm CPUs. + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64PmullV); + } + else if (Optimizations.UsePclmulqdq && op.Size == 3) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -2214,33 +2664,65 @@ namespace ARMeilleure.Instructions public static void Raddhn_V(ArmEmitterContext context) { - EmitHighNarrow(context, (op1, op2) => context.Add(op1, op2), round: true); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64RaddhnV); + } + else + { + EmitHighNarrow(context, (op1, op2) => context.Add(op1, op2), round: true); + } } public static void Rsubhn_V(ArmEmitterContext context) { - EmitHighNarrow(context, (op1, op2) => context.Subtract(op1, op2), round: true); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64RsubhnV); + } + else + { + EmitHighNarrow(context, (op1, op2) => context.Subtract(op1, op2), round: true); + } } public static void Saba_V(ArmEmitterContext context) { - EmitVectorTernaryOpSx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64SabaV); + } + else + { + EmitVectorTernaryOpSx(context, (op1, op2, op3) => + { + return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); + }); + } } public static void Sabal_V(ArmEmitterContext context) { - EmitVectorWidenRnRmTernaryOpSx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64SabalV); + } + else + { + EmitVectorWidenRnRmTernaryOpSx(context, (op1, op2, op3) => + { + return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); + }); + } } public static void Sabd_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SabdV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2262,7 +2744,11 @@ namespace ARMeilleure.Instructions { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse41 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SabdlV); + } + else if (Optimizations.UseSse41 && op.Size < 2) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -2293,12 +2779,23 @@ namespace ARMeilleure.Instructions public static void Sadalp_V(ArmEmitterContext context) { - EmitAddLongPairwise(context, signed: true, accumulate: true); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpRd(context, Intrinsic.Arm64SadalpV); + } + else + { + EmitAddLongPairwise(context, signed: true, accumulate: true); + } } public static void Saddl_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SaddlV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2328,17 +2825,35 @@ namespace ARMeilleure.Instructions public static void Saddlp_V(ArmEmitterContext context) { - EmitAddLongPairwise(context, signed: true, accumulate: false); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64SaddlpV); + } + else + { + EmitAddLongPairwise(context, signed: true, accumulate: false); + } } public static void Saddlv_V(ArmEmitterContext context) { - EmitVectorLongAcrossVectorOpSx(context, (op1, op2) => context.Add(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64SaddlvV); + } + else + { + EmitVectorLongAcrossVectorOpSx(context, (op1, op2) => context.Add(op1, op2)); + } } public static void Saddw_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SaddwV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2368,7 +2883,11 @@ namespace ARMeilleure.Instructions { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse2 && op.Size > 0) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64ShaddV); + } + else if (Optimizations.UseSse2 && op.Size > 0) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -2404,7 +2923,11 @@ namespace ARMeilleure.Instructions { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse2 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64ShsubV); + } + else if (Optimizations.UseSse2 && op.Size < 2) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -2442,7 +2965,11 @@ namespace ARMeilleure.Instructions public static void Smax_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SmaxV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2468,7 +2995,11 @@ namespace ARMeilleure.Instructions public static void Smaxp_V(ArmEmitterContext context) { - if (Optimizations.UseSsse3) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SmaxpV); + } + else if (Optimizations.UseSsse3) { EmitSsse3VectorPairwiseOp(context, X86PmaxsInstruction); } @@ -2480,12 +3011,23 @@ namespace ARMeilleure.Instructions public static void Smaxv_V(ArmEmitterContext context) { - EmitVectorAcrossVectorOpSx(context, (op1, op2) => EmitMax64Op(context, op1, op2, signed: true)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64SmaxvV); + } + else + { + EmitVectorAcrossVectorOpSx(context, (op1, op2) => EmitMax64Op(context, op1, op2, signed: true)); + } } public static void Smin_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SminV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2511,7 +3053,11 @@ namespace ARMeilleure.Instructions public static void Sminp_V(ArmEmitterContext context) { - if (Optimizations.UseSsse3) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SminpV); + } + else if (Optimizations.UseSsse3) { EmitSsse3VectorPairwiseOp(context, X86PminsInstruction); } @@ -2523,14 +3069,25 @@ namespace ARMeilleure.Instructions public static void Sminv_V(ArmEmitterContext context) { - EmitVectorAcrossVectorOpSx(context, (op1, op2) => EmitMin64Op(context, op1, op2, signed: true)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64SminvV); + } + else + { + EmitVectorAcrossVectorOpSx(context, (op1, op2) => EmitMin64Op(context, op1, op2, signed: true)); + } } public static void Smlal_V(ArmEmitterContext context) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse41 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64SmlalV); + } + else if (Optimizations.UseSse41 && op.Size < 2) { Operand d = GetVec(op.Rd); Operand n = GetVec(op.Rn); @@ -2566,17 +3123,28 @@ namespace ARMeilleure.Instructions public static void Smlal_Ve(ArmEmitterContext context) { - EmitVectorWidenTernaryOpByElemSx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRdByElem(context, Intrinsic.Arm64SmlalVe); + } + else + { + EmitVectorWidenTernaryOpByElemSx(context, (op1, op2, op3) => + { + return context.Add(op1, context.Multiply(op2, op3)); + }); + } } public static void Smlsl_V(ArmEmitterContext context) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse41 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64SmlslV); + } + else if (Optimizations.UseSse41 && op.Size < 2) { Operand d = GetVec(op.Rd); Operand n = GetVec(op.Rn); @@ -2612,117 +3180,268 @@ namespace ARMeilleure.Instructions public static void Smlsl_Ve(ArmEmitterContext context) { - EmitVectorWidenTernaryOpByElemSx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Subtract(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRdByElem(context, Intrinsic.Arm64SmlslVe); + } + else + { + EmitVectorWidenTernaryOpByElemSx(context, (op1, op2, op3) => + { + return context.Subtract(op1, context.Multiply(op2, op3)); + }); + } } public static void Smull_V(ArmEmitterContext context) { - EmitVectorWidenRnRmBinaryOpSx(context, (op1, op2) => context.Multiply(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SmullV); + } + else + { + EmitVectorWidenRnRmBinaryOpSx(context, (op1, op2) => context.Multiply(op1, op2)); + } } public static void Smull_Ve(ArmEmitterContext context) { - EmitVectorWidenBinaryOpByElemSx(context, (op1, op2) => context.Multiply(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpByElem(context, Intrinsic.Arm64SmullVe); + } + else + { + EmitVectorWidenBinaryOpByElemSx(context, (op1, op2) => context.Multiply(op1, op2)); + } } public static void Sqabs_S(ArmEmitterContext context) { - EmitScalarSaturatingUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingUnaryOp(context, Intrinsic.Arm64SqabsS); + } + else + { + EmitScalarSaturatingUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + } } public static void Sqabs_V(ArmEmitterContext context) { - EmitVectorSaturatingUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingUnaryOp(context, Intrinsic.Arm64SqabsV); + } + else + { + EmitVectorSaturatingUnaryOpSx(context, (op1) => EmitAbs(context, op1)); + } } public static void Sqadd_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpSx(context, flags: SaturatingFlags.Add); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOp(context, Intrinsic.Arm64SqaddS); + } + else + { + EmitScalarSaturatingBinaryOpSx(context, flags: SaturatingFlags.Add); + } } public static void Sqadd_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpSx(context, flags: SaturatingFlags.Add); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOp(context, Intrinsic.Arm64SqaddV); + } + else + { + EmitVectorSaturatingBinaryOpSx(context, flags: SaturatingFlags.Add); + } } public static void Sqdmulh_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: false)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOp(context, Intrinsic.Arm64SqdmulhS); + } + else + { + EmitScalarSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: false)); + } } public static void Sqdmulh_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: false)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOp(context, Intrinsic.Arm64SqdmulhV); + } + else + { + EmitVectorSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: false)); + } } public static void Sqdmulh_Ve(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpByElemSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: false)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOpByElem(context, Intrinsic.Arm64SqdmulhVe); + } + else + { + EmitVectorSaturatingBinaryOpByElemSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: false)); + } } public static void Sqneg_S(ArmEmitterContext context) { - EmitScalarSaturatingUnaryOpSx(context, (op1) => context.Negate(op1)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingUnaryOp(context, Intrinsic.Arm64SqnegS); + } + else + { + EmitScalarSaturatingUnaryOpSx(context, (op1) => context.Negate(op1)); + } } public static void Sqneg_V(ArmEmitterContext context) { - EmitVectorSaturatingUnaryOpSx(context, (op1) => context.Negate(op1)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingUnaryOp(context, Intrinsic.Arm64SqnegV); + } + else + { + EmitVectorSaturatingUnaryOpSx(context, (op1) => context.Negate(op1)); + } } public static void Sqrdmulh_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: true)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOp(context, Intrinsic.Arm64SqrdmulhS); + } + else + { + EmitScalarSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: true)); + } } public static void Sqrdmulh_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: true)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOp(context, Intrinsic.Arm64SqrdmulhV); + } + else + { + EmitVectorSaturatingBinaryOpSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: true)); + } } public static void Sqrdmulh_Ve(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpByElemSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: true)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOpByElem(context, Intrinsic.Arm64SqrdmulhVe); + } + else + { + EmitVectorSaturatingBinaryOpByElemSx(context, (op1, op2) => EmitDoublingMultiplyHighHalf(context, op1, op2, round: true)); + } } public static void Sqsub_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpSx(context, flags: SaturatingFlags.Sub); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOp(context, Intrinsic.Arm64SqsubS); + } + else + { + EmitScalarSaturatingBinaryOpSx(context, flags: SaturatingFlags.Sub); + } } public static void Sqsub_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpSx(context, flags: SaturatingFlags.Sub); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOp(context, Intrinsic.Arm64SqsubV); + } + else + { + EmitVectorSaturatingBinaryOpSx(context, flags: SaturatingFlags.Sub); + } } public static void Sqxtn_S(ArmEmitterContext context) { - EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.ScalarSxSx); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOpRd(context, Intrinsic.Arm64SqxtnS); + } + else + { + EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.ScalarSxSx); + } } public static void Sqxtn_V(ArmEmitterContext context) { - EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.VectorSxSx); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOpRd(context, Intrinsic.Arm64SqxtnV); + } + else + { + EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.VectorSxSx); + } } public static void Sqxtun_S(ArmEmitterContext context) { - EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.ScalarSxZx); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOpRd(context, Intrinsic.Arm64SqxtunS); + } + else + { + EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.ScalarSxZx); + } } public static void Sqxtun_V(ArmEmitterContext context) { - EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.VectorSxZx); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOpRd(context, Intrinsic.Arm64SqxtunV); + } + else + { + EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.VectorSxZx); + } } public static void Srhadd_V(ArmEmitterContext context) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse2 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SrhaddV); + } + else if (Optimizations.UseSse2 && op.Size < 2) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -2764,7 +3483,11 @@ namespace ARMeilleure.Instructions public static void Ssubl_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SsublV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2794,7 +3517,11 @@ namespace ARMeilleure.Instructions public static void Ssubw_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SsubwV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2822,12 +3549,23 @@ namespace ARMeilleure.Instructions public static void Sub_S(ArmEmitterContext context) { - EmitScalarBinaryOpZx(context, (op1, op2) => context.Subtract(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarBinaryOp(context, Intrinsic.Arm64SubS); + } + else + { + EmitScalarBinaryOpZx(context, (op1, op2) => context.Subtract(op1, op2)); + } } public static void Sub_V(ArmEmitterContext context) { - if (Optimizations.UseSse2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64SubV); + } + else if (Optimizations.UseSse2) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2853,38 +3591,77 @@ namespace ARMeilleure.Instructions public static void Subhn_V(ArmEmitterContext context) { - EmitHighNarrow(context, (op1, op2) => context.Subtract(op1, op2), round: false); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64SubhnV); + } + else + { + EmitHighNarrow(context, (op1, op2) => context.Subtract(op1, op2), round: false); + } } public static void Suqadd_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpSx(context, flags: SaturatingFlags.Accumulate); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOpRd(context, Intrinsic.Arm64SuqaddS); + } + else + { + EmitScalarSaturatingBinaryOpSx(context, flags: SaturatingFlags.Accumulate); + } } public static void Suqadd_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpSx(context, flags: SaturatingFlags.Accumulate); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOpRd(context, Intrinsic.Arm64SuqaddV); + } + else + { + EmitVectorSaturatingBinaryOpSx(context, flags: SaturatingFlags.Accumulate); + } } public static void Uaba_V(ArmEmitterContext context) { - EmitVectorTernaryOpZx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64UabaV); + } + else + { + EmitVectorTernaryOpZx(context, (op1, op2, op3) => + { + return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); + }); + } } public static void Uabal_V(ArmEmitterContext context) { - EmitVectorWidenRnRmTernaryOpZx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64UabalV); + } + else + { + EmitVectorWidenRnRmTernaryOpZx(context, (op1, op2, op3) => + { + return context.Add(op1, EmitAbs(context, context.Subtract(op2, op3))); + }); + } } public static void Uabd_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UabdV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2906,7 +3683,11 @@ namespace ARMeilleure.Instructions { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse41 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UabdlV); + } + else if (Optimizations.UseSse41 && op.Size < 2) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -2937,12 +3718,23 @@ namespace ARMeilleure.Instructions public static void Uadalp_V(ArmEmitterContext context) { - EmitAddLongPairwise(context, signed: false, accumulate: true); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpRd(context, Intrinsic.Arm64UadalpV); + } + else + { + EmitAddLongPairwise(context, signed: false, accumulate: true); + } } public static void Uaddl_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UaddlV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -2972,17 +3764,35 @@ namespace ARMeilleure.Instructions public static void Uaddlp_V(ArmEmitterContext context) { - EmitAddLongPairwise(context, signed: false, accumulate: false); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64UaddlpV); + } + else + { + EmitAddLongPairwise(context, signed: false, accumulate: false); + } } public static void Uaddlv_V(ArmEmitterContext context) { - EmitVectorLongAcrossVectorOpZx(context, (op1, op2) => context.Add(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64UaddlvV); + } + else + { + EmitVectorLongAcrossVectorOpZx(context, (op1, op2) => context.Add(op1, op2)); + } } public static void Uaddw_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UaddwV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -3012,7 +3822,11 @@ namespace ARMeilleure.Instructions { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse2 && op.Size > 0) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UhaddV); + } + else if (Optimizations.UseSse2 && op.Size > 0) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -3048,7 +3862,11 @@ namespace ARMeilleure.Instructions { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse2 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UhsubV); + } + else if (Optimizations.UseSse2 && op.Size < 2) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -3079,7 +3897,11 @@ namespace ARMeilleure.Instructions public static void Umax_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UmaxV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -3105,7 +3927,11 @@ namespace ARMeilleure.Instructions public static void Umaxp_V(ArmEmitterContext context) { - if (Optimizations.UseSsse3) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UmaxpV); + } + else if (Optimizations.UseSsse3) { EmitSsse3VectorPairwiseOp(context, X86PmaxuInstruction); } @@ -3117,12 +3943,23 @@ namespace ARMeilleure.Instructions public static void Umaxv_V(ArmEmitterContext context) { - EmitVectorAcrossVectorOpZx(context, (op1, op2) => EmitMax64Op(context, op1, op2, signed: false)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64UmaxvV); + } + else + { + EmitVectorAcrossVectorOpZx(context, (op1, op2) => EmitMax64Op(context, op1, op2, signed: false)); + } } public static void Umin_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UminV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -3148,7 +3985,11 @@ namespace ARMeilleure.Instructions public static void Uminp_V(ArmEmitterContext context) { - if (Optimizations.UseSsse3) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UminpV); + } + else if (Optimizations.UseSsse3) { EmitSsse3VectorPairwiseOp(context, X86PminuInstruction); } @@ -3160,14 +4001,25 @@ namespace ARMeilleure.Instructions public static void Uminv_V(ArmEmitterContext context) { - EmitVectorAcrossVectorOpZx(context, (op1, op2) => EmitMin64Op(context, op1, op2, signed: false)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorUnaryOp(context, Intrinsic.Arm64UminvV); + } + else + { + EmitVectorAcrossVectorOpZx(context, (op1, op2) => EmitMin64Op(context, op1, op2, signed: false)); + } } public static void Umlal_V(ArmEmitterContext context) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse41 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64UmlalV); + } + else if (Optimizations.UseSse41 && op.Size < 2) { Operand d = GetVec(op.Rd); Operand n = GetVec(op.Rn); @@ -3203,17 +4055,28 @@ namespace ARMeilleure.Instructions public static void Umlal_Ve(ArmEmitterContext context) { - EmitVectorWidenTernaryOpByElemZx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Add(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRdByElem(context, Intrinsic.Arm64UmlalVe); + } + else + { + EmitVectorWidenTernaryOpByElemZx(context, (op1, op2, op3) => + { + return context.Add(op1, context.Multiply(op2, op3)); + }); + } } public static void Umlsl_V(ArmEmitterContext context) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse41 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorTernaryOpRd(context, Intrinsic.Arm64UmlslV); + } + else if (Optimizations.UseSse41 && op.Size < 2) { Operand d = GetVec(op.Rd); Operand n = GetVec(op.Rn); @@ -3249,57 +4112,124 @@ namespace ARMeilleure.Instructions public static void Umlsl_Ve(ArmEmitterContext context) { - EmitVectorWidenTernaryOpByElemZx(context, (op1, op2, op3) => + if (Optimizations.UseAdvSimd) { - return context.Subtract(op1, context.Multiply(op2, op3)); - }); + InstEmitSimdHelperArm64.EmitVectorTernaryOpRdByElem(context, Intrinsic.Arm64UmlslVe); + } + else + { + EmitVectorWidenTernaryOpByElemZx(context, (op1, op2, op3) => + { + return context.Subtract(op1, context.Multiply(op2, op3)); + }); + } } public static void Umull_V(ArmEmitterContext context) { - EmitVectorWidenRnRmBinaryOpZx(context, (op1, op2) => context.Multiply(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UmullV); + } + else + { + EmitVectorWidenRnRmBinaryOpZx(context, (op1, op2) => context.Multiply(op1, op2)); + } } public static void Umull_Ve(ArmEmitterContext context) { - EmitVectorWidenBinaryOpByElemZx(context, (op1, op2) => context.Multiply(op1, op2)); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOpByElem(context, Intrinsic.Arm64UmullVe); + } + else + { + EmitVectorWidenBinaryOpByElemZx(context, (op1, op2) => context.Multiply(op1, op2)); + } } public static void Uqadd_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpZx(context, SaturatingFlags.Add); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOp(context, Intrinsic.Arm64UqaddS); + } + else + { + EmitScalarSaturatingBinaryOpZx(context, SaturatingFlags.Add); + } } public static void Uqadd_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpZx(context, SaturatingFlags.Add); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOp(context, Intrinsic.Arm64UqaddV); + } + else + { + EmitVectorSaturatingBinaryOpZx(context, SaturatingFlags.Add); + } } public static void Uqsub_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpZx(context, SaturatingFlags.Sub); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOp(context, Intrinsic.Arm64UqsubS); + } + else + { + EmitScalarSaturatingBinaryOpZx(context, SaturatingFlags.Sub); + } } public static void Uqsub_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpZx(context, SaturatingFlags.Sub); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOp(context, Intrinsic.Arm64UqsubV); + } + else + { + EmitVectorSaturatingBinaryOpZx(context, SaturatingFlags.Sub); + } } public static void Uqxtn_S(ArmEmitterContext context) { - EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.ScalarZxZx); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOpRd(context, Intrinsic.Arm64UqxtnS); + } + else + { + EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.ScalarZxZx); + } } public static void Uqxtn_V(ArmEmitterContext context) { - EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.VectorZxZx); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOpRd(context, Intrinsic.Arm64UqxtnV); + } + else + { + EmitSaturatingNarrowOp(context, SaturatingNarrowFlags.VectorZxZx); + } } public static void Urhadd_V(ArmEmitterContext context) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; - if (Optimizations.UseSse2 && op.Size < 2) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UrhaddV); + } + else if (Optimizations.UseSse2 && op.Size < 2) { Operand n = GetVec(op.Rn); Operand m = GetVec(op.Rm); @@ -3330,17 +4260,35 @@ namespace ARMeilleure.Instructions public static void Usqadd_S(ArmEmitterContext context) { - EmitScalarSaturatingBinaryOpZx(context, SaturatingFlags.Accumulate); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitScalarSaturatingBinaryOpRd(context, Intrinsic.Arm64UsqaddS); + } + else + { + EmitScalarSaturatingBinaryOpZx(context, SaturatingFlags.Accumulate); + } } public static void Usqadd_V(ArmEmitterContext context) { - EmitVectorSaturatingBinaryOpZx(context, SaturatingFlags.Accumulate); + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorSaturatingBinaryOpRd(context, Intrinsic.Arm64UsqaddV); + } + else + { + EmitVectorSaturatingBinaryOpZx(context, SaturatingFlags.Accumulate); + } } public static void Usubl_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UsublV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; @@ -3370,7 +4318,11 @@ namespace ARMeilleure.Instructions public static void Usubw_V(ArmEmitterContext context) { - if (Optimizations.UseSse41) + if (Optimizations.UseAdvSimd) + { + InstEmitSimdHelperArm64.EmitVectorBinaryOp(context, Intrinsic.Arm64UsubwV); + } + else if (Optimizations.UseSse41) { OpCodeSimdReg op = (OpCodeSimdReg)context.CurrOp; |