7 files changed, 184 insertions, 18 deletions
diff --git a/ARMeilleure/Translation/ArmEmitterContext.cs b/ARMeilleure/Translation/ArmEmitterContext.cs
index 48254de4..238f8508 100644
--- a/ARMeilleure/Translation/ArmEmitterContext.cs
+++ b/ARMeilleure/Translation/ArmEmitterContext.cs
@@ -39,6 +39,8 @@ namespace ARMeilleure.Translation
             }
         }
 
+        private bool _pendingQcFlagSync;
+
         public OpCode CurrOp { get; set; }
 
         public IMemoryManager Memory { get; }
@@ -81,6 +83,8 @@ namespace ARMeilleure.Translation
 
         public override Operand Call(MethodInfo info, params Operand[] callArgs)
         {
+            SyncQcFlag();
+
             if (!HasPtc)
             {
                 return base.Call(info, callArgs);
@@ -139,6 +143,51 @@ namespace ARMeilleure.Translation
             _optOpLastFlagSet = null;
         }
 
+        public void SetPendingQcFlagSync()
+        {
+            _pendingQcFlagSync = true;
+        }
+
+        public void SyncQcFlag()
+        {
+            if (_pendingQcFlagSync)
+            {
+                if (Optimizations.UseAdvSimd)
+                {
+                    Operand fpsr = AddIntrinsicInt(Intrinsic.Arm64MrsFpsr);
+
+                    uint qcFlagMask = (uint)FPSR.Qc;
+
+                    Operand qcClearLabel = Label();
+
+                    BranchIfFalse(qcClearLabel, BitwiseAnd(fpsr, Const(qcFlagMask)));
+
+                    AddIntrinsicNoRet(Intrinsic.Arm64MsrFpsr, Const(0));
+                    InstEmitHelper.SetFpFlag(this, FPState.QcFlag, Const(1));
+
+                    MarkLabel(qcClearLabel);
+                }
+
+                _pendingQcFlagSync = false;
+            }
+        }
+
+        public void ClearQcFlag()
+        {
+            if (Optimizations.UseAdvSimd)
+            {
+                AddIntrinsicNoRet(Intrinsic.Arm64MsrFpsr, Const(0));
+            }
+        }
+
+        public void ClearQcFlagIfModified()
+        {
+            if (_pendingQcFlagSync && Optimizations.UseAdvSimd)
+            {
+                AddIntrinsicNoRet(Intrinsic.Arm64MsrFpsr, Const(0));
+            }
+        }
+
         public Operand TryGetComparisonResult(Condition condition)
         {
             if (_optOpLastCompare == null || _optOpLastCompare != _optOpLastFlagSet)
diff --git a/ARMeilleure/Translation/Cache/JitCache.cs b/ARMeilleure/Translation/Cache/JitCache.cs
index 24affa34..f496a8e9 100644
--- a/ARMeilleure/Translation/Cache/JitCache.cs
+++ b/ARMeilleure/Translation/Cache/JitCache.cs
@@ -1,6 +1,7 @@
 using ARMeilleure.CodeGen;
 using ARMeilleure.CodeGen.Unwinding;
 using ARMeilleure.Memory;
+using ARMeilleure.Native;
 using System;
 using System.Collections.Generic;
 using System.Diagnostics;
@@ -17,6 +18,7 @@ namespace ARMeilleure.Translation.Cache
         private const int CacheSize = 2047 * 1024 * 1024;
 
         private static ReservedRegion _jitRegion;
+        private static JitCacheInvalidation _jitCacheInvalidator;
 
         private static CacheMemoryAllocator _cacheAllocator;
 
@@ -25,8 +27,6 @@ namespace ARMeilleure.Translation.Cache
         private static readonly object _lock = new object();
         private static bool _initialized;
 
-        public static IntPtr Base => _jitRegion.Pointer;
-
         public static void Initialize(IJitMemoryAllocator allocator)
         {
             if (_initialized) return;
@@ -36,6 +36,7 @@ namespace ARMeilleure.Translation.Cache
                 if (_initialized) return;
 
                 _jitRegion = new ReservedRegion(allocator, CacheSize);
+                _jitCacheInvalidator = new JitCacheInvalidation(allocator);
 
                 _cacheAllocator = new CacheMemoryAllocator(CacheSize);
 
@@ -60,11 +61,24 @@ namespace ARMeilleure.Translation.Cache
 
                 IntPtr funcPtr = _jitRegion.Pointer + funcOffset;
 
-                ReprotectAsWritable(funcOffset, code.Length);
-
-                Marshal.Copy(code, 0, funcPtr, code.Length);
+                if (OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
+                {
+                    unsafe
+                    {
+                        fixed (byte *codePtr = code)
+                        {
+                            JitSupportDarwin.Copy(funcPtr, (IntPtr)codePtr, (ulong)code.Length);
+                        }
+                    }
+                }
+                else
+                {
+                    ReprotectAsWritable(funcOffset, code.Length);
+                    Marshal.Copy(code, 0, funcPtr, code.Length);
+                    ReprotectAsExecutable(funcOffset, code.Length);
 
-                ReprotectAsExecutable(funcOffset, code.Length);
+                    _jitCacheInvalidator.Invalidate(funcPtr, (ulong)code.Length);
+                }
 
                 Add(funcOffset, code.Length, func.UnwindInfo);
 
diff --git a/ARMeilleure/Translation/Cache/JitCacheInvalidation.cs b/ARMeilleure/Translation/Cache/JitCacheInvalidation.cs
new file mode 100644
index 00000000..ec2ae73b
--- /dev/null
+++ b/ARMeilleure/Translation/Cache/JitCacheInvalidation.cs
@@ -0,0 +1,79 @@
+using ARMeilleure.Memory;
+using System;
+using System.Runtime.InteropServices;
+
+namespace ARMeilleure.Translation.Cache
+{
+    class JitCacheInvalidation
+    {
+        private static int[] _invalidationCode = new int[]
+        {
+            unchecked((int)0xd53b0022), // mrs  x2, ctr_el0
+            unchecked((int)0xd3504c44), // ubfx x4, x2, #16, #4
+            unchecked((int)0x52800083), // mov  w3, #0x4
+            unchecked((int)0x12000c45), // and  w5, w2, #0xf
+            unchecked((int)0x1ac42064), // lsl  w4, w3, w4
+            unchecked((int)0x51000482), // sub  w2, w4, #0x1
+            unchecked((int)0x8a220002), // bic  x2, x0, x2
+            unchecked((int)0x1ac52063), // lsl  w3, w3, w5
+            unchecked((int)0xeb01005f), // cmp  x2, x1
+            unchecked((int)0x93407c84), // sxtw x4, w4
+            unchecked((int)0x540000a2), // b.cs 3c <do_ic_clear>
+            unchecked((int)0xd50b7b22), // dc   cvau, x2
+            unchecked((int)0x8b040042), // add  x2, x2, x4
+            unchecked((int)0xeb02003f), // cmp  x1, x2
+            unchecked((int)0x54ffffa8), // b.hi 2c <dc_clear_loop>
+            unchecked((int)0xd5033b9f), // dsb  ish
+            unchecked((int)0x51000462), // sub  w2, w3, #0x1
+            unchecked((int)0x93407c63), // sxtw x3, w3
+            unchecked((int)0x8a220000), // bic  x0, x0, x2
+            unchecked((int)0xeb00003f), // cmp  x1, x0
+            unchecked((int)0x540000a9), // b.ls 64 <exit>
+            unchecked((int)0xd50b7520), // ic   ivau, x0
+            unchecked((int)0x8b030000), // add  x0, x0, x3
+            unchecked((int)0xeb00003f), // cmp  x1, x0
+            unchecked((int)0x54ffffa8), // b.hi 54 <ic_clear_loop>
+            unchecked((int)0xd5033b9f), // dsb  ish
+            unchecked((int)0xd5033fdf), // isb
+            unchecked((int)0xd65f03c0), // ret
+        };
+
+        private delegate void InvalidateCache(ulong start, ulong end);
+
+        private InvalidateCache _invalidateCache;
+        private ReservedRegion _invalidateCacheCodeRegion;
+
+        private readonly bool _needsInvalidation;
+
+        public JitCacheInvalidation(IJitMemoryAllocator allocator)
+        {
+            // On macOS, a different path is used to write to the JIT cache, which does the invalidation.
+            if (!OperatingSystem.IsMacOS() && RuntimeInformation.ProcessArchitecture == Architecture.Arm64)
+            {
+                ulong size = (ulong)_invalidationCode.Length * sizeof(int);
+                ulong mask = (ulong)ReservedRegion.DefaultGranularity - 1;
+
+                size = (size + mask) & ~mask;
+
+                _invalidateCacheCodeRegion = new ReservedRegion(allocator, size);
+                _invalidateCacheCodeRegion.ExpandIfNeeded(size);
+
+                Marshal.Copy(_invalidationCode, 0, _invalidateCacheCodeRegion.Pointer, _invalidationCode.Length);
+
+                _invalidateCacheCodeRegion.Block.MapAsRx(0, size);
+
+                _invalidateCache = Marshal.GetDelegateForFunctionPointer<InvalidateCache>(_invalidateCacheCodeRegion.Pointer);
+
+                _needsInvalidation = true;
+            }
+        }
+
+        public void Invalidate(IntPtr basePointer, ulong size)
+        {
+            if (_needsInvalidation)
+            {
+                _invalidateCache((ulong)basePointer, (ulong)basePointer + size);
+            }
+        }
+    }
+}
+\ No newline at end of file
diff --git a/ARMeilleure/Translation/Compiler.cs b/ARMeilleure/Translation/Compiler.cs
index 817bd487..d4aa5cd9 100644
--- a/ARMeilleure/Translation/Compiler.cs
+++ b/ARMeilleure/Translation/Compiler.cs
@@ -1,8 +1,9 @@
 using ARMeilleure.CodeGen;
 using ARMeilleure.CodeGen.Optimizations;
-using ARMeilleure.CodeGen.X86;
 using ARMeilleure.Diagnostics;
 using ARMeilleure.IntermediateRepresentation;
+using System;
+using System.Runtime.InteropServices;
 
 namespace ARMeilleure.Translation
 {
@@ -12,7 +13,8 @@ namespace ARMeilleure.Translation
             ControlFlowGraph cfg,
             OperandType[]    argTypes,
             OperandType      retType,
-            CompilerOptions  options)
+            CompilerOptions  options,
+            Architecture     target)
         {
             CompilerContext cctx = new(cfg, argTypes, retType, options);
 
@@ -49,7 +51,18 @@ namespace ARMeilleure.Translation
                 Logger.EndPass(PassName.RegisterToLocal, cfg);
             }
 
-            return CodeGenerator.Generate(cctx);
+            if (target == Architecture.X64)
+            {
+                return CodeGen.X86.CodeGenerator.Generate(cctx);
+            }
+            else if (target == Architecture.Arm64)
+            {
+                return CodeGen.Arm64.CodeGenerator.Generate(cctx);
+            }
+            else
+            {
+                throw new NotImplementedException(target.ToString());
+            }
         }
     }
 }
 \ No newline at end of file
diff --git a/ARMeilleure/Translation/PTC/Ptc.cs b/ARMeilleure/Translation/PTC/Ptc.cs
index f99d6e51..6f57e188 100644
--- a/ARMeilleure/Translation/PTC/Ptc.cs
+++ b/ARMeilleure/Translation/PTC/Ptc.cs
@@ -27,7 +27,7 @@ namespace ARMeilleure.Translation.PTC
         private const string OuterHeaderMagicString = "PTCohd\0\0";
         private const string InnerHeaderMagicString = "PTCihd\0\0";
 
-        private const uint InternalVersion = 4159; //! To be incremented manually for each change to the ARMeilleure project.
+        private const uint InternalVersion = 4114; //! To be incremented manually for each change to the ARMeilleure project.
 
         private const string ActualDir = "0";
         private const string BackupDir = "1";
diff --git a/ARMeilleure/Translation/Translator.cs b/ARMeilleure/Translation/Translator.cs
index 77ccdaea..75c4df23 100644
--- a/ARMeilleure/Translation/Translator.cs
+++ b/ARMeilleure/Translation/Translator.cs
@@ -14,6 +14,7 @@ using System;
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.Runtime.InteropServices;
 using System.Threading;
 using static ARMeilleure.IntermediateRepresentation.Operand.Factory;
 
@@ -282,7 +283,7 @@ namespace ARMeilleure.Translation
                 options |= CompilerOptions.Relocatable;
             }
 
-            CompiledFunction compiledFunc = Compiler.Compile(cfg, argTypes, retType, options);
+            CompiledFunction compiledFunc = Compiler.Compile(cfg, argTypes, retType, options, RuntimeInformation.ProcessArchitecture);
 
             if (context.HasPtc && !singleStep)
             {
@@ -359,9 +360,14 @@ namespace ARMeilleure.Translation
                     }
                 }
 
-                if (block.Address == context.EntryAddress && !context.HighCq)
+                if (block.Address == context.EntryAddress)
                 {
-                    EmitRejitCheck(context, out counter);
+                    if (!context.HighCq)
+                    {
+                        EmitRejitCheck(context, out counter);
+                    }
+
+                    context.ClearQcFlag();
                 }
 
                 context.CurrBlock = block;
@@ -386,9 +392,14 @@ namespace ARMeilleure.Translation
 
                         bool isLastOp = opcIndex == block.OpCodes.Count - 1;
 
-                        if (isLastOp && block.Branch != null && !block.Branch.Exit && block.Branch.Address <= block.Address)
+                        if (isLastOp)
                         {
-                            EmitSynchronization(context);
+                            context.SyncQcFlag();
+
+                            if (block.Branch != null && !block.Branch.Exit && block.Branch.Address <= block.Address)
+                            {
+                                EmitSynchronization(context);
+                            }
                         }
 
                         Operand lblPredicateSkip = default;
diff --git a/ARMeilleure/Translation/TranslatorStubs.cs b/ARMeilleure/Translation/TranslatorStubs.cs
index 67d2bba8..6ed84de8 100644
--- a/ARMeilleure/Translation/TranslatorStubs.cs
+++ b/ARMeilleure/Translation/TranslatorStubs.cs
@@ -171,7 +171,7 @@ namespace ARMeilleure.Translation
             var retType = OperandType.I64;
             var argTypes = new[] { OperandType.I64 };
 
-            var func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq).Map<GuestFunction>();
+            var func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<GuestFunction>();
 
             return Marshal.GetFunctionPointerForDelegate(func);
         }
@@ -197,7 +197,7 @@ namespace ARMeilleure.Translation
             var retType = OperandType.I64;
             var argTypes = new[] { OperandType.I64 };
 
-            var func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq).Map<GuestFunction>();
+            var func = Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<GuestFunction>();
 
             return Marshal.GetFunctionPointerForDelegate(func);
         }
@@ -235,7 +235,7 @@ namespace ARMeilleure.Translation
             var retType = OperandType.None;
             var argTypes = new[] { OperandType.I64, OperandType.I64 };
 
-            return Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq).Map<DispatcherFunction>();
+            return Compiler.Compile(cfg, argTypes, retType, CompilerOptions.HighCq, RuntimeInformation.ProcessArchitecture).Map<DispatcherFunction>();
         }
     }
 }