kernel: Implement thread pinning support (#2840)

* kernel: Implement Thread pinning support This commit adds support for 8.x thread pinning changes and implement SynchronizePreemptionState syscall. Based on kernel 13.x reverse. * Address gdkchan's comment * kernel: fix missing critical section leave in SetActivity Fix Unity games * Implement missing bits on the interrupt handler and inline update pinning function as it cannot be generic * Fix some bugs in SetActivity and SetCoreAndAffinityMask * Address gdkchan's comments
author: Mary <me@thog.eu> 2021-12-30 10:55:06 +0100
committer: GitHub <noreply@github.com> 2021-12-30 10:55:06 +0100
commit: e96ef6d53250b72d084f7e6baf13e9bab485bca2 (patch)
tree: 5701772c89dbad3bf51bdf736714f9feb889472a /Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
parent: 8544b1445b33381fca63714249ac36598c413004 (diff)
1 files changed, 307 insertions, 69 deletions
diff --git a/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs b/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
index 64629248..cf95b015 100644
--- a/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
+++ b/Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
@@ -11,6 +11,9 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 {
     class KThread : KSynchronizationObject, IKFutureSchedulerObject
     {
+        private const int TlsUserDisableCountOffset = 0x100;
+        private const int TlsUserInterruptFlagOffset = 0x102;
+
         public const int MaxWaitSyncObjects = 64;
 
         private ManualResetEvent _schedulerWaitEvent;
@@ -43,6 +46,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
         public bool IsSchedulable => _customThreadStart == null && !_forcedUnschedulable;
 
         public ulong MutexAddress { get; set; }
+        public int KernelWaitersCount { get; private set; }
 
         public KProcess Owner { get; private set; }
 
@@ -65,11 +69,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
         private LinkedList<KThread> _mutexWaiters;
         private LinkedListNode<KThread> _mutexWaiterNode;
 
+        private LinkedList<KThread> _pinnedWaiters;
+
         public KThread MutexOwner { get; private set; }
 
         public int ThreadHandleForUserMutex { get; set; }
 
         private ThreadSchedState _forcePauseFlags;
+        private ThreadSchedState _forcePausePermissionFlags;
 
         public KernelResult ObjSyncResult { get; set; }
 
@@ -79,11 +86,12 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
         public int CurrentCore { get; set; }
         public int ActiveCore { get; set; }
 
-        private long _affinityMaskOverride;
-        private int _preferredCoreOverride;
-#pragma warning disable CS0649
-        private int _affinityOverrideCount;
-#pragma warning restore CS0649
+        public bool IsPinned { get; private set; }
+
+        private long _originalAffinityMask;
+        private int _originalPreferredCore;
+        private int _originalBasePriority;
+        private int _coreMigrationDisableCount;
 
         public ThreadSchedState SchedFlags { get; private set; }
 
@@ -108,6 +116,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
         public long LastPc { get; set; }
 
+        private object ActivityOperationLock = new object();
+
         public KThread(KernelContext context) : base(context)
         {
             WaitSyncObjects = new KSynchronizationObject[MaxWaitSyncObjects];
@@ -116,6 +126,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             SiblingsPerCore = new LinkedListNode<KThread>[KScheduler.CpuCoresCount];
 
             _mutexWaiters = new LinkedList<KThread>();
+            _pinnedWaiters = new LinkedList<KThread>();
         }
 
         public KernelResult Initialize(
@@ -147,6 +158,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             DynamicPriority = priority;
             BasePriority = priority;
             CurrentCore = cpuCore;
+            IsPinned = false;
 
             _entrypoint = entrypoint;
             _customThreadStart = customThreadStart;
@@ -204,6 +216,8 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
             _hasBeenInitialized = true;
 
+            _forcePausePermissionFlags = ThreadSchedState.ForcePauseMask;
+
             if (owner != null)
             {
                 owner.SubscribeThreadEventHandlers(Context);
@@ -301,6 +315,11 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
         {
             KernelContext.CriticalSection.Enter();
 
+            if (Owner != null && Owner.PinnedThreads[KernelStatic.GetCurrentThread().CurrentCore] == this)
+            {
+                Owner.UnpinThread(this);
+            }
+
             ThreadSchedState result;
 
             if (Interlocked.CompareExchange(ref _shallBeTerminated, 1, 0) == 0)
@@ -405,6 +424,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             KernelContext.CriticalSection.Enter();
 
             _forcePauseFlags &= ~ThreadSchedState.ForcePauseMask;
+            _forcePausePermissionFlags = 0;
 
             bool decRef = ExitImpl();
 
@@ -433,6 +453,19 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             return decRef;
         }
 
+        private int GetEffectiveRunningCore()
+        {
+            for (int coreNumber = 0; coreNumber < KScheduler.CpuCoresCount; coreNumber++)
+            {
+                if (KernelContext.Schedulers[coreNumber].CurrentThread == this)
+                {
+                    return coreNumber;
+                }
+            }
+
+            return -1;
+        }
+
         public KernelResult Sleep(long timeout)
         {
             KernelContext.CriticalSection.Enter();
@@ -465,7 +498,14 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
         {
             KernelContext.CriticalSection.Enter();
 
-            BasePriority = priority;
+            if (IsPinned)
+            {
+                _originalBasePriority = priority;
+            }
+            else
+            {
+                BasePriority = priority;
+            }
 
             UpdatePriorityInheritance();
 
@@ -497,53 +537,96 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
         public KernelResult SetActivity(bool pause)
         {
-            KernelResult result = KernelResult.Success;
-
-            KernelContext.CriticalSection.Enter();
+            lock (ActivityOperationLock)
+            {
+                KernelResult result = KernelResult.Success;
 
-            ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
+                KernelContext.CriticalSection.Enter();
 
-            if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
-            {
-                KernelContext.CriticalSection.Leave();
+                ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
 
-                return KernelResult.InvalidState;
-            }
+                if (lowNibble != ThreadSchedState.Paused && lowNibble != ThreadSchedState.Running)
+                {
+                    KernelContext.CriticalSection.Leave();
 
-            KernelContext.CriticalSection.Enter();
+                    return KernelResult.InvalidState;
+                }
 
-            if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
-            {
-                if (pause)
+                if (!ShallBeTerminated && SchedFlags != ThreadSchedState.TerminationPending)
                 {
-                    // Pause, the force pause flag should be clear (thread is NOT paused).
-                    if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
+                    if (pause)
                     {
-                        Suspend(ThreadSchedState.ThreadPauseFlag);
+                        // Pause, the force pause flag should be clear (thread is NOT paused).
+                        if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) == 0)
+                        {
+                            Suspend(ThreadSchedState.ThreadPauseFlag);
+                        }
+                        else
+                        {
+                            result = KernelResult.InvalidState;
+                        }
                     }
                     else
                     {
-                        result = KernelResult.InvalidState;
+                        // Unpause, the force pause flag should be set (thread is paused).
+                        if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
+                        {
+                            Resume(ThreadSchedState.ThreadPauseFlag);
+                        }
+                        else
+                        {
+                            result = KernelResult.InvalidState;
+                        }
                     }
                 }
-                else
+
+                KernelContext.CriticalSection.Leave();
+
+                if (result == KernelResult.Success && pause)
                 {
-                    // Unpause, the force pause flag should be set (thread is paused).
-                    if ((_forcePauseFlags & ThreadSchedState.ThreadPauseFlag) != 0)
-                    {
-                        Resume(ThreadSchedState.ThreadPauseFlag);
-                    }
-                    else
+                    bool isThreadRunning = true;
+
+                    while (isThreadRunning)
                     {
-                        result = KernelResult.InvalidState;
+                        KernelContext.CriticalSection.Enter();
+
+                        if (TerminationRequested)
+                        {
+                            KernelContext.CriticalSection.Leave();
+
+                            break;
+                        }
+
+                        isThreadRunning = false;
+
+                        if (IsPinned)
+                        {
+                            KThread currentThread = KernelStatic.GetCurrentThread();
+
+                            if (currentThread.TerminationRequested)
+                            {
+                                KernelContext.CriticalSection.Leave();
+
+                                result = KernelResult.ThreadTerminating;
+
+                                break;
+                            }
+
+                            _pinnedWaiters.AddLast(currentThread);
+
+                            currentThread.Reschedule(ThreadSchedState.Paused);
+                        }
+                        else
+                        {
+                            isThreadRunning = GetEffectiveRunningCore() >= 0;
+                        }
+
+                        KernelContext.CriticalSection.Leave();
                     }
                 }
-            }
-
-            KernelContext.CriticalSection.Leave();
-            KernelContext.CriticalSection.Leave();
 
-            return result;
+                return result;
+            }
         }
 
         public void CancelSynchronization()
@@ -579,58 +662,105 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
         public KernelResult SetCoreAndAffinityMask(int newCore, long newAffinityMask)
         {
-            KernelContext.CriticalSection.Enter();
+            lock (ActivityOperationLock)
+            {
+                KernelContext.CriticalSection.Enter();
 
-            bool useOverride = _affinityOverrideCount != 0;
+                bool isCoreMigrationDisabled = _coreMigrationDisableCount != 0;
 
-            // The value -3 is "do not change the preferred core".
-            if (newCore == -3)
-            {
-                newCore = useOverride ? _preferredCoreOverride : PreferredCore;
+                // The value -3 is "do not change the preferred core".
+                if (newCore == -3)
+                {
+                    newCore = isCoreMigrationDisabled ? _originalPreferredCore : PreferredCore;
+
+                    if ((newAffinityMask & (1 << newCore)) == 0)
+                    {
+                        KernelContext.CriticalSection.Leave();
 
-                if ((newAffinityMask & (1 << newCore)) == 0)
+                        return KernelResult.InvalidCombination;
+                    }
+                }
+
+                if (isCoreMigrationDisabled)
                 {
-                    KernelContext.CriticalSection.Leave();
+                    _originalPreferredCore = newCore;
+                    _originalAffinityMask = newAffinityMask;
+                }
+                else
+                {
+                    long oldAffinityMask = AffinityMask;
+
+                    PreferredCore = newCore;
+                    AffinityMask = newAffinityMask;
+
+                    if (oldAffinityMask != newAffinityMask)
+                    {
+                        int oldCore = ActiveCore;
+
+                        if (oldCore >= 0 && ((AffinityMask >> oldCore) & 1) == 0)
+                        {
+                            if (PreferredCore < 0)
+                            {
+                                ActiveCore = sizeof(ulong) * 8 - 1 - BitOperations.LeadingZeroCount((ulong)AffinityMask);
+                            }
+                            else
+                            {
+                                ActiveCore = PreferredCore;
+                            }
+                        }
 
-                    return KernelResult.InvalidCombination;
+                        AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
+                    }
                 }
-            }
 
-            if (useOverride)
-            {
-                _preferredCoreOverride = newCore;
-                _affinityMaskOverride = newAffinityMask;
-            }
-            else
-            {
-                long oldAffinityMask = AffinityMask;
+                KernelContext.CriticalSection.Leave();
 
-                PreferredCore = newCore;
-                AffinityMask = newAffinityMask;
+                bool targetThreadPinned = true;
 
-                if (oldAffinityMask != newAffinityMask)
+                while (targetThreadPinned)
                 {
-                    int oldCore = ActiveCore;
+                    KernelContext.CriticalSection.Enter();
 
-                    if (oldCore >= 0 && ((AffinityMask >> oldCore) & 1) == 0)
+                    if (TerminationRequested)
                     {
-                        if (PreferredCore < 0)
+                        KernelContext.CriticalSection.Leave();
+
+                        break;
+                    }
+
+                    targetThreadPinned = false;
+
+                    int coreNumber = GetEffectiveRunningCore();
+                    bool isPinnedThreadCurrentlyRunning = coreNumber >= 0;
+
+                    if (isPinnedThreadCurrentlyRunning && ((1 << coreNumber) & AffinityMask) == 0)
+                    {
+                        if (IsPinned)
                         {
-                            ActiveCore = sizeof(ulong) * 8 - 1 - BitOperations.LeadingZeroCount((ulong)AffinityMask);
+                            KThread currentThread = KernelStatic.GetCurrentThread();
+
+                            if (currentThread.TerminationRequested)
+                            {
+                                KernelContext.CriticalSection.Leave();
+
+                                return KernelResult.ThreadTerminating;
+                            }
+
+                            _pinnedWaiters.AddLast(currentThread);
+
+                            currentThread.Reschedule(ThreadSchedState.Paused);
                         }
                         else
                         {
-                            ActiveCore = PreferredCore;
+                            targetThreadPinned = true;
                         }
                     }
 
-                    AdjustSchedulingForNewAffinity(oldAffinityMask, oldCore);
+                    KernelContext.CriticalSection.Leave();
                 }
-            }
-
-            KernelContext.CriticalSection.Leave();
 
-            return KernelResult.Success;
+                return KernelResult.Success;
+            }
         }
 
         private void CombineForcePauseFlags()
@@ -638,7 +768,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             ThreadSchedState oldFlags = SchedFlags;
             ThreadSchedState lowNibble = SchedFlags & ThreadSchedState.LowMask;
 
-            SchedFlags = lowNibble | _forcePauseFlags;
+            SchedFlags = lowNibble | (_forcePauseFlags & _forcePausePermissionFlags);
 
             AdjustScheduling(oldFlags);
         }
@@ -1106,7 +1236,7 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
             foreach (KThread thread in _mutexWaiters)
             {
                 thread.MutexOwner = null;
-                thread._preferredCoreOverride = 0;
+                thread._originalPreferredCore = 0;
                 thread.ObjSyncResult = KernelResult.InvalidState;
 
                 thread.ReleaseAndResume();
@@ -1116,5 +1246,113 @@ namespace Ryujinx.HLE.HOS.Kernel.Threading
 
             Owner?.DecrementThreadCountAndTerminateIfZero();
         }
+
+        public void Pin()
+        {
+            IsPinned = true;
+            _coreMigrationDisableCount++;
+
+            int activeCore = ActiveCore;
+
+            _originalPreferredCore = PreferredCore;
+            _originalAffinityMask = AffinityMask;
+
+            ActiveCore = CurrentCore;
+            PreferredCore = CurrentCore;
+            AffinityMask = 1 << CurrentCore;
+
+            if (activeCore != CurrentCore || _originalAffinityMask != AffinityMask)
+            {
+                AdjustSchedulingForNewAffinity(_originalAffinityMask, activeCore);
+            }
+
+            _originalBasePriority = BasePriority;
+            BasePriority = Math.Min(_originalBasePriority, BitOperations.TrailingZeroCount(Owner.Capabilities.AllowedThreadPriosMask) - 1);
+            UpdatePriorityInheritance();
+
+            // Disallows thread pausing
+            _forcePausePermissionFlags &= ~ThreadSchedState.ThreadPauseFlag;
+            CombineForcePauseFlags();
+
+            // TODO: Assign reduced SVC permissions
+        }
+
+        public void Unpin()
+        {
+            IsPinned = false;
+            _coreMigrationDisableCount--;
+
+            long affinityMask = AffinityMask;
+            int activeCore = ActiveCore;
+
+            PreferredCore = _originalPreferredCore;
+            AffinityMask = _originalAffinityMask;
+            
+            if (AffinityMask != affinityMask)
+            {
+                if ((AffinityMask & 1 << ActiveCore) != 0)
+                {
+                    if (PreferredCore >= 0)
+                    {
+                        ActiveCore = PreferredCore;
+                    }
+                    else
+                    {
+                        ActiveCore = sizeof(ulong) * 8 - 1 - BitOperations.LeadingZeroCount((ulong)AffinityMask);
+                    }
+
+                    AdjustSchedulingForNewAffinity(affinityMask, activeCore);
+                }
+            }
+
+            BasePriority = _originalBasePriority;
+            UpdatePriorityInheritance();
+
+            if (!TerminationRequested)
+            {
+                // Allows thread pausing
+                _forcePausePermissionFlags |= ThreadSchedState.ThreadPauseFlag;
+                CombineForcePauseFlags();
+
+                // TODO: Restore SVC permissions
+            }
+
+            // Wake up waiters
+            foreach (KThread waiter in _pinnedWaiters)
+            {
+                waiter.ReleaseAndResume();
+            }
+
+            _pinnedWaiters.Clear();
+        }
+
+        public void SynchronizePreemptionState()
+        {
+            KernelContext.CriticalSection.Enter();
+
+            if (Owner != null && Owner.PinnedThreads[CurrentCore] == this)
+            {
+                ClearUserInterruptFlag();
+
+                Owner.UnpinThread(this);
+            }
+
+            KernelContext.CriticalSection.Leave();
+        }
+
+        public ushort GetUserDisableCount()
+        {
+            return Owner.CpuMemory.Read<ushort>(_tlsAddress + TlsUserDisableCountOffset);
+        }
+
+        public void SetUserInterruptFlag()
+        {
+            Owner.CpuMemory.Write<ushort>(_tlsAddress + TlsUserInterruptFlagOffset, 1);
+        }
+
+        public void ClearUserInterruptFlag()
+        {
+            Owner.CpuMemory.Write<ushort>(_tlsAddress + TlsUserInterruptFlagOffset, 0);
+        }
     }
 }
 \ No newline at end of file
author	Mary <me@thog.eu>	2021-12-30 10:55:06 +0100
committer	GitHub <noreply@github.com>	2021-12-30 10:55:06 +0100
commit	e96ef6d53250b72d084f7e6baf13e9bab485bca2 (patch)
tree	5701772c89dbad3bf51bdf736714f9feb889472a /Ryujinx.HLE/HOS/Kernel/Threading/KThread.cs
parent	8544b1445b33381fca63714249ac36598c413004 (diff)