Add support for inline software keyboard (#1868)

* Add background mode configuration to SoftwareKeyboardApplet * Add placeholder text generator for Software Keyboard in background mode * Add stub for GetIndirectLayerImageMap * Fix default state of DecidedCancel response * Add GUI text input to Software Keyboard in background mode * Fix graphical glitch when Inline Software Keyboard appears * Improve readability of InlineResponses class * Improve code styling and fix compiler warnings * Replace ServiceDisplay log class by ServiceVi * Replace static readonly by const * Add proper finalization to the keyboard applet in inline mode * Rename constants to start with uppercase * Fix inline keyboard not working with some games * Improve code readability * Fix code styling
author: Caian Benedicto <caianbene@gmail.com> 2021-01-11 15:27:55 -0300
committer: GitHub <noreply@github.com> 2021-01-11 19:27:55 +0100
commit: e57b14042910eac2f876549b4a1e6fd5ba027368 (patch)
tree: dcfeabf03776310bb7095076d1a1e9300b069bfc /Ryujinx.HLE/HOS/Applets/SoftwareKeyboard/SoftwareKeyboardApplet.cs
parent: b81f19613f306267ee0f448d2f23caef74afef79 (diff)
1 files changed, 221 insertions, 37 deletions
diff --git a/Ryujinx.HLE/HOS/Applets/SoftwareKeyboard/SoftwareKeyboardApplet.cs b/Ryujinx.HLE/HOS/Applets/SoftwareKeyboard/SoftwareKeyboardApplet.cs
index cec466ca..89ed5592 100644
--- a/Ryujinx.HLE/HOS/Applets/SoftwareKeyboard/SoftwareKeyboardApplet.cs
+++ b/Ryujinx.HLE/HOS/Applets/SoftwareKeyboard/SoftwareKeyboardApplet.cs
@@ -5,6 +5,8 @@ using System;
 using System.IO;
 using System.Runtime.InteropServices;
 using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
 
 namespace Ryujinx.HLE.HOS.Applets
 {
@@ -19,10 +21,19 @@ namespace Ryujinx.HLE.HOS.Applets
 
         private SoftwareKeyboardState _state = SoftwareKeyboardState.Uninitialized;
 
+        private bool _isBackground = false;
+
         private AppletSession _normalSession;
         private AppletSession _interactiveSession;
 
-        private SoftwareKeyboardConfig _keyboardConfig;
+        // Configuration for foreground mode
+        private SoftwareKeyboardConfig  _keyboardFgConfig;
+        private SoftwareKeyboardCalc    _keyboardCalc;
+        private SoftwareKeyboardDictSet _keyboardDict;
+
+        // Configuration for background mode
+        private SoftwareKeyboardInitialize _keyboardBgInitialize;
+
         private byte[] _transferMemory;
 
         private string   _textValue = null;
@@ -47,30 +58,46 @@ namespace Ryujinx.HLE.HOS.Applets
             var launchParams   = _normalSession.Pop();
             var keyboardConfig = _normalSession.Pop();
 
-            if (keyboardConfig.Length < Marshal.SizeOf<SoftwareKeyboardConfig>())
+            // TODO: A better way would be handling the background creation properly
+            // in LibraryAppleCreator / Acessor instead of guessing by size.
+            if (keyboardConfig.Length == Marshal.SizeOf<SoftwareKeyboardInitialize>())
             {
-                Logger.Error?.Print(LogClass.ServiceAm, $"SoftwareKeyboardConfig size mismatch. Expected {Marshal.SizeOf<SoftwareKeyboardConfig>():x}. Got {keyboardConfig.Length:x}");
+                _isBackground = true;
+
+                _keyboardBgInitialize = ReadStruct<SoftwareKeyboardInitialize>(keyboardConfig);
+                _state = SoftwareKeyboardState.Uninitialized;
+
+                return ResultCode.Success;
             }
             else
             {
-                _keyboardConfig = ReadStruct<SoftwareKeyboardConfig>(keyboardConfig);
-            }
+                _isBackground = false;
 
-            if (!_normalSession.TryPop(out _transferMemory))
-            {
-                Logger.Error?.Print(LogClass.ServiceAm, "SwKbd Transfer Memory is null");
-            }
+                if (keyboardConfig.Length < Marshal.SizeOf<SoftwareKeyboardConfig>())
+                {
+                    Logger.Error?.Print(LogClass.ServiceAm, $"SoftwareKeyboardConfig size mismatch. Expected {Marshal.SizeOf<SoftwareKeyboardConfig>():x}. Got {keyboardConfig.Length:x}");
+                }
+                else
+                {
+                    _keyboardFgConfig = ReadStruct<SoftwareKeyboardConfig>(keyboardConfig);
+                }
 
-            if (_keyboardConfig.UseUtf8)
-            {
-                _encoding = Encoding.UTF8;
-            }
+                if (!_normalSession.TryPop(out _transferMemory))
+                {
+                    Logger.Error?.Print(LogClass.ServiceAm, "SwKbd Transfer Memory is null");
+                }
 
-            _state = SoftwareKeyboardState.Ready;
+                if (_keyboardFgConfig.UseUtf8)
+                {
+                    _encoding = Encoding.UTF8;
+                }
 
-            Execute();
+                _state = SoftwareKeyboardState.Ready;
 
-            return ResultCode.Success;
+                ExecuteForegroundKeyboard();
+
+                return ResultCode.Success;
+            }
         }
 
         public ResultCode GetResult()
@@ -78,39 +105,39 @@ namespace Ryujinx.HLE.HOS.Applets
             return ResultCode.Success;
         }
 
-        private void Execute()
+        private void ExecuteForegroundKeyboard()
         {
             string initialText = null;
 
             // Initial Text is always encoded as a UTF-16 string in the work buffer (passed as transfer memory)
             // InitialStringOffset points to the memory offset and InitialStringLength is the number of UTF-16 characters
-            if (_transferMemory != null && _keyboardConfig.InitialStringLength > 0)
+            if (_transferMemory != null && _keyboardFgConfig.InitialStringLength > 0)
             {
-                initialText = Encoding.Unicode.GetString(_transferMemory, _keyboardConfig.InitialStringOffset, 2 * _keyboardConfig.InitialStringLength);
+                initialText = Encoding.Unicode.GetString(_transferMemory, _keyboardFgConfig.InitialStringOffset, 2 * _keyboardFgConfig.InitialStringLength);
             }
 
             // If the max string length is 0, we set it to a large default
             // length.
-            if (_keyboardConfig.StringLengthMax == 0)
+            if (_keyboardFgConfig.StringLengthMax == 0)
             {
-                _keyboardConfig.StringLengthMax = 100;
+                _keyboardFgConfig.StringLengthMax = 100;
             }
 
             var args = new SoftwareKeyboardUiArgs
             {
-                HeaderText = _keyboardConfig.HeaderText,
-                SubtitleText = _keyboardConfig.SubtitleText,
-                GuideText = _keyboardConfig.GuideText,
-                SubmitText = (!string.IsNullOrWhiteSpace(_keyboardConfig.SubmitText) ? _keyboardConfig.SubmitText : "OK"),
-                StringLengthMin = _keyboardConfig.StringLengthMin, 
-                StringLengthMax = _keyboardConfig.StringLengthMax,
+                HeaderText = _keyboardFgConfig.HeaderText,
+                SubtitleText = _keyboardFgConfig.SubtitleText,
+                GuideText = _keyboardFgConfig.GuideText,
+                SubmitText = (!string.IsNullOrWhiteSpace(_keyboardFgConfig.SubmitText) ? _keyboardFgConfig.SubmitText : "OK"),
+                StringLengthMin = _keyboardFgConfig.StringLengthMin,
+                StringLengthMax = _keyboardFgConfig.StringLengthMax,
                 InitialText = initialText
             };
 
             // Call the configured GUI handler to get user's input
             if (_device.UiHandler == null)
             {
-                Logger.Warning?.Print(LogClass.Application, $"GUI Handler is not set. Falling back to default");
+                Logger.Warning?.Print(LogClass.Application, "GUI Handler is not set. Falling back to default");
                 _okPressed = true;
             }
             else
@@ -122,22 +149,22 @@ namespace Ryujinx.HLE.HOS.Applets
 
             // If the game requests a string with a minimum length less
             // than our default text, repeat our default text until we meet
-            // the minimum length requirement. 
+            // the minimum length requirement.
             // This should always be done before the text truncation step.
-            while (_textValue.Length < _keyboardConfig.StringLengthMin)
+            while (_textValue.Length < _keyboardFgConfig.StringLengthMin)
             {
                 _textValue = String.Join(" ", _textValue, _textValue);
             }
 
             // If our default text is longer than the allowed length,
             // we truncate it.
-            if (_textValue.Length > _keyboardConfig.StringLengthMax)
+            if (_textValue.Length > _keyboardFgConfig.StringLengthMax)
             {
-                _textValue = _textValue.Substring(0, (int)_keyboardConfig.StringLengthMax);
+                _textValue = _textValue.Substring(0, (int)_keyboardFgConfig.StringLengthMax);
             }
 
             // Does the application want to validate the text itself?
-            if (_keyboardConfig.CheckText)
+            if (_keyboardFgConfig.CheckText)
             {
                 // The application needs to validate the response, so we
                 // submit it to the interactive output buffer, and poll it
@@ -151,7 +178,7 @@ namespace Ryujinx.HLE.HOS.Applets
             {
                 // If the application doesn't need to validate the response,
                 // we push the data to the non-interactive output buffer
-                // and poll it for completion.             
+                // and poll it for completion.
                 _state = SoftwareKeyboardState.Complete;
 
                 _normalSession.Push(BuildResponse(_textValue, false));
@@ -162,16 +189,28 @@ namespace Ryujinx.HLE.HOS.Applets
 
         private void OnInteractiveData(object sender, EventArgs e)
         {
-            // Obtain the validation status response, 
+            // Obtain the validation status response.
             var data = _interactiveSession.Pop();
 
+            if (_isBackground)
+            {
+                OnBackgroundInteractiveData(data);
+            }
+            else
+            {
+                OnForegroundInteractiveData(data);
+            }
+        }
+
+        private void OnForegroundInteractiveData(byte[] data)
+        {
             if (_state == SoftwareKeyboardState.ValidationPending)
             {
                 // TODO(jduncantor):
                 // If application rejects our "attempt", submit another attempt,
                 // and put the applet back in PendingValidation state.
 
-                // For now we assume success, so we push the final result 
+                // For now we assume success, so we push the final result
                 // to the standard output buffer and carry on our merry way.
                 _normalSession.Push(BuildResponse(_textValue, false));
 
@@ -194,6 +233,151 @@ namespace Ryujinx.HLE.HOS.Applets
             }
         }
 
+        private void OnBackgroundInteractiveData(byte[] data)
+        {
+            // WARNING: Only invoke applet state changes after an explicit finalization
+            // request from the game, this is because the inline keyboard is expected to
+            // keep running in the background sending data by itself.
+
+            using (MemoryStream stream = new MemoryStream(data))
+            using (BinaryReader reader = new BinaryReader(stream))
+            {
+                var request = (InlineKeyboardRequest)reader.ReadUInt32();
+
+                long remaining;
+
+                // Always show the keyboard if the state is 'Ready'.
+                bool showKeyboard = _state == SoftwareKeyboardState.Ready;
+
+                switch (request)
+                {
+                    case InlineKeyboardRequest.Unknown0: // Unknown request sent by some games after calc
+                        _interactiveSession.Push(InlineResponses.Default());
+                        break;
+                    case InlineKeyboardRequest.UseChangedStringV2:
+                        // Not used because we only send the entire string after confirmation.
+                        _interactiveSession.Push(InlineResponses.Default());
+                        break;
+                    case InlineKeyboardRequest.UseMovedCursorV2:
+                        // Not used because we only send the entire string after confirmation.
+                        _interactiveSession.Push(InlineResponses.Default());
+                        break;
+                    case InlineKeyboardRequest.SetCustomizeDic:
+                        remaining = stream.Length - stream.Position;
+                        if (remaining != Marshal.SizeOf<SoftwareKeyboardDictSet>())
+                        {
+                            Logger.Error?.Print(LogClass.ServiceAm, $"Received invalid Software Keyboard DictSet of {remaining} bytes!");
+                        }
+                        else
+                        {
+                            var keyboardDictData = reader.ReadBytes((int)remaining);
+                            _keyboardDict = ReadStruct<SoftwareKeyboardDictSet>(keyboardDictData);
+                        }
+                        _interactiveSession.Push(InlineResponses.Default());
+                        break;
+                    case InlineKeyboardRequest.Calc:
+                        // Put the keyboard in a Ready state, this will force showing
+                        _state = SoftwareKeyboardState.Ready;
+                        remaining = stream.Length - stream.Position;
+                        if (remaining != Marshal.SizeOf<SoftwareKeyboardCalc>())
+                        {
+                            Logger.Error?.Print(LogClass.ServiceAm, $"Received invalid Software Keyboard Calc of {remaining} bytes!");
+                        }
+                        else
+                        {
+                            var keyboardCalcData = reader.ReadBytes((int)remaining);
+                            _keyboardCalc = ReadStruct<SoftwareKeyboardCalc>(keyboardCalcData);
+
+                            if (_keyboardCalc.Utf8Mode == 0x1)
+                            {
+                                _encoding = Encoding.UTF8;
+                            }
+
+                            // Force showing the keyboard regardless of the state, an unwanted
+                            // input dialog may show, but it is better than a soft lock.
+                            if (_keyboardCalc.Appear.ShouldBeHidden == 0)
+                            {
+                                showKeyboard = true;
+                            }
+                        }
+                        // Send an initialization finished signal.
+                        _interactiveSession.Push(InlineResponses.FinishedInitialize());
+                        // Start a task with the GUI handler to get user's input.
+                        new Task(() =>
+                        {
+                            bool submit = true;
+                            string inputText = (!string.IsNullOrWhiteSpace(_keyboardCalc.InputText) ? _keyboardCalc.InputText : DefaultText);
+
+                            // Call the configured GUI handler to get user's input.
+                            if (!showKeyboard)
+                            {
+                                // Submit the default text to avoid soft locking if the keyboard was ignored by
+                                // accident. It's better to change the name than being locked out of the game.
+                                submit = true;
+                                inputText = DefaultText;
+
+                                Logger.Debug?.Print(LogClass.Application, "Received a dummy Calc, keyboard will not be shown");
+                            }
+                            else if (_device.UiHandler == null)
+                            {
+                                Logger.Warning?.Print(LogClass.Application, "GUI Handler is not set. Falling back to default");
+                            }
+                            else
+                            {
+                                var args = new SoftwareKeyboardUiArgs
+                                {
+                                    HeaderText = "", // The inline keyboard lacks these texts
+                                    SubtitleText = "",
+                                    GuideText = "",
+                                    SubmitText = (!string.IsNullOrWhiteSpace(_keyboardCalc.Appear.OkText) ? _keyboardCalc.Appear.OkText : "OK"),
+                                    StringLengthMin = 0,
+                                    StringLengthMax = 100,
+                                    InitialText = inputText
+                                };
+
+                                submit = _device.UiHandler.DisplayInputDialog(args, out inputText);
+                            }
+
+                            if (submit)
+                            {
+                                Logger.Debug?.Print(LogClass.ServiceAm, "Sending keyboard OK...");
+
+                                if (_encoding == Encoding.UTF8)
+                                {
+                                    _interactiveSession.Push(InlineResponses.DecidedEnterUtf8(inputText));
+                                }
+                                else
+                                {
+                                    _interactiveSession.Push(InlineResponses.DecidedEnter(inputText));
+                                }
+                            }
+                            else
+                            {
+                                Logger.Debug?.Print(LogClass.ServiceAm, "Sending keyboard Cancel...");
+                                _interactiveSession.Push(InlineResponses.DecidedCancel());
+                            }
+
+                            // TODO: Why is this necessary? Does the software expect a constant stream of responses?
+                            Thread.Sleep(500);
+
+                            Logger.Debug?.Print(LogClass.ServiceAm, "Resetting state of the keyboard...");
+                            _interactiveSession.Push(InlineResponses.Default());
+                        }).Start();
+                        break;
+                    case InlineKeyboardRequest.Finalize:
+                        // The game wants to close the keyboard applet and will wait for a state change.
+                        _state = SoftwareKeyboardState.Uninitialized;
+                        AppletStateChanged?.Invoke(this, null);
+                        break;
+                    default:
+                        // We shouldn't be able to get here through standard swkbd execution.
+                        Logger.Error?.Print(LogClass.ServiceAm, $"Invalid Software Keyboard request {request} during state {_state}!");
+                        _interactiveSession.Push(InlineResponses.Default());
+                        break;
+                }
+            }
+        }
+
         private byte[] BuildResponse(string text, bool interactive)
         {
             int bufferSize = interactive ? InteractiveBufferSize : StandardBufferSize;
@@ -227,7 +411,7 @@ namespace Ryujinx.HLE.HOS.Applets
             GCHandle handle = GCHandle.Alloc(data, GCHandleType.Pinned);
 
             try
-            {    
+            {
                 return Marshal.PtrToStructure<T>(handle.AddrOfPinnedObject());
             }
             finally
author	Caian Benedicto <caianbene@gmail.com>	2021-01-11 15:27:55 -0300
committer	GitHub <noreply@github.com>	2021-01-11 19:27:55 +0100
commit	e57b14042910eac2f876549b4a1e6fd5ba027368 (patch)
tree	dcfeabf03776310bb7095076d1a1e9300b069bfc /Ryujinx.HLE/HOS/Applets/SoftwareKeyboard/SoftwareKeyboardApplet.cs
parent	b81f19613f306267ee0f448d2f23caef74afef79 (diff)