applets/swkbd: Implement the Qt Software Keyboard frontend

The Qt Software Keyboard frontend attempts to mimic the software keyboard rendered by the Nintendo Switch.
This frontend implements multiple keyboard types, such as the normal software keyboard, the numeric pad software keyboard and the inline software keyboard.
Keyboard and controller input is also supported in this frontend.
Keyboard input is handled as native keyboard input, and so the on-screen keyboard cannot be navigated with the keyboard arrow keys as the arrow keys are used to move the text cursor.
Controller input is translated into mouse hover movements on the onscreen keyboard or their respective button actions (B for backspace, A for entering the selected button, L/R for moving the text cursor, etc).
The text check dialogs can also be confirmed with controller input through the use of the OverlayDialog

Massive thanks to Rei for creating all the UI for the various keyboards and OverlayDialog. This would not have been possible without his excellent work.

Co-authored-by: Its-Rei <kupfel@gmail.com>

src/yuzu/CMakeLists.txt

@@ -18,6 +18,7 @@ add_executable(yuzu
     applets/profile_select.h
     applets/software_keyboard.cpp
     applets/software_keyboard.h
+    applets/software_keyboard.ui
     applets/web_browser.cpp
     applets/web_browser.h
     bootmanager.cpp

src/yuzu/applets/software_keyboard.h

@@ -1,28 +1,228 @@
-// Copyright 2018 yuzu Emulator Project
+// Copyright 2021 yuzu Emulator Project
 // Licensed under GPLv2 or any later version
 // Refer to the license.txt file included.
 
 #pragma once
 
+#include <array>
+#include <atomic>
+#include <memory>
+#include <thread>
+
 #include <QDialog>
 #include <QValidator>
 
 #include "core/frontend/applets/software_keyboard.h"
 
-class GMainWindow;
+enum class HIDButton : u8;
 
-class QtSoftwareKeyboardValidator final : public QValidator {
+class InputInterpreter;
-public:
+
-    explicit QtSoftwareKeyboardValidator();
+namespace Core {
-    State validate(QString& input, int& pos) const override;
+class System;
-};
+}
+
+namespace Ui {
+class QtSoftwareKeyboardDialog;
+}
+
+class GMainWindow;
 
 class QtSoftwareKeyboardDialog final : public QDialog {
     Q_OBJECT
 
 public:
-    QtSoftwareKeyboardDialog(QWidget* parent);
+    QtSoftwareKeyboardDialog(QWidget* parent, Core::System& system_, bool is_inline_,
+                             Core::Frontend::KeyboardInitializeParameters initialize_parameters_);
     ~QtSoftwareKeyboardDialog() override;
+
+    void ShowNormalKeyboard(QPoint pos, QSize size);
+
+    void ShowTextCheckDialog(Service::AM::Applets::SwkbdTextCheckResult text_check_result,
+                             std::u16string text_check_message);
+
+    void ShowInlineKeyboard(Core::Frontend::InlineAppearParameters appear_parameters, QPoint pos,
+                            QSize size);
+
+    void HideInlineKeyboard();
+
+    void InlineTextChanged(Core::Frontend::InlineTextParameters text_parameters);
+
+    void ExitKeyboard();
+
+signals:
+    void SubmitNormalText(Service::AM::Applets::SwkbdResult result,
+                          std::u16string submitted_text) const;
+
+    void SubmitInlineText(Service::AM::Applets::SwkbdReplyType reply_type,
+                          std::u16string submitted_text, s32 cursor_position) const;
+
+public slots:
+    void open() override;
+    void reject() override;
+
+protected:
+    /// We override the keyPressEvent for inputting text into the inline software keyboard.
+    void keyPressEvent(QKeyEvent* event) override;
+
+private:
+    enum class Direction {
+        Left,
+        Up,
+        Right,
+        Down,
+    };
+
+    enum class BottomOSKIndex {
+        LowerCase,
+        UpperCase,
+        NumberPad,
+    };
+
+    /**
+     * Moves and resizes the window to a specified position and size.
+     *
+     * @param pos Top-left window position
+     * @param size Window size
+     */
+    void MoveAndResizeWindow(QPoint pos, QSize size);
+
+    /**
+     * Rescales all keyboard elements to account for High DPI displays.
+     *
+     * @param width Window width
+     * @param height Window height
+     * @param dpi_scale Display scaling factor
+     */
+    void RescaleKeyboardElements(float width, float height, float dpi_scale);
+
+    /// Sets the keyboard type based on initialize_parameters.
+    void SetKeyboardType();
+
+    /// Sets the password mode based on initialize_parameters.
+    void SetPasswordMode();
+
+    /// Sets the text draw type based on initialize_parameters.
+    void SetTextDrawType();
+
+    /// Sets the controller image at the bottom left of the software keyboard.
+    void SetControllerImage();
+
+    /// Disables buttons based on initialize_parameters.
+    void DisableKeyboardButtons();
+
+    /// Changes whether the backspace or/and ok buttons should be enabled or disabled.
+    void SetBackspaceOkEnabled();
+
+    /**
+     * Validates the input text sent in based on the parameters in initialize_parameters.
+     *
+     * @param input_text Input text
+     *
+     * @returns True if the input text is valid, false otherwise.
+     */
+    bool ValidateInputText(const QString& input_text);
+
+    /// Switches between LowerCase and UpperCase (Shift and Caps Lock)
+    void ChangeBottomOSKIndex();
+
+    /// Processes a keyboard button click from the UI as normal keyboard input.
+    void NormalKeyboardButtonClicked(QPushButton* button);
+
+    /// Processes a keyboard button click from the UI as inline keyboard input.
+    void InlineKeyboardButtonClicked(QPushButton* button);
+
+    /**
+     * Inserts a string of arbitrary length into the current_text at the current cursor position.
+     * This is only used for the inline software keyboard.
+     */
+    void InlineTextInsertString(std::u16string_view string);
+
+    /// Setup the mouse hover workaround for "focusing" buttons. This should only be called once.
+    void SetupMouseHover();
+
+    /**
+     * Handles button presses and converts them into keyboard input.
+     *
+     * @tparam HIDButton The list of buttons that can be converted into keyboard input.
+     */
+    template <HIDButton... T>
+    void HandleButtonPressedOnce();
+
+    /**
+     * Handles button holds and converts them into keyboard input.
+     *
+     * @tparam HIDButton The list of buttons that can be converted into keyboard input.
+     */
+    template <HIDButton... T>
+    void HandleButtonHold();
+
+    /**
+     * Translates a button press to focus or click a keyboard button.
+     *
+     * @param button The button press to process.
+     */
+    void TranslateButtonPress(HIDButton button);
+
+    /**
+     * Moves the focus of a button in a certain direction.
+     *
+     * @param direction The direction to move.
+     */
+    void MoveButtonDirection(Direction direction);
+
+    /**
+     * Moves the text cursor in a certain direction.
+     *
+     * @param direction The direction to move.
+     */
+    void MoveTextCursorDirection(Direction direction);
+
+    void StartInputThread();
+    void StopInputThread();
+
+    /// The thread where input is being polled and processed.
+    void InputThread();
+
+    std::unique_ptr<Ui::QtSoftwareKeyboardDialog> ui;
+
+    Core::System& system;
+
+    // True if it is the inline software keyboard.
+    bool is_inline;
+
+    // Common software keyboard initialize parameters.
+    Core::Frontend::KeyboardInitializeParameters initialize_parameters;
+
+    // Used only by the inline software keyboard since the QLineEdit or QTextEdit is hidden.
+    std::u16string current_text;
+    s32 cursor_position{0};
+
+    static constexpr std::size_t NUM_ROWS_NORMAL = 5;
+    static constexpr std::size_t NUM_COLUMNS_NORMAL = 12;
+    static constexpr std::size_t NUM_ROWS_NUMPAD = 4;
+    static constexpr std::size_t NUM_COLUMNS_NUMPAD = 4;
+
+    // Stores the normal keyboard layout.
+    std::array<std::array<std::array<QPushButton*, NUM_COLUMNS_NORMAL>, NUM_ROWS_NORMAL>, 2>
+        keyboard_buttons;
+    // Stores the numberpad keyboard layout.
+    std::array<std::array<QPushButton*, NUM_COLUMNS_NUMPAD>, NUM_ROWS_NUMPAD> numberpad_buttons;
+
+    // Contains a set of all buttons used in keyboard_buttons and numberpad_buttons.
+    std::array<QPushButton*, 110> all_buttons;
+
+    std::size_t row{0};
+    std::size_t column{0};
+
+    BottomOSKIndex bottom_osk_index{BottomOSKIndex::LowerCase};
+    std::atomic<bool> caps_lock_enabled{false};
+
+    std::unique_ptr<InputInterpreter> input_interpreter;
+
+    std::thread input_thread;
+
+    std::atomic<bool> input_thread_running{};
 };
 
 class QtSoftwareKeyboard final : public QObject, public Core::Frontend::SoftwareKeyboardApplet {
@@ -31,4 +231,55 @@ class QtSoftwareKeyboard final : public QObject, public Core::Frontend::Software
 public:
     explicit QtSoftwareKeyboard(GMainWindow& parent);
     ~QtSoftwareKeyboard() override;
+
+    void InitializeKeyboard(
+        bool is_inline, Core::Frontend::KeyboardInitializeParameters initialize_parameters,
+        std::function<void(Service::AM::Applets::SwkbdResult, std::u16string)>
+            submit_normal_callback_,
+        std::function<void(Service::AM::Applets::SwkbdReplyType, std::u16string, s32)>
+            submit_inline_callback_) override;
+
+    void ShowNormalKeyboard() const override;
+
+    void ShowTextCheckDialog(Service::AM::Applets::SwkbdTextCheckResult text_check_result,
+                             std::u16string text_check_message) const override;
+
+    void ShowInlineKeyboard(
+        Core::Frontend::InlineAppearParameters appear_parameters) const override;
+
+    void HideInlineKeyboard() const override;
+
+    void InlineTextChanged(Core::Frontend::InlineTextParameters text_parameters) const override;
+
+    void ExitKeyboard() const override;
+
+signals:
+    void MainWindowInitializeKeyboard(
+        bool is_inline, Core::Frontend::KeyboardInitializeParameters initialize_parameters) const;
+
+    void MainWindowShowNormalKeyboard() const;
+
+    void MainWindowShowTextCheckDialog(Service::AM::Applets::SwkbdTextCheckResult text_check_result,
+                                       std::u16string text_check_message) const;
+
+    void MainWindowShowInlineKeyboard(
+        Core::Frontend::InlineAppearParameters appear_parameters) const;
+
+    void MainWindowHideInlineKeyboard() const;
+
+    void MainWindowInlineTextChanged(Core::Frontend::InlineTextParameters text_parameters) const;
+
+    void MainWindowExitKeyboard() const;
+
+private:
+    void SubmitNormalText(Service::AM::Applets::SwkbdResult result,
+                          std::u16string submitted_text) const;
+
+    void SubmitInlineText(Service::AM::Applets::SwkbdReplyType reply_type,
+                          std::u16string submitted_text, s32 cursor_position) const;
+
+    mutable std::function<void(Service::AM::Applets::SwkbdResult, std::u16string)>
+        submit_normal_callback;
+    mutable std::function<void(Service::AM::Applets::SwkbdReplyType, std::u16string, s32)>
+        submit_inline_callback;
 };

src/yuzu/main.cpp

@@ -466,6 +466,114 @@ void GMainWindow::ProfileSelectorSelectProfile() {
     emit ProfileSelectorFinishedSelection(uuid);
 }
 
+void GMainWindow::SoftwareKeyboardInitialize(
+    bool is_inline, Core::Frontend::KeyboardInitializeParameters initialize_parameters) {
+    if (software_keyboard) {
+        LOG_ERROR(Frontend, "The software keyboard is already initialized!");
+        return;
+    }
+
+    software_keyboard = new QtSoftwareKeyboardDialog(render_window, Core::System::GetInstance(),
+                                                     is_inline, std::move(initialize_parameters));
+
+    if (is_inline) {
+        connect(
+            software_keyboard, &QtSoftwareKeyboardDialog::SubmitInlineText, this,
+            [this](Service::AM::Applets::SwkbdReplyType reply_type, std::u16string submitted_text,
+                   s32 cursor_position) {
+                emit SoftwareKeyboardSubmitInlineText(reply_type, submitted_text, cursor_position);
+            },
+            Qt::QueuedConnection);
+    } else {
+        connect(
+            software_keyboard, &QtSoftwareKeyboardDialog::SubmitNormalText, this,
+            [this](Service::AM::Applets::SwkbdResult result, std::u16string submitted_text) {
+                emit SoftwareKeyboardSubmitNormalText(result, submitted_text);
+            },
+            Qt::QueuedConnection);
+    }
+}
+
+void GMainWindow::SoftwareKeyboardShowNormal() {
+    if (!software_keyboard) {
+        LOG_ERROR(Frontend, "The software keyboard is not initialized!");
+        return;
+    }
+
+    const auto& layout = render_window->GetFramebufferLayout();
+
+    const auto x = layout.screen.left;
+    const auto y = layout.screen.top;
+    const auto w = layout.screen.GetWidth();
+    const auto h = layout.screen.GetHeight();
+
+    software_keyboard->ShowNormalKeyboard(render_window->mapToGlobal(QPoint(x, y)), QSize(w, h));
+}
+
+void GMainWindow::SoftwareKeyboardShowTextCheck(
+    Service::AM::Applets::SwkbdTextCheckResult text_check_result,
+    std::u16string text_check_message) {
+    if (!software_keyboard) {
+        LOG_ERROR(Frontend, "The software keyboard is not initialized!");
+        return;
+    }
+
+    software_keyboard->ShowTextCheckDialog(text_check_result, text_check_message);
+}
+
+void GMainWindow::SoftwareKeyboardShowInline(
+    Core::Frontend::InlineAppearParameters appear_parameters) {
+    if (!software_keyboard) {
+        LOG_ERROR(Frontend, "The software keyboard is not initialized!");
+        return;
+    }
+
+    const auto& layout = render_window->GetFramebufferLayout();
+
+    const auto x =
+        static_cast<int>(layout.screen.left + (0.5f * layout.screen.GetWidth() *
+                                               ((2.0f * appear_parameters.key_top_translate_x) +
+                                                (1.0f - appear_parameters.key_top_scale_x))));
+    const auto y =
+        static_cast<int>(layout.screen.top + (layout.screen.GetHeight() *
+                                              ((2.0f * appear_parameters.key_top_translate_y) +
+                                               (1.0f - appear_parameters.key_top_scale_y))));
+    const auto w = static_cast<int>(layout.screen.GetWidth() * appear_parameters.key_top_scale_x);
+    const auto h = static_cast<int>(layout.screen.GetHeight() * appear_parameters.key_top_scale_y);
+
+    software_keyboard->ShowInlineKeyboard(std::move(appear_parameters),
+                                          render_window->mapToGlobal(QPoint(x, y)), QSize(w, h));
+}
+
+void GMainWindow::SoftwareKeyboardHideInline() {
+    if (!software_keyboard) {
+        LOG_ERROR(Frontend, "The software keyboard is not initialized!");
+        return;
+    }
+
+    software_keyboard->HideInlineKeyboard();
+}
+
+void GMainWindow::SoftwareKeyboardInlineTextChanged(
+    Core::Frontend::InlineTextParameters text_parameters) {
+    if (!software_keyboard) {
+        LOG_ERROR(Frontend, "The software keyboard is not initialized!");
+        return;
+    }
+
+    software_keyboard->InlineTextChanged(std::move(text_parameters));
+}
+
+void GMainWindow::SoftwareKeyboardExit() {
+    if (!software_keyboard) {
+        return;
+    }
+
+    software_keyboard->ExitKeyboard();
+
+    software_keyboard = nullptr;
+}
+
 void GMainWindow::WebBrowserOpenWebPage(std::string_view main_url, std::string_view additional_args,
                                         bool is_local) {
 #ifdef YUZU_USE_QT_WEB_ENGINE
@@ -1009,6 +1117,10 @@ void GMainWindow::ConnectWidgetEvents() {
     connect(this, &GMainWindow::EmulationStopping, render_window,
             &GRenderWindow::OnEmulationStopping);
 
+    // Software Keyboard Applet
+    connect(this, &GMainWindow::EmulationStarting, this, &GMainWindow::SoftwareKeyboardExit);
+    connect(this, &GMainWindow::EmulationStopping, this, &GMainWindow::SoftwareKeyboardExit);
+
     connect(&status_bar_update_timer, &QTimer::timeout, this, &GMainWindow::UpdateStatusBar);
 }
 

src/yuzu/main.h

@@ -135,6 +135,11 @@ signals:
 
     void ProfileSelectorFinishedSelection(std::optional<Common::UUID> uuid);
 
+    void SoftwareKeyboardSubmitNormalText(Service::AM::Applets::SwkbdResult result,
+                                          std::u16string submitted_text);
+    void SoftwareKeyboardSubmitInlineText(Service::AM::Applets::SwkbdReplyType reply_type,
+                                          std::u16string submitted_text, s32 cursor_position);
+
     void WebBrowserExtractOfflineRomFS();
     void WebBrowserClosed(Service::AM::Applets::WebExitReason exit_reason, std::string last_url);
 
@@ -143,6 +148,15 @@ public slots:
     void OnExecuteProgram(std::size_t program_index);
     void ControllerSelectorReconfigureControllers(
         const Core::Frontend::ControllerParameters& parameters);
+    void SoftwareKeyboardInitialize(
+        bool is_inline, Core::Frontend::KeyboardInitializeParameters initialize_parameters);
+    void SoftwareKeyboardShowNormal();
+    void SoftwareKeyboardShowTextCheck(Service::AM::Applets::SwkbdTextCheckResult text_check_result,
+                                       std::u16string text_check_message);
+    void SoftwareKeyboardShowInline(Core::Frontend::InlineAppearParameters appear_parameters);
+    void SoftwareKeyboardHideInline();
+    void SoftwareKeyboardInlineTextChanged(Core::Frontend::InlineTextParameters text_parameters);
+    void SoftwareKeyboardExit();
     void ErrorDisplayDisplayError(QString error_code, QString error_text);
     void ProfileSelectorSelectProfile();
     void WebBrowserOpenWebPage(std::string_view main_url, std::string_view additional_args,