core: hid: Add cammera support

src/core/CMakeLists.txt

@@ -158,6 +158,7 @@ add_library(core STATIC
     hid/input_converter.h
     hid/input_interpreter.cpp
     hid/input_interpreter.h
+    hid/irs_types.h
     hid/motion_input.cpp
     hid/motion_input.h
     hle/api_version.h

src/core/hid/emulated_controller.cpp

@@ -126,10 +126,14 @@ void EmulatedController::LoadDevices() {
     battery_params[LeftIndex].Set("battery", true);
     battery_params[RightIndex].Set("battery", true);
 
+    camera_params = Common::ParamPackage{"engine:camera,camera:1"};
+
     output_params[LeftIndex] = left_joycon;
     output_params[RightIndex] = right_joycon;
+    output_params[2] = camera_params;
     output_params[LeftIndex].Set("output", true);
     output_params[RightIndex].Set("output", true);
+    output_params[2].Set("output", true);
 
     LoadTASParams();
 
@@ -146,6 +150,7 @@ void EmulatedController::LoadDevices() {
                    Common::Input::CreateDevice<Common::Input::InputDevice>);
     std::transform(battery_params.begin(), battery_params.end(), battery_devices.begin(),
                    Common::Input::CreateDevice<Common::Input::InputDevice>);
+    camera_devices = Common::Input::CreateDevice<Common::Input::InputDevice>(camera_params);
     std::transform(output_params.begin(), output_params.end(), output_devices.begin(),
                    Common::Input::CreateDevice<Common::Input::OutputDevice>);
 
@@ -267,6 +272,14 @@ void EmulatedController::ReloadInput() {
         motion_devices[index]->ForceUpdate();
     }
 
+    if (camera_devices) {
+        camera_devices->SetCallback({
+            .on_change =
+                [this](const Common::Input::CallbackStatus& callback) { SetCamera(callback); },
+        });
+        camera_devices->ForceUpdate();
+    }
+
     // Use a common UUID for TAS
     static constexpr Common::UUID TAS_UUID = Common::UUID{
         {0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7, 0xA5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}};
@@ -851,6 +864,25 @@ void EmulatedController::SetBattery(const Common::Input::CallbackStatus& callbac
     TriggerOnChange(ControllerTriggerType::Battery, true);
 }
 
+void EmulatedController::SetCamera(const Common::Input::CallbackStatus& callback) {
+    std::unique_lock lock{mutex};
+    controller.camera_values = TransformToCamera(callback);
+
+    if (is_configuring) {
+        lock.unlock();
+        TriggerOnChange(ControllerTriggerType::IrSensor, false);
+        return;
+    }
+
+    controller.camera_state.sample++;
+    controller.camera_state.format =
+        static_cast<Core::IrSensor::ImageTransferProcessorFormat>(controller.camera_values.format);
+    controller.camera_state.data = controller.camera_values.data;
+
+    lock.unlock();
+    TriggerOnChange(ControllerTriggerType::IrSensor, true);
+}
+
 bool EmulatedController::SetVibration(std::size_t device_index, VibrationValue vibration) {
     if (device_index >= output_devices.size()) {
         return false;
@@ -928,6 +960,23 @@ bool EmulatedController::SetPollingMode(Common::Input::PollingMode polling_mode)
     return output_device->SetPollingMode(polling_mode) == Common::Input::PollingError::None;
 }
 
+bool EmulatedController::SetCameraFormat(
+    Core::IrSensor::ImageTransferProcessorFormat camera_format) {
+    LOG_INFO(Service_HID, "Set camera format {}", camera_format);
+
+    auto& right_output_device = output_devices[static_cast<std::size_t>(DeviceIndex::Right)];
+    auto& camera_output_device = output_devices[2];
+
+    if (right_output_device->SetCameraFormat(static_cast<Common::Input::CameraFormat>(
+            camera_format)) == Common::Input::CameraError::None) {
+        return true;
+    }
+
+    // Fallback to Qt camera if native device doesn't have support
+    return camera_output_device->SetCameraFormat(static_cast<Common::Input::CameraFormat>(
+               camera_format)) == Common::Input::CameraError::None;
+}
+
 void EmulatedController::SetLedPattern() {
     for (auto& device : output_devices) {
         if (!device) {
@@ -1163,6 +1212,11 @@ BatteryValues EmulatedController::GetBatteryValues() const {
     return controller.battery_values;
 }
 
+CameraValues EmulatedController::GetCameraValues() const {
+    std::scoped_lock lock{mutex};
+    return controller.camera_values;
+}
+
 HomeButtonState EmulatedController::GetHomeButtons() const {
     std::scoped_lock lock{mutex};
     if (is_configuring) {
@@ -1251,6 +1305,11 @@ BatteryLevelState EmulatedController::GetBattery() const {
     return controller.battery_state;
 }
 
+const CameraState& EmulatedController::GetCamera() const {
+    std::scoped_lock lock{mutex};
+    return controller.camera_state;
+}
+
 void EmulatedController::TriggerOnChange(ControllerTriggerType type, bool is_npad_service_update) {
     std::scoped_lock lock{callback_mutex};
     for (const auto& poller_pair : callback_list) {

src/core/hid/emulated_controller.h

@@ -15,10 +15,12 @@
 #include "common/settings.h"
 #include "common/vector_math.h"
 #include "core/hid/hid_types.h"
+#include "core/hid/irs_types.h"
 #include "core/hid/motion_input.h"
 
 namespace Core::HID {
 const std::size_t max_emulated_controllers = 2;
+const std::size_t output_devices = 3;
 struct ControllerMotionInfo {
     Common::Input::MotionStatus raw_status{};
     MotionInput emulated{};
@@ -34,15 +36,16 @@ using TriggerDevices =
     std::array<std::unique_ptr<Common::Input::InputDevice>, Settings::NativeTrigger::NumTriggers>;
 using BatteryDevices =
     std::array<std::unique_ptr<Common::Input::InputDevice>, max_emulated_controllers>;
-using OutputDevices =
+using CameraDevices = std::unique_ptr<Common::Input::InputDevice>;
-    std::array<std::unique_ptr<Common::Input::OutputDevice>, max_emulated_controllers>;
+using OutputDevices = std::array<std::unique_ptr<Common::Input::OutputDevice>, output_devices>;
 
 using ButtonParams = std::array<Common::ParamPackage, Settings::NativeButton::NumButtons>;
 using StickParams = std::array<Common::ParamPackage, Settings::NativeAnalog::NumAnalogs>;
 using ControllerMotionParams = std::array<Common::ParamPackage, Settings::NativeMotion::NumMotions>;
 using TriggerParams = std::array<Common::ParamPackage, Settings::NativeTrigger::NumTriggers>;
 using BatteryParams = std::array<Common::ParamPackage, max_emulated_controllers>;
-using OutputParams = std::array<Common::ParamPackage, max_emulated_controllers>;
+using CameraParams = Common::ParamPackage;
+using OutputParams = std::array<Common::ParamPackage, output_devices>;
 
 using ButtonValues = std::array<Common::Input::ButtonStatus, Settings::NativeButton::NumButtons>;
 using SticksValues = std::array<Common::Input::StickStatus, Settings::NativeAnalog::NumAnalogs>;
@@ -51,6 +54,7 @@ using TriggerValues =
 using ControllerMotionValues = std::array<ControllerMotionInfo, Settings::NativeMotion::NumMotions>;
 using ColorValues = std::array<Common::Input::BodyColorStatus, max_emulated_controllers>;
 using BatteryValues = std::array<Common::Input::BatteryStatus, max_emulated_controllers>;
+using CameraValues = Common::Input::CameraStatus;
 using VibrationValues = std::array<Common::Input::VibrationStatus, max_emulated_controllers>;
 
 struct AnalogSticks {
@@ -70,6 +74,12 @@ struct BatteryLevelState {
     NpadPowerInfo right{};
 };
 
+struct CameraState {
+    Core::IrSensor::ImageTransferProcessorFormat format{};
+    std::vector<u8> data{};
+    std::size_t sample{};
+};
+
 struct ControllerMotion {
     Common::Vec3f accel{};
     Common::Vec3f gyro{};
@@ -96,6 +106,7 @@ struct ControllerStatus {
     ColorValues color_values{};
     BatteryValues battery_values{};
     VibrationValues vibration_values{};
+    CameraValues camera_values{};
 
     // Data for HID serices
     HomeButtonState home_button_state{};
@@ -107,6 +118,7 @@ struct ControllerStatus {
     NpadGcTriggerState gc_trigger_state{};
     ControllerColors colors_state{};
     BatteryLevelState battery_state{};
+    CameraState camera_state{};
 };
 
 enum class ControllerTriggerType {
@@ -117,6 +129,7 @@ enum class ControllerTriggerType {
     Color,
     Battery,
     Vibration,
+    IrSensor,
     Connected,
     Disconnected,
     Type,
@@ -269,6 +282,9 @@ public:
     /// Returns the latest battery status from the controller with parameters
     BatteryValues GetBatteryValues() const;
 
+    /// Returns the latest camera status from the controller with parameters
+    CameraValues GetCameraValues() const;
+
     /// Returns the latest status of button input for the hid::HomeButton service
     HomeButtonState GetHomeButtons() const;
 
@@ -296,6 +312,9 @@ public:
     /// Returns the latest battery status from the controller
     BatteryLevelState GetBattery() const;
 
+    /// Returns the latest camera status from the controller
+    const CameraState& GetCamera() const;
+
     /**
      * Sends a specific vibration to the output device
      * @return true if vibration had no errors
@@ -315,6 +334,13 @@ public:
      */
     bool SetPollingMode(Common::Input::PollingMode polling_mode);
 
+    /**
+     * Sets the desired camera format to be polled from a controller
+     * @param camera_format size of each frame
+     * @return true if SetCameraFormat was successfull
+     */
+    bool SetCameraFormat(Core::IrSensor::ImageTransferProcessorFormat camera_format);
+
     /// Returns the led pattern corresponding to this emulated controller
     LedPattern GetLedPattern() const;
 
@@ -392,6 +418,12 @@ private:
      */
     void SetBattery(const Common::Input::CallbackStatus& callback, std::size_t index);
 
+    /**
+     * Updates the camera status of the controller
+     * @param callback A CallbackStatus containing the camera status
+     */
+    void SetCamera(const Common::Input::CallbackStatus& callback);
+
     /**
      * Triggers a callback that something has changed on the controller status
      * @param type Input type of the event to trigger
@@ -417,6 +449,7 @@ private:
     ControllerMotionParams motion_params;
     TriggerParams trigger_params;
     BatteryParams battery_params;
+    CameraParams camera_params;
     OutputParams output_params;
 
     ButtonDevices button_devices;
@@ -424,6 +457,7 @@ private:
     ControllerMotionDevices motion_devices;
     TriggerDevices trigger_devices;
     BatteryDevices battery_devices;
+    CameraDevices camera_devices;
     OutputDevices output_devices;
 
     // TAS related variables

src/core/hid/input_converter.cpp

@@ -270,6 +270,20 @@ Common::Input::AnalogStatus TransformToAnalog(const Common::Input::CallbackStatu
     return status;
 }
 
+Common::Input::CameraStatus TransformToCamera(const Common::Input::CallbackStatus& callback) {
+    Common::Input::CameraStatus camera{};
+    switch (callback.type) {
+    case Common::Input::InputType::IrSensor:
+        camera = callback.camera_status;
+        break;
+    default:
+        LOG_ERROR(Input, "Conversion from type {} to camera not implemented", callback.type);
+        break;
+    }
+
+    return camera;
+}
+
 void SanitizeAnalog(Common::Input::AnalogStatus& analog, bool clamp_value) {
     const auto& properties = analog.properties;
     float& raw_value = analog.raw_value;

src/core/hid/input_converter.h

@@ -76,6 +76,14 @@ Common::Input::TriggerStatus TransformToTrigger(const Common::Input::CallbackSta
  */
 Common::Input::AnalogStatus TransformToAnalog(const Common::Input::CallbackStatus& callback);
 
+/**
+ * Converts raw input data into a valid camera status.
+ *
+ * @param callback Supported callbacks: Camera.
+ * @return A valid CameraObject object.
+ */
+Common::Input::CameraStatus TransformToCamera(const Common::Input::CallbackStatus& callback);
+
 /**
  * Converts raw analog data into a valid analog value
  * @param analog An analog object containing raw data and properties

src/core/hid/irs_types.h

@@ -0,0 +1,304 @@
+// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/common_funcs.h"
+#include "common/common_types.h"
+#include "core/hid/hid_types.h"
+
+namespace Core::IrSensor {
+
+// This is nn::irsensor::CameraAmbientNoiseLevel
+enum class CameraAmbientNoiseLevel : u32 {
+    Low,
+    Medium,
+    High,
+    Unkown3, // This level can't be reached
+};
+
+// This is nn::irsensor::CameraLightTarget
+enum class CameraLightTarget : u32 {
+    AllLeds,
+    BrightLeds,
+    DimLeds,
+    None,
+};
+
+// This is nn::irsensor::PackedCameraLightTarget
+enum class PackedCameraLightTarget : u8 {
+    AllLeds,
+    BrightLeds,
+    DimLeds,
+    None,
+};
+
+// This is nn::irsensor::AdaptiveClusteringMode
+enum class AdaptiveClusteringMode : u32 {
+    StaticFov,
+    DynamicFov,
+};
+
+// This is nn::irsensor::AdaptiveClusteringTargetDistance
+enum class AdaptiveClusteringTargetDistance : u32 {
+    Near,
+    Middle,
+    Far,
+};
+
+// This is nn::irsensor::ImageTransferProcessorFormat
+enum class ImageTransferProcessorFormat : u32 {
+    Size320x240,
+    Size160x120,
+    Size80x60,
+    Size40x30,
+    Size20x15,
+};
+
+// This is nn::irsensor::PackedImageTransferProcessorFormat
+enum class PackedImageTransferProcessorFormat : u8 {
+    Size320x240,
+    Size160x120,
+    Size80x60,
+    Size40x30,
+    Size20x15,
+};
+
+// This is nn::irsensor::IrCameraStatus
+enum class IrCameraStatus : u32 {
+    Available,
+    Unsupported,
+    Unconnected,
+};
+
+// This is nn::irsensor::IrCameraInternalStatus
+enum class IrCameraInternalStatus : u32 {
+    Stopped,
+    FirmwareUpdateNeeded,
+    Unkown2,
+    Unkown3,
+    Unkown4,
+    FirmwareVersionRequested,
+    FirmwareVersionIsInvalid,
+    Ready,
+    Setting,
+};
+
+// This is nn::irsensor::detail::StatusManager::IrSensorMode
+enum class IrSensorMode : u64 {
+    None,
+    MomentProcessor,
+    ClusteringProcessor,
+    ImageTransferProcessor,
+    PointingProcessorMarker,
+    TeraPluginProcessor,
+    IrLedProcessor,
+};
+
+// This is nn::irsensor::ImageProcessorStatus
+enum ImageProcessorStatus : u32 {
+    Stopped,
+    Running,
+};
+
+// This is nn::irsensor::HandAnalysisMode
+enum class HandAnalysisMode : u32 {
+    None,
+    Silhouette,
+    Image,
+    SilhoueteAndImage,
+    SilhuetteOnly,
+};
+
+// This is nn::irsensor::IrSensorFunctionLevel
+enum class IrSensorFunctionLevel : u8 {
+    unknown0,
+    unknown1,
+    unknown2,
+    unknown3,
+    unknown4,
+};
+
+// This is nn::irsensor::MomentProcessorPreprocess
+enum class MomentProcessorPreprocess : u32 {
+    Unkown0,
+    Unkown1,
+};
+
+// This is nn::irsensor::PackedMomentProcessorPreprocess
+enum class PackedMomentProcessorPreprocess : u8 {
+    Unkown0,
+    Unkown1,
+};
+
+// This is nn::irsensor::PointingStatus
+enum class PointingStatus : u32 {
+    Unkown0,
+    Unkown1,
+};
+
+struct IrsRect {
+    s16 x;
+    s16 y;
+    s16 width;
+    s16 height;
+};
+
+struct IrsCentroid {
+    f32 x;
+    f32 y;
+};
+
+struct CameraConfig {
+    u64 exposure_time;
+    CameraLightTarget light_target;
+    u32 gain;
+    bool is_negative_used;
+    INSERT_PADDING_BYTES(7);
+};
+static_assert(sizeof(CameraConfig) == 0x18, "CameraConfig is an invalid size");
+
+struct PackedCameraConfig {
+    u64 exposure_time;
+    PackedCameraLightTarget light_target;
+    u8 gain;
+    bool is_negative_used;
+    INSERT_PADDING_BYTES(5);
+};
+static_assert(sizeof(PackedCameraConfig) == 0x10, "PackedCameraConfig is an invalid size");
+
+// This is nn::irsensor::IrCameraHandle
+struct IrCameraHandle {
+    u8 npad_id{};
+    Core::HID::NpadStyleIndex npad_type{Core::HID::NpadStyleIndex::None};
+    INSERT_PADDING_BYTES(2);
+};
+static_assert(sizeof(IrCameraHandle) == 4, "IrCameraHandle is an invalid size");
+
+// This is nn::irsensor::PackedMcuVersion
+struct PackedMcuVersion {
+    u16 major;
+    u16 minor;
+};
+static_assert(sizeof(PackedMcuVersion) == 4, "PackedMcuVersion is an invalid size");
+
+// This is nn::irsensor::PackedMomentProcessorConfig
+struct PackedMomentProcessorConfig {
+    PackedCameraConfig camera_config;
+    IrsRect window_of_interest;
+    PackedMcuVersion required_mcu_version;
+    PackedMomentProcessorPreprocess preprocess;
+    u8 preprocess_intensity_threshold;
+    INSERT_PADDING_BYTES(2);
+};
+static_assert(sizeof(PackedMomentProcessorConfig) == 0x20,
+              "PackedMomentProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedClusteringProcessorConfig
+struct PackedClusteringProcessorConfig {
+    PackedCameraConfig camera_config;
+    IrsRect window_of_interest;
+    PackedMcuVersion required_mcu_version;
+    u32 pixel_count_min;
+    u32 pixel_count_max;
+    u32 object_intensity_min;
+    bool is_external_light_filter_enabled;
+    INSERT_PADDING_BYTES(2);
+};
+static_assert(sizeof(PackedClusteringProcessorConfig) == 0x30,
+              "PackedClusteringProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedImageTransferProcessorConfig
+struct PackedImageTransferProcessorConfig {
+    PackedCameraConfig camera_config;
+    PackedMcuVersion required_mcu_version;
+    PackedImageTransferProcessorFormat format;
+    INSERT_PADDING_BYTES(3);
+};
+static_assert(sizeof(PackedImageTransferProcessorConfig) == 0x18,
+              "PackedImageTransferProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedTeraPluginProcessorConfig
+struct PackedTeraPluginProcessorConfig {
+    PackedMcuVersion required_mcu_version;
+    u8 mode;
+    u8 unknown_1;
+    u8 unknown_2;
+    u8 unknown_3;
+};
+static_assert(sizeof(PackedTeraPluginProcessorConfig) == 0x8,
+              "PackedTeraPluginProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedPointingProcessorConfig
+struct PackedPointingProcessorConfig {
+    IrsRect window_of_interest;
+    PackedMcuVersion required_mcu_version;
+};
+static_assert(sizeof(PackedPointingProcessorConfig) == 0xC,
+              "PackedPointingProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedFunctionLevel
+struct PackedFunctionLevel {
+    IrSensorFunctionLevel function_level;
+    INSERT_PADDING_BYTES(3);
+};
+static_assert(sizeof(PackedFunctionLevel) == 0x4, "PackedFunctionLevel is an invalid size");
+
+// This is nn::irsensor::PackedImageTransferProcessorExConfig
+struct PackedImageTransferProcessorExConfig {
+    PackedCameraConfig camera_config;
+    PackedMcuVersion required_mcu_version;
+    PackedImageTransferProcessorFormat origin_format;
+    PackedImageTransferProcessorFormat trimming_format;
+    u16 trimming_start_x;
+    u16 trimming_start_y;
+    bool is_external_light_filter_enabled;
+    INSERT_PADDING_BYTES(5);
+};
+static_assert(sizeof(PackedImageTransferProcessorExConfig) == 0x20,
+              "PackedImageTransferProcessorExConfig is an invalid size");
+
+// This is nn::irsensor::PackedIrLedProcessorConfig
+struct PackedIrLedProcessorConfig {
+    PackedMcuVersion required_mcu_version;
+    u8 light_target;
+    INSERT_PADDING_BYTES(3);
+};
+static_assert(sizeof(PackedIrLedProcessorConfig) == 0x8,
+              "PackedIrLedProcessorConfig is an invalid size");
+
+// This is nn::irsensor::HandAnalysisConfig
+struct HandAnalysisConfig {
+    HandAnalysisMode mode;
+};
+static_assert(sizeof(HandAnalysisConfig) == 0x4, "HandAnalysisConfig is an invalid size");
+
+// This is nn::irsensor::detail::ProcessorState
+struct ProcessorState {
+    u64 start{};
+    u32 count{};
+    INSERT_PADDING_BYTES(4);
+    std::array<u8, 0xE10> processor_raw_data{};
+};
+static_assert(sizeof(ProcessorState) == 0xE20, "ProcessorState is an invalid size");
+
+// This is nn::irsensor::detail::DeviceFormat
+struct DeviceFormat {
+    Core::IrSensor::IrCameraStatus camera_status{Core::IrSensor::IrCameraStatus::Unconnected};
+    Core::IrSensor::IrCameraInternalStatus camera_internal_status{
+        Core::IrSensor::IrCameraInternalStatus::Ready};
+    Core::IrSensor::IrSensorMode mode{Core::IrSensor::IrSensorMode::None};
+    ProcessorState state{};
+};
+static_assert(sizeof(DeviceFormat) == 0xE30, "DeviceFormat is an invalid size");
+
+// This is nn::irsensor::ImageTransferProcessorState
+struct ImageTransferProcessorState {
+    u64 sampling_number;
+    Core::IrSensor::CameraAmbientNoiseLevel ambient_noise_level;
+    INSERT_PADDING_BYTES(4);
+};
+static_assert(sizeof(ImageTransferProcessorState) == 0x10,
+              "ImageTransferProcessorState is an invalid size");
+
+} // namespace Core::IrSensor