summaryrefslogtreecommitdiffstats
path: root/src/hid_core/irsensor
diff options
context:
space:
mode:
Diffstat (limited to 'src/hid_core/irsensor')
-rw-r--r--src/hid_core/irsensor/clustering_processor.cpp267
-rw-r--r--src/hid_core/irsensor/clustering_processor.h115
-rw-r--r--src/hid_core/irsensor/image_transfer_processor.cpp155
-rw-r--r--src/hid_core/irsensor/image_transfer_processor.h77
-rw-r--r--src/hid_core/irsensor/ir_led_processor.cpp27
-rw-r--r--src/hid_core/irsensor/ir_led_processor.h47
-rw-r--r--src/hid_core/irsensor/irs_types.h301
-rw-r--r--src/hid_core/irsensor/moment_processor.cpp149
-rw-r--r--src/hid_core/irsensor/moment_processor.h91
-rw-r--r--src/hid_core/irsensor/pointing_processor.cpp26
-rw-r--r--src/hid_core/irsensor/pointing_processor.h61
-rw-r--r--src/hid_core/irsensor/processor_base.cpp67
-rw-r--r--src/hid_core/irsensor/processor_base.h33
-rw-r--r--src/hid_core/irsensor/tera_plugin_processor.cpp29
-rw-r--r--src/hid_core/irsensor/tera_plugin_processor.h53
15 files changed, 1498 insertions, 0 deletions
diff --git a/src/hid_core/irsensor/clustering_processor.cpp b/src/hid_core/irsensor/clustering_processor.cpp
new file mode 100644
index 000000000..3abe19365
--- /dev/null
+++ b/src/hid_core/irsensor/clustering_processor.cpp
@@ -0,0 +1,267 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include <queue>
+
+#include "core/core.h"
+#include "core/core_timing.h"
+#include "hid_core/frontend/emulated_controller.h"
+#include "hid_core/hid_core.h"
+#include "hid_core/irsensor/clustering_processor.h"
+
+namespace Service::IRS {
+ClusteringProcessor::ClusteringProcessor(Core::System& system_,
+ Core::IrSensor::DeviceFormat& device_format,
+ std::size_t npad_index)
+ : device{device_format}, system{system_} {
+ npad_device = system.HIDCore().GetEmulatedControllerByIndex(npad_index);
+
+ device.mode = Core::IrSensor::IrSensorMode::ClusteringProcessor;
+ device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Stopped;
+ SetDefaultConfig();
+
+ shared_memory = std::construct_at(
+ reinterpret_cast<ClusteringSharedMemory*>(&device_format.state.processor_raw_data));
+
+ Core::HID::ControllerUpdateCallback engine_callback{
+ .on_change = [this](Core::HID::ControllerTriggerType type) { OnControllerUpdate(type); },
+ .is_npad_service = true,
+ };
+ callback_key = npad_device->SetCallback(engine_callback);
+}
+
+ClusteringProcessor::~ClusteringProcessor() {
+ npad_device->DeleteCallback(callback_key);
+};
+
+void ClusteringProcessor::StartProcessor() {
+ device.camera_status = Core::IrSensor::IrCameraStatus::Available;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Ready;
+}
+
+void ClusteringProcessor::SuspendProcessor() {}
+
+void ClusteringProcessor::StopProcessor() {}
+
+void ClusteringProcessor::OnControllerUpdate(Core::HID::ControllerTriggerType type) {
+ if (type != Core::HID::ControllerTriggerType::IrSensor) {
+ return;
+ }
+
+ next_state = {};
+ const auto& camera_data = npad_device->GetCamera();
+ auto filtered_image = camera_data.data;
+
+ RemoveLowIntensityData(filtered_image);
+
+ const auto window_start_x = static_cast<std::size_t>(current_config.window_of_interest.x);
+ const auto window_start_y = static_cast<std::size_t>(current_config.window_of_interest.y);
+ const auto window_end_x =
+ window_start_x + static_cast<std::size_t>(current_config.window_of_interest.width);
+ const auto window_end_y =
+ window_start_y + static_cast<std::size_t>(current_config.window_of_interest.height);
+
+ for (std::size_t y = window_start_y; y < window_end_y; y++) {
+ for (std::size_t x = window_start_x; x < window_end_x; x++) {
+ u8 pixel = GetPixel(filtered_image, x, y);
+ if (pixel == 0) {
+ continue;
+ }
+ const auto cluster = GetClusterProperties(filtered_image, x, y);
+ if (cluster.pixel_count > current_config.pixel_count_max) {
+ continue;
+ }
+ if (cluster.pixel_count < current_config.pixel_count_min) {
+ continue;
+ }
+ // Cluster object limit reached
+ if (next_state.object_count >= next_state.data.size()) {
+ continue;
+ }
+ next_state.data[next_state.object_count] = cluster;
+ next_state.object_count++;
+ }
+ }
+
+ next_state.sampling_number = camera_data.sample;
+ next_state.timestamp = system.CoreTiming().GetGlobalTimeNs().count();
+ next_state.ambient_noise_level = Core::IrSensor::CameraAmbientNoiseLevel::Low;
+ shared_memory->clustering_lifo.WriteNextEntry(next_state);
+
+ if (!IsProcessorActive()) {
+ StartProcessor();
+ }
+}
+
+void ClusteringProcessor::RemoveLowIntensityData(std::vector<u8>& data) {
+ for (u8& pixel : data) {
+ if (pixel < current_config.pixel_count_min) {
+ pixel = 0;
+ }
+ }
+}
+
+ClusteringProcessor::ClusteringData ClusteringProcessor::GetClusterProperties(std::vector<u8>& data,
+ std::size_t x,
+ std::size_t y) {
+ using DataPoint = Common::Point<std::size_t>;
+ std::queue<DataPoint> search_points{};
+ ClusteringData current_cluster = GetPixelProperties(data, x, y);
+ SetPixel(data, x, y, 0);
+ search_points.emplace<DataPoint>({x, y});
+
+ while (!search_points.empty()) {
+ const auto point = search_points.front();
+ search_points.pop();
+
+ // Avoid negative numbers
+ if (point.x == 0 || point.y == 0) {
+ continue;
+ }
+
+ std::array<DataPoint, 4> new_points{
+ DataPoint{point.x - 1, point.y},
+ {point.x, point.y - 1},
+ {point.x + 1, point.y},
+ {point.x, point.y + 1},
+ };
+
+ for (const auto new_point : new_points) {
+ if (new_point.x >= width) {
+ continue;
+ }
+ if (new_point.y >= height) {
+ continue;
+ }
+ if (GetPixel(data, new_point.x, new_point.y) < current_config.object_intensity_min) {
+ continue;
+ }
+ const ClusteringData cluster = GetPixelProperties(data, new_point.x, new_point.y);
+ current_cluster = MergeCluster(current_cluster, cluster);
+ SetPixel(data, new_point.x, new_point.y, 0);
+ search_points.emplace<DataPoint>({new_point.x, new_point.y});
+ }
+ }
+
+ return current_cluster;
+}
+
+ClusteringProcessor::ClusteringData ClusteringProcessor::GetPixelProperties(
+ const std::vector<u8>& data, std::size_t x, std::size_t y) const {
+ return {
+ .average_intensity = GetPixel(data, x, y) / 255.0f,
+ .centroid =
+ {
+ .x = static_cast<f32>(x),
+ .y = static_cast<f32>(y),
+
+ },
+ .pixel_count = 1,
+ .bound =
+ {
+ .x = static_cast<s16>(x),
+ .y = static_cast<s16>(y),
+ .width = 1,
+ .height = 1,
+ },
+ };
+}
+
+ClusteringProcessor::ClusteringData ClusteringProcessor::MergeCluster(
+ const ClusteringData a, const ClusteringData b) const {
+ const f32 a_pixel_count = static_cast<f32>(a.pixel_count);
+ const f32 b_pixel_count = static_cast<f32>(b.pixel_count);
+ const f32 pixel_count = a_pixel_count + b_pixel_count;
+ const f32 average_intensity =
+ (a.average_intensity * a_pixel_count + b.average_intensity * b_pixel_count) / pixel_count;
+ const Core::IrSensor::IrsCentroid centroid = {
+ .x = (a.centroid.x * a_pixel_count + b.centroid.x * b_pixel_count) / pixel_count,
+ .y = (a.centroid.y * a_pixel_count + b.centroid.y * b_pixel_count) / pixel_count,
+ };
+ s16 bound_start_x = a.bound.x < b.bound.x ? a.bound.x : b.bound.x;
+ s16 bound_start_y = a.bound.y < b.bound.y ? a.bound.y : b.bound.y;
+ s16 a_bound_end_x = a.bound.x + a.bound.width;
+ s16 a_bound_end_y = a.bound.y + a.bound.height;
+ s16 b_bound_end_x = b.bound.x + b.bound.width;
+ s16 b_bound_end_y = b.bound.y + b.bound.height;
+
+ const Core::IrSensor::IrsRect bound = {
+ .x = bound_start_x,
+ .y = bound_start_y,
+ .width = a_bound_end_x > b_bound_end_x ? static_cast<s16>(a_bound_end_x - bound_start_x)
+ : static_cast<s16>(b_bound_end_x - bound_start_x),
+ .height = a_bound_end_y > b_bound_end_y ? static_cast<s16>(a_bound_end_y - bound_start_y)
+ : static_cast<s16>(b_bound_end_y - bound_start_y),
+ };
+
+ return {
+ .average_intensity = average_intensity,
+ .centroid = centroid,
+ .pixel_count = static_cast<u32>(pixel_count),
+ .bound = bound,
+ };
+}
+
+u8 ClusteringProcessor::GetPixel(const std::vector<u8>& data, std::size_t x, std::size_t y) const {
+ if ((y * width) + x >= data.size()) {
+ return 0;
+ }
+ return data[(y * width) + x];
+}
+
+void ClusteringProcessor::SetPixel(std::vector<u8>& data, std::size_t x, std::size_t y, u8 value) {
+ if ((y * width) + x >= data.size()) {
+ return;
+ }
+ data[(y * width) + x] = value;
+}
+
+void ClusteringProcessor::SetDefaultConfig() {
+ using namespace std::literals::chrono_literals;
+ current_config.camera_config.exposure_time = std::chrono::microseconds(200ms).count();
+ current_config.camera_config.gain = 2;
+ current_config.camera_config.is_negative_used = false;
+ current_config.camera_config.light_target = Core::IrSensor::CameraLightTarget::BrightLeds;
+ current_config.window_of_interest = {
+ .x = 0,
+ .y = 0,
+ .width = width,
+ .height = height,
+ };
+ current_config.pixel_count_min = 3;
+ current_config.pixel_count_max = static_cast<u32>(GetDataSize(format));
+ current_config.is_external_light_filter_enabled = true;
+ current_config.object_intensity_min = 150;
+
+ npad_device->SetCameraFormat(format);
+}
+
+void ClusteringProcessor::SetConfig(Core::IrSensor::PackedClusteringProcessorConfig config) {
+ current_config.camera_config.exposure_time = config.camera_config.exposure_time;
+ current_config.camera_config.gain = config.camera_config.gain;
+ current_config.camera_config.is_negative_used = config.camera_config.is_negative_used;
+ current_config.camera_config.light_target =
+ static_cast<Core::IrSensor::CameraLightTarget>(config.camera_config.light_target);
+ current_config.window_of_interest = config.window_of_interest;
+ current_config.pixel_count_min = config.pixel_count_min;
+ current_config.pixel_count_max = config.pixel_count_max;
+ current_config.is_external_light_filter_enabled = config.is_external_light_filter_enabled;
+ current_config.object_intensity_min = config.object_intensity_min;
+
+ LOG_INFO(Service_IRS,
+ "Processor config, exposure_time={}, gain={}, is_negative_used={}, "
+ "light_target={}, window_of_interest=({}, {}, {}, {}), pixel_count_min={}, "
+ "pixel_count_max={}, is_external_light_filter_enabled={}, object_intensity_min={}",
+ current_config.camera_config.exposure_time, current_config.camera_config.gain,
+ current_config.camera_config.is_negative_used,
+ current_config.camera_config.light_target, current_config.window_of_interest.x,
+ current_config.window_of_interest.y, current_config.window_of_interest.width,
+ current_config.window_of_interest.height, current_config.pixel_count_min,
+ current_config.pixel_count_max, current_config.is_external_light_filter_enabled,
+ current_config.object_intensity_min);
+
+ npad_device->SetCameraFormat(format);
+}
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/clustering_processor.h b/src/hid_core/irsensor/clustering_processor.h
new file mode 100644
index 000000000..e3b60d9b0
--- /dev/null
+++ b/src/hid_core/irsensor/clustering_processor.h
@@ -0,0 +1,115 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/common_types.h"
+#include "hid_core/irsensor/irs_types.h"
+#include "hid_core/irsensor/processor_base.h"
+#include "hid_core/resources/irs_ring_lifo.h"
+
+namespace Core {
+class System;
+}
+
+namespace Core::HID {
+class EmulatedController;
+} // namespace Core::HID
+
+namespace Service::IRS {
+class ClusteringProcessor final : public ProcessorBase {
+public:
+ explicit ClusteringProcessor(Core::System& system_, Core::IrSensor::DeviceFormat& device_format,
+ std::size_t npad_index);
+ ~ClusteringProcessor() override;
+
+ // Called when the processor is initialized
+ void StartProcessor() override;
+
+ // Called when the processor is suspended
+ void SuspendProcessor() override;
+
+ // Called when the processor is stopped
+ void StopProcessor() override;
+
+ // Sets config parameters of the camera
+ void SetConfig(Core::IrSensor::PackedClusteringProcessorConfig config);
+
+private:
+ static constexpr auto format = Core::IrSensor::ImageTransferProcessorFormat::Size320x240;
+ static constexpr std::size_t width = 320;
+ static constexpr std::size_t height = 240;
+
+ // This is nn::irsensor::ClusteringProcessorConfig
+ struct ClusteringProcessorConfig {
+ Core::IrSensor::CameraConfig camera_config;
+ Core::IrSensor::IrsRect window_of_interest;
+ u32 pixel_count_min;
+ u32 pixel_count_max;
+ u32 object_intensity_min;
+ bool is_external_light_filter_enabled;
+ INSERT_PADDING_BYTES(3);
+ };
+ static_assert(sizeof(ClusteringProcessorConfig) == 0x30,
+ "ClusteringProcessorConfig is an invalid size");
+
+ // This is nn::irsensor::AdaptiveClusteringProcessorConfig
+ struct AdaptiveClusteringProcessorConfig {
+ Core::IrSensor::AdaptiveClusteringMode mode;
+ Core::IrSensor::AdaptiveClusteringTargetDistance target_distance;
+ };
+ static_assert(sizeof(AdaptiveClusteringProcessorConfig) == 0x8,
+ "AdaptiveClusteringProcessorConfig is an invalid size");
+
+ // This is nn::irsensor::ClusteringData
+ struct ClusteringData {
+ f32 average_intensity;
+ Core::IrSensor::IrsCentroid centroid;
+ u32 pixel_count;
+ Core::IrSensor::IrsRect bound;
+ };
+ static_assert(sizeof(ClusteringData) == 0x18, "ClusteringData is an invalid size");
+
+ // This is nn::irsensor::ClusteringProcessorState
+ struct ClusteringProcessorState {
+ s64 sampling_number;
+ u64 timestamp;
+ u8 object_count;
+ INSERT_PADDING_BYTES(3);
+ Core::IrSensor::CameraAmbientNoiseLevel ambient_noise_level;
+ std::array<ClusteringData, 0x10> data;
+ };
+ static_assert(sizeof(ClusteringProcessorState) == 0x198,
+ "ClusteringProcessorState is an invalid size");
+
+ struct ClusteringSharedMemory {
+ Service::IRS::Lifo<ClusteringProcessorState, 6> clustering_lifo;
+ static_assert(sizeof(clustering_lifo) == 0x9A0, "clustering_lifo is an invalid size");
+ INSERT_PADDING_WORDS(0x11F);
+ };
+ static_assert(sizeof(ClusteringSharedMemory) == 0xE20,
+ "ClusteringSharedMemory is an invalid size");
+
+ void OnControllerUpdate(Core::HID::ControllerTriggerType type);
+ void RemoveLowIntensityData(std::vector<u8>& data);
+ ClusteringData GetClusterProperties(std::vector<u8>& data, std::size_t x, std::size_t y);
+ ClusteringData GetPixelProperties(const std::vector<u8>& data, std::size_t x,
+ std::size_t y) const;
+ ClusteringData MergeCluster(const ClusteringData a, const ClusteringData b) const;
+ u8 GetPixel(const std::vector<u8>& data, std::size_t x, std::size_t y) const;
+ void SetPixel(std::vector<u8>& data, std::size_t x, std::size_t y, u8 value);
+
+ // Sets config parameters of the camera
+ void SetDefaultConfig();
+
+ ClusteringSharedMemory* shared_memory = nullptr;
+ ClusteringProcessorState next_state{};
+
+ ClusteringProcessorConfig current_config{};
+ Core::IrSensor::DeviceFormat& device;
+ Core::HID::EmulatedController* npad_device;
+ int callback_key{};
+
+ Core::System& system;
+};
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/image_transfer_processor.cpp b/src/hid_core/irsensor/image_transfer_processor.cpp
new file mode 100644
index 000000000..d6573f8dc
--- /dev/null
+++ b/src/hid_core/irsensor/image_transfer_processor.cpp
@@ -0,0 +1,155 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "core/core.h"
+#include "core/memory.h"
+#include "hid_core/frontend/emulated_controller.h"
+#include "hid_core/hid_core.h"
+#include "hid_core/irsensor/image_transfer_processor.h"
+
+namespace Service::IRS {
+ImageTransferProcessor::ImageTransferProcessor(Core::System& system_,
+ Core::IrSensor::DeviceFormat& device_format,
+ std::size_t npad_index)
+ : device{device_format}, system{system_} {
+ npad_device = system.HIDCore().GetEmulatedControllerByIndex(npad_index);
+
+ Core::HID::ControllerUpdateCallback engine_callback{
+ .on_change = [this](Core::HID::ControllerTriggerType type) { OnControllerUpdate(type); },
+ .is_npad_service = true,
+ };
+ callback_key = npad_device->SetCallback(engine_callback);
+
+ device.mode = Core::IrSensor::IrSensorMode::ImageTransferProcessor;
+ device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Stopped;
+}
+
+ImageTransferProcessor::~ImageTransferProcessor() {
+ npad_device->DeleteCallback(callback_key);
+};
+
+void ImageTransferProcessor::StartProcessor() {
+ is_active = true;
+ device.camera_status = Core::IrSensor::IrCameraStatus::Available;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Ready;
+ processor_state.sampling_number = 0;
+ processor_state.ambient_noise_level = Core::IrSensor::CameraAmbientNoiseLevel::Low;
+}
+
+void ImageTransferProcessor::SuspendProcessor() {}
+
+void ImageTransferProcessor::StopProcessor() {}
+
+void ImageTransferProcessor::OnControllerUpdate(Core::HID::ControllerTriggerType type) {
+ if (type != Core::HID::ControllerTriggerType::IrSensor) {
+ return;
+ }
+ if (transfer_memory == 0) {
+ return;
+ }
+
+ const auto& camera_data = npad_device->GetCamera();
+
+ // This indicates how much ambient light is present
+ processor_state.ambient_noise_level = Core::IrSensor::CameraAmbientNoiseLevel::Low;
+ processor_state.sampling_number = camera_data.sample;
+
+ if (camera_data.format != current_config.origin_format) {
+ LOG_WARNING(Service_IRS, "Wrong Input format {} expected {}", camera_data.format,
+ current_config.origin_format);
+ system.ApplicationMemory().ZeroBlock(transfer_memory,
+ GetDataSize(current_config.trimming_format));
+ return;
+ }
+
+ if (current_config.origin_format > current_config.trimming_format) {
+ LOG_WARNING(Service_IRS, "Origin format {} is smaller than trimming format {}",
+ current_config.origin_format, current_config.trimming_format);
+ system.ApplicationMemory().ZeroBlock(transfer_memory,
+ GetDataSize(current_config.trimming_format));
+ return;
+ }
+
+ std::vector<u8> window_data{};
+ const auto origin_width = GetDataWidth(current_config.origin_format);
+ const auto origin_height = GetDataHeight(current_config.origin_format);
+ const auto trimming_width = GetDataWidth(current_config.trimming_format);
+ const auto trimming_height = GetDataHeight(current_config.trimming_format);
+ window_data.resize(GetDataSize(current_config.trimming_format));
+
+ if (trimming_width + current_config.trimming_start_x > origin_width ||
+ trimming_height + current_config.trimming_start_y > origin_height) {
+ LOG_WARNING(Service_IRS,
+ "Trimming area ({}, {}, {}, {}) is outside of origin area ({}, {})",
+ current_config.trimming_start_x, current_config.trimming_start_y,
+ trimming_width, trimming_height, origin_width, origin_height);
+ system.ApplicationMemory().ZeroBlock(transfer_memory,
+ GetDataSize(current_config.trimming_format));
+ return;
+ }
+
+ for (std::size_t y = 0; y < trimming_height; y++) {
+ for (std::size_t x = 0; x < trimming_width; x++) {
+ const std::size_t window_index = (y * trimming_width) + x;
+ const std::size_t origin_index =
+ ((y + current_config.trimming_start_y) * origin_width) + x +
+ current_config.trimming_start_x;
+ window_data[window_index] = camera_data.data[origin_index];
+ }
+ }
+
+ system.ApplicationMemory().WriteBlock(transfer_memory, window_data.data(),
+ GetDataSize(current_config.trimming_format));
+
+ if (!IsProcessorActive()) {
+ StartProcessor();
+ }
+}
+
+void ImageTransferProcessor::SetConfig(Core::IrSensor::PackedImageTransferProcessorConfig config) {
+ current_config.camera_config.exposure_time = config.camera_config.exposure_time;
+ current_config.camera_config.gain = config.camera_config.gain;
+ current_config.camera_config.is_negative_used = config.camera_config.is_negative_used;
+ current_config.camera_config.light_target =
+ static_cast<Core::IrSensor::CameraLightTarget>(config.camera_config.light_target);
+ current_config.origin_format =
+ static_cast<Core::IrSensor::ImageTransferProcessorFormat>(config.format);
+ current_config.trimming_format =
+ static_cast<Core::IrSensor::ImageTransferProcessorFormat>(config.format);
+ current_config.trimming_start_x = 0;
+ current_config.trimming_start_y = 0;
+
+ npad_device->SetCameraFormat(current_config.origin_format);
+}
+
+void ImageTransferProcessor::SetConfig(
+ Core::IrSensor::PackedImageTransferProcessorExConfig config) {
+ current_config.camera_config.exposure_time = config.camera_config.exposure_time;
+ current_config.camera_config.gain = config.camera_config.gain;
+ current_config.camera_config.is_negative_used = config.camera_config.is_negative_used;
+ current_config.camera_config.light_target =
+ static_cast<Core::IrSensor::CameraLightTarget>(config.camera_config.light_target);
+ current_config.origin_format =
+ static_cast<Core::IrSensor::ImageTransferProcessorFormat>(config.origin_format);
+ current_config.trimming_format =
+ static_cast<Core::IrSensor::ImageTransferProcessorFormat>(config.trimming_format);
+ current_config.trimming_start_x = config.trimming_start_x;
+ current_config.trimming_start_y = config.trimming_start_y;
+
+ npad_device->SetCameraFormat(current_config.origin_format);
+}
+
+void ImageTransferProcessor::SetTransferMemoryAddress(Common::ProcessAddress t_mem) {
+ transfer_memory = t_mem;
+}
+
+Core::IrSensor::ImageTransferProcessorState ImageTransferProcessor::GetState(
+ std::vector<u8>& data) const {
+ const auto size = GetDataSize(current_config.trimming_format);
+ data.resize(size);
+ system.ApplicationMemory().ReadBlock(transfer_memory, data.data(), size);
+ return processor_state;
+}
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/image_transfer_processor.h b/src/hid_core/irsensor/image_transfer_processor.h
new file mode 100644
index 000000000..4e0117084
--- /dev/null
+++ b/src/hid_core/irsensor/image_transfer_processor.h
@@ -0,0 +1,77 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/typed_address.h"
+#include "hid_core/irsensor/irs_types.h"
+#include "hid_core/irsensor/processor_base.h"
+
+namespace Core {
+class System;
+}
+
+namespace Core::HID {
+class EmulatedController;
+} // namespace Core::HID
+
+namespace Service::IRS {
+class ImageTransferProcessor final : public ProcessorBase {
+public:
+ explicit ImageTransferProcessor(Core::System& system_,
+ Core::IrSensor::DeviceFormat& device_format,
+ std::size_t npad_index);
+ ~ImageTransferProcessor() override;
+
+ // Called when the processor is initialized
+ void StartProcessor() override;
+
+ // Called when the processor is suspended
+ void SuspendProcessor() override;
+
+ // Called when the processor is stopped
+ void StopProcessor() override;
+
+ // Sets config parameters of the camera
+ void SetConfig(Core::IrSensor::PackedImageTransferProcessorConfig config);
+ void SetConfig(Core::IrSensor::PackedImageTransferProcessorExConfig config);
+
+ // Transfer memory where the image data will be stored
+ void SetTransferMemoryAddress(Common::ProcessAddress t_mem);
+
+ Core::IrSensor::ImageTransferProcessorState GetState(std::vector<u8>& data) const;
+
+private:
+ // This is nn::irsensor::ImageTransferProcessorConfig
+ struct ImageTransferProcessorConfig {
+ Core::IrSensor::CameraConfig camera_config;
+ Core::IrSensor::ImageTransferProcessorFormat format;
+ };
+ static_assert(sizeof(ImageTransferProcessorConfig) == 0x20,
+ "ImageTransferProcessorConfig is an invalid size");
+
+ // This is nn::irsensor::ImageTransferProcessorExConfig
+ struct ImageTransferProcessorExConfig {
+ Core::IrSensor::CameraConfig camera_config;
+ Core::IrSensor::ImageTransferProcessorFormat origin_format;
+ Core::IrSensor::ImageTransferProcessorFormat trimming_format;
+ u16 trimming_start_x;
+ u16 trimming_start_y;
+ bool is_external_light_filter_enabled;
+ INSERT_PADDING_BYTES(3);
+ };
+ static_assert(sizeof(ImageTransferProcessorExConfig) == 0x28,
+ "ImageTransferProcessorExConfig is an invalid size");
+
+ void OnControllerUpdate(Core::HID::ControllerTriggerType type);
+
+ ImageTransferProcessorExConfig current_config{};
+ Core::IrSensor::ImageTransferProcessorState processor_state{};
+ Core::IrSensor::DeviceFormat& device;
+ Core::HID::EmulatedController* npad_device;
+ int callback_key{};
+
+ Core::System& system;
+ Common::ProcessAddress transfer_memory{};
+};
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/ir_led_processor.cpp b/src/hid_core/irsensor/ir_led_processor.cpp
new file mode 100644
index 000000000..4b04e05b5
--- /dev/null
+++ b/src/hid_core/irsensor/ir_led_processor.cpp
@@ -0,0 +1,27 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "hid_core/irsensor/ir_led_processor.h"
+
+namespace Service::IRS {
+IrLedProcessor::IrLedProcessor(Core::IrSensor::DeviceFormat& device_format)
+ : device(device_format) {
+ device.mode = Core::IrSensor::IrSensorMode::IrLedProcessor;
+ device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Stopped;
+}
+
+IrLedProcessor::~IrLedProcessor() = default;
+
+void IrLedProcessor::StartProcessor() {}
+
+void IrLedProcessor::SuspendProcessor() {}
+
+void IrLedProcessor::StopProcessor() {}
+
+void IrLedProcessor::SetConfig(Core::IrSensor::PackedIrLedProcessorConfig config) {
+ current_config.light_target =
+ static_cast<Core::IrSensor::CameraLightTarget>(config.light_target);
+}
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/ir_led_processor.h b/src/hid_core/irsensor/ir_led_processor.h
new file mode 100644
index 000000000..03d0c4245
--- /dev/null
+++ b/src/hid_core/irsensor/ir_led_processor.h
@@ -0,0 +1,47 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/bit_field.h"
+#include "common/common_types.h"
+#include "hid_core/irsensor/irs_types.h"
+#include "hid_core/irsensor/processor_base.h"
+
+namespace Service::IRS {
+class IrLedProcessor final : public ProcessorBase {
+public:
+ explicit IrLedProcessor(Core::IrSensor::DeviceFormat& device_format);
+ ~IrLedProcessor() override;
+
+ // Called when the processor is initialized
+ void StartProcessor() override;
+
+ // Called when the processor is suspended
+ void SuspendProcessor() override;
+
+ // Called when the processor is stopped
+ void StopProcessor() override;
+
+ // Sets config parameters of the camera
+ void SetConfig(Core::IrSensor::PackedIrLedProcessorConfig config);
+
+private:
+ // This is nn::irsensor::IrLedProcessorConfig
+ struct IrLedProcessorConfig {
+ Core::IrSensor::CameraLightTarget light_target;
+ };
+ static_assert(sizeof(IrLedProcessorConfig) == 0x4, "IrLedProcessorConfig is an invalid size");
+
+ struct IrLedProcessorState {
+ s64 sampling_number;
+ u64 timestamp;
+ std::array<u8, 0x8> data;
+ };
+ static_assert(sizeof(IrLedProcessorState) == 0x18, "IrLedProcessorState is an invalid size");
+
+ IrLedProcessorConfig current_config{};
+ Core::IrSensor::DeviceFormat& device;
+};
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/irs_types.h b/src/hid_core/irsensor/irs_types.h
new file mode 100644
index 000000000..017f38e6c
--- /dev/null
+++ b/src/hid_core/irsensor/irs_types.h
@@ -0,0 +1,301 @@
+// SPDX-FileCopyrightText: Copyright 2021 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/common_funcs.h"
+#include "common/common_types.h"
+#include "hid_core/hid_types.h"
+
+namespace Core::IrSensor {
+
+// This is nn::irsensor::CameraAmbientNoiseLevel
+enum class CameraAmbientNoiseLevel : u32 {
+ Low,
+ Medium,
+ High,
+ Unknown3, // This level can't be reached
+};
+
+// This is nn::irsensor::CameraLightTarget
+enum class CameraLightTarget : u32 {
+ AllLeds,
+ BrightLeds,
+ DimLeds,
+ None,
+};
+
+// This is nn::irsensor::PackedCameraLightTarget
+enum class PackedCameraLightTarget : u8 {
+ AllLeds,
+ BrightLeds,
+ DimLeds,
+ None,
+};
+
+// This is nn::irsensor::AdaptiveClusteringMode
+enum class AdaptiveClusteringMode : u32 {
+ StaticFov,
+ DynamicFov,
+};
+
+// This is nn::irsensor::AdaptiveClusteringTargetDistance
+enum class AdaptiveClusteringTargetDistance : u32 {
+ Near,
+ Middle,
+ Far,
+};
+
+// This is nn::irsensor::ImageTransferProcessorFormat
+enum class ImageTransferProcessorFormat : u32 {
+ Size320x240,
+ Size160x120,
+ Size80x60,
+ Size40x30,
+ Size20x15,
+};
+
+// This is nn::irsensor::PackedImageTransferProcessorFormat
+enum class PackedImageTransferProcessorFormat : u8 {
+ Size320x240,
+ Size160x120,
+ Size80x60,
+ Size40x30,
+ Size20x15,
+};
+
+// This is nn::irsensor::IrCameraStatus
+enum class IrCameraStatus : u32 {
+ Available,
+ Unsupported,
+ Unconnected,
+};
+
+// This is nn::irsensor::IrCameraInternalStatus
+enum class IrCameraInternalStatus : u32 {
+ Stopped,
+ FirmwareUpdateNeeded,
+ Unknown2,
+ Unknown3,
+ Unknown4,
+ FirmwareVersionRequested,
+ FirmwareVersionIsInvalid,
+ Ready,
+ Setting,
+};
+
+// This is nn::irsensor::detail::StatusManager::IrSensorMode
+enum class IrSensorMode : u64 {
+ None,
+ MomentProcessor,
+ ClusteringProcessor,
+ ImageTransferProcessor,
+ PointingProcessorMarker,
+ TeraPluginProcessor,
+ IrLedProcessor,
+};
+
+// This is nn::irsensor::ImageProcessorStatus
+enum ImageProcessorStatus : u32 {
+ Stopped,
+ Running,
+};
+
+// This is nn::irsensor::HandAnalysisMode
+enum class HandAnalysisMode : u32 {
+ None,
+ Silhouette,
+ Image,
+ SilhoueteAndImage,
+ SilhuetteOnly,
+};
+
+// This is nn::irsensor::IrSensorFunctionLevel
+enum class IrSensorFunctionLevel : u8 {
+ unknown0,
+ unknown1,
+ unknown2,
+ unknown3,
+ unknown4,
+};
+
+// This is nn::irsensor::MomentProcessorPreprocess
+enum class MomentProcessorPreprocess : u32 {
+ Unknown0,
+ Unknown1,
+};
+
+// This is nn::irsensor::PackedMomentProcessorPreprocess
+enum class PackedMomentProcessorPreprocess : u8 {
+ Unknown0,
+ Unknown1,
+};
+
+// This is nn::irsensor::PointingStatus
+enum class PointingStatus : u32 {
+ Unknown0,
+ Unknown1,
+};
+
+struct IrsRect {
+ s16 x;
+ s16 y;
+ s16 width;
+ s16 height;
+};
+
+struct IrsCentroid {
+ f32 x;
+ f32 y;
+};
+
+struct CameraConfig {
+ u64 exposure_time;
+ CameraLightTarget light_target;
+ u32 gain;
+ bool is_negative_used;
+ INSERT_PADDING_BYTES(7);
+};
+static_assert(sizeof(CameraConfig) == 0x18, "CameraConfig is an invalid size");
+
+struct PackedCameraConfig {
+ u64 exposure_time;
+ PackedCameraLightTarget light_target;
+ u8 gain;
+ bool is_negative_used;
+ INSERT_PADDING_BYTES(5);
+};
+static_assert(sizeof(PackedCameraConfig) == 0x10, "PackedCameraConfig is an invalid size");
+
+// This is nn::irsensor::IrCameraHandle
+struct IrCameraHandle {
+ u8 npad_id{};
+ Core::HID::NpadStyleIndex npad_type{Core::HID::NpadStyleIndex::None};
+ INSERT_PADDING_BYTES(2);
+};
+static_assert(sizeof(IrCameraHandle) == 4, "IrCameraHandle is an invalid size");
+
+// This is nn::irsensor::PackedMcuVersion
+struct PackedMcuVersion {
+ u16 major;
+ u16 minor;
+};
+static_assert(sizeof(PackedMcuVersion) == 4, "PackedMcuVersion is an invalid size");
+
+// This is nn::irsensor::PackedMomentProcessorConfig
+struct PackedMomentProcessorConfig {
+ PackedCameraConfig camera_config;
+ IrsRect window_of_interest;
+ PackedMcuVersion required_mcu_version;
+ PackedMomentProcessorPreprocess preprocess;
+ u8 preprocess_intensity_threshold;
+ INSERT_PADDING_BYTES(2);
+};
+static_assert(sizeof(PackedMomentProcessorConfig) == 0x20,
+ "PackedMomentProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedClusteringProcessorConfig
+struct PackedClusteringProcessorConfig {
+ PackedCameraConfig camera_config;
+ IrsRect window_of_interest;
+ PackedMcuVersion required_mcu_version;
+ u32 pixel_count_min;
+ u32 pixel_count_max;
+ u8 object_intensity_min;
+ bool is_external_light_filter_enabled;
+ INSERT_PADDING_BYTES(2);
+};
+static_assert(sizeof(PackedClusteringProcessorConfig) == 0x28,
+ "PackedClusteringProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedImageTransferProcessorConfig
+struct PackedImageTransferProcessorConfig {
+ PackedCameraConfig camera_config;
+ PackedMcuVersion required_mcu_version;
+ PackedImageTransferProcessorFormat format;
+ INSERT_PADDING_BYTES(3);
+};
+static_assert(sizeof(PackedImageTransferProcessorConfig) == 0x18,
+ "PackedImageTransferProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedTeraPluginProcessorConfig
+struct PackedTeraPluginProcessorConfig {
+ PackedMcuVersion required_mcu_version;
+ u8 mode;
+ u8 unknown_1;
+ u8 unknown_2;
+ u8 unknown_3;
+};
+static_assert(sizeof(PackedTeraPluginProcessorConfig) == 0x8,
+ "PackedTeraPluginProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedPointingProcessorConfig
+struct PackedPointingProcessorConfig {
+ IrsRect window_of_interest;
+ PackedMcuVersion required_mcu_version;
+};
+static_assert(sizeof(PackedPointingProcessorConfig) == 0xC,
+ "PackedPointingProcessorConfig is an invalid size");
+
+// This is nn::irsensor::PackedFunctionLevel
+struct PackedFunctionLevel {
+ IrSensorFunctionLevel function_level;
+ INSERT_PADDING_BYTES(3);
+};
+static_assert(sizeof(PackedFunctionLevel) == 0x4, "PackedFunctionLevel is an invalid size");
+
+// This is nn::irsensor::PackedImageTransferProcessorExConfig
+struct PackedImageTransferProcessorExConfig {
+ PackedCameraConfig camera_config;
+ PackedMcuVersion required_mcu_version;
+ PackedImageTransferProcessorFormat origin_format;
+ PackedImageTransferProcessorFormat trimming_format;
+ u16 trimming_start_x;
+ u16 trimming_start_y;
+ bool is_external_light_filter_enabled;
+ INSERT_PADDING_BYTES(5);
+};
+static_assert(sizeof(PackedImageTransferProcessorExConfig) == 0x20,
+ "PackedImageTransferProcessorExConfig is an invalid size");
+
+// This is nn::irsensor::PackedIrLedProcessorConfig
+struct PackedIrLedProcessorConfig {
+ PackedMcuVersion required_mcu_version;
+ u8 light_target;
+ INSERT_PADDING_BYTES(3);
+};
+static_assert(sizeof(PackedIrLedProcessorConfig) == 0x8,
+ "PackedIrLedProcessorConfig is an invalid size");
+
+// This is nn::irsensor::HandAnalysisConfig
+struct HandAnalysisConfig {
+ HandAnalysisMode mode;
+};
+static_assert(sizeof(HandAnalysisConfig) == 0x4, "HandAnalysisConfig is an invalid size");
+
+// This is nn::irsensor::detail::ProcessorState contents are different for each processor
+struct ProcessorState {
+ std::array<u8, 0xE20> processor_raw_data{};
+};
+static_assert(sizeof(ProcessorState) == 0xE20, "ProcessorState is an invalid size");
+
+// This is nn::irsensor::detail::DeviceFormat
+struct DeviceFormat {
+ Core::IrSensor::IrCameraStatus camera_status{Core::IrSensor::IrCameraStatus::Unconnected};
+ Core::IrSensor::IrCameraInternalStatus camera_internal_status{
+ Core::IrSensor::IrCameraInternalStatus::Ready};
+ Core::IrSensor::IrSensorMode mode{Core::IrSensor::IrSensorMode::None};
+ ProcessorState state{};
+};
+static_assert(sizeof(DeviceFormat) == 0xE30, "DeviceFormat is an invalid size");
+
+// This is nn::irsensor::ImageTransferProcessorState
+struct ImageTransferProcessorState {
+ u64 sampling_number;
+ Core::IrSensor::CameraAmbientNoiseLevel ambient_noise_level;
+ INSERT_PADDING_BYTES(4);
+};
+static_assert(sizeof(ImageTransferProcessorState) == 0x10,
+ "ImageTransferProcessorState is an invalid size");
+
+} // namespace Core::IrSensor
diff --git a/src/hid_core/irsensor/moment_processor.cpp b/src/hid_core/irsensor/moment_processor.cpp
new file mode 100644
index 000000000..0284a58bd
--- /dev/null
+++ b/src/hid_core/irsensor/moment_processor.cpp
@@ -0,0 +1,149 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "core/core.h"
+#include "core/core_timing.h"
+#include "hid_core/frontend/emulated_controller.h"
+#include "hid_core/hid_core.h"
+#include "hid_core/irsensor/moment_processor.h"
+
+namespace Service::IRS {
+static constexpr auto format = Core::IrSensor::ImageTransferProcessorFormat::Size40x30;
+static constexpr std::size_t ImageWidth = 40;
+static constexpr std::size_t ImageHeight = 30;
+
+MomentProcessor::MomentProcessor(Core::System& system_, Core::IrSensor::DeviceFormat& device_format,
+ std::size_t npad_index)
+ : device(device_format), system{system_} {
+ npad_device = system.HIDCore().GetEmulatedControllerByIndex(npad_index);
+
+ device.mode = Core::IrSensor::IrSensorMode::MomentProcessor;
+ device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Stopped;
+
+ shared_memory = std::construct_at(
+ reinterpret_cast<MomentSharedMemory*>(&device_format.state.processor_raw_data));
+
+ Core::HID::ControllerUpdateCallback engine_callback{
+ .on_change = [this](Core::HID::ControllerTriggerType type) { OnControllerUpdate(type); },
+ .is_npad_service = true,
+ };
+ callback_key = npad_device->SetCallback(engine_callback);
+}
+
+MomentProcessor::~MomentProcessor() {
+ npad_device->DeleteCallback(callback_key);
+};
+
+void MomentProcessor::StartProcessor() {
+ device.camera_status = Core::IrSensor::IrCameraStatus::Available;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Ready;
+}
+
+void MomentProcessor::SuspendProcessor() {}
+
+void MomentProcessor::StopProcessor() {}
+
+void MomentProcessor::OnControllerUpdate(Core::HID::ControllerTriggerType type) {
+ if (type != Core::HID::ControllerTriggerType::IrSensor) {
+ return;
+ }
+
+ next_state = {};
+ const auto& camera_data = npad_device->GetCamera();
+
+ const auto window_width = static_cast<std::size_t>(current_config.window_of_interest.width);
+ const auto window_height = static_cast<std::size_t>(current_config.window_of_interest.height);
+ const auto window_start_x = static_cast<std::size_t>(current_config.window_of_interest.x);
+ const auto window_start_y = static_cast<std::size_t>(current_config.window_of_interest.y);
+
+ const std::size_t block_width = window_width / Columns;
+ const std::size_t block_height = window_height / Rows;
+
+ for (std::size_t row = 0; row < Rows; row++) {
+ for (std::size_t column = 0; column < Columns; column++) {
+ const size_t x_pos = (column * block_width) + window_start_x;
+ const size_t y_pos = (row * block_height) + window_start_y;
+ auto& statistic = next_state.statistic[column + (row * Columns)];
+ statistic = GetStatistic(camera_data.data, x_pos, y_pos, block_width, block_height);
+ }
+ }
+
+ next_state.sampling_number = camera_data.sample;
+ next_state.timestamp = system.CoreTiming().GetGlobalTimeNs().count();
+ next_state.ambient_noise_level = Core::IrSensor::CameraAmbientNoiseLevel::Low;
+ shared_memory->moment_lifo.WriteNextEntry(next_state);
+
+ if (!IsProcessorActive()) {
+ StartProcessor();
+ }
+}
+
+u8 MomentProcessor::GetPixel(const std::vector<u8>& data, std::size_t x, std::size_t y) const {
+ if ((y * ImageWidth) + x >= data.size()) {
+ return 0;
+ }
+ return data[(y * ImageWidth) + x];
+}
+
+MomentProcessor::MomentStatistic MomentProcessor::GetStatistic(const std::vector<u8>& data,
+ std::size_t start_x,
+ std::size_t start_y,
+ std::size_t width,
+ std::size_t height) const {
+ // The actual implementation is always 320x240
+ static constexpr std::size_t RealWidth = 320;
+ static constexpr std::size_t RealHeight = 240;
+ static constexpr std::size_t Threshold = 30;
+ MomentStatistic statistic{};
+ std::size_t active_points{};
+
+ // Sum all data points on the block that meet with the threshold
+ for (std::size_t y = 0; y < width; y++) {
+ for (std::size_t x = 0; x < height; x++) {
+ const size_t x_pos = x + start_x;
+ const size_t y_pos = y + start_y;
+ const auto pixel =
+ GetPixel(data, x_pos * ImageWidth / RealWidth, y_pos * ImageHeight / RealHeight);
+
+ if (pixel < Threshold) {
+ continue;
+ }
+
+ statistic.average_intensity += pixel;
+
+ statistic.centroid.x += static_cast<float>(x_pos);
+ statistic.centroid.y += static_cast<float>(y_pos);
+
+ active_points++;
+ }
+ }
+
+ // Return an empty field if no points were available
+ if (active_points == 0) {
+ return {};
+ }
+
+ // Finally calculate the actual centroid and average intensity
+ statistic.centroid.x /= static_cast<float>(active_points);
+ statistic.centroid.y /= static_cast<float>(active_points);
+ statistic.average_intensity /= static_cast<f32>(width * height);
+
+ return statistic;
+}
+
+void MomentProcessor::SetConfig(Core::IrSensor::PackedMomentProcessorConfig config) {
+ current_config.camera_config.exposure_time = config.camera_config.exposure_time;
+ current_config.camera_config.gain = config.camera_config.gain;
+ current_config.camera_config.is_negative_used = config.camera_config.is_negative_used;
+ current_config.camera_config.light_target =
+ static_cast<Core::IrSensor::CameraLightTarget>(config.camera_config.light_target);
+ current_config.window_of_interest = config.window_of_interest;
+ current_config.preprocess =
+ static_cast<Core::IrSensor::MomentProcessorPreprocess>(config.preprocess);
+ current_config.preprocess_intensity_threshold = config.preprocess_intensity_threshold;
+
+ npad_device->SetCameraFormat(format);
+}
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/moment_processor.h b/src/hid_core/irsensor/moment_processor.h
new file mode 100644
index 000000000..78c9c035f
--- /dev/null
+++ b/src/hid_core/irsensor/moment_processor.h
@@ -0,0 +1,91 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/bit_field.h"
+#include "common/common_types.h"
+#include "hid_core/irsensor/irs_types.h"
+#include "hid_core/irsensor/processor_base.h"
+#include "hid_core/resources/irs_ring_lifo.h"
+
+namespace Core {
+class System;
+}
+
+namespace Core::HID {
+class EmulatedController;
+} // namespace Core::HID
+
+namespace Service::IRS {
+class MomentProcessor final : public ProcessorBase {
+public:
+ explicit MomentProcessor(Core::System& system_, Core::IrSensor::DeviceFormat& device_format,
+ std::size_t npad_index);
+ ~MomentProcessor() override;
+
+ // Called when the processor is initialized
+ void StartProcessor() override;
+
+ // Called when the processor is suspended
+ void SuspendProcessor() override;
+
+ // Called when the processor is stopped
+ void StopProcessor() override;
+
+ // Sets config parameters of the camera
+ void SetConfig(Core::IrSensor::PackedMomentProcessorConfig config);
+
+private:
+ static constexpr std::size_t Columns = 8;
+ static constexpr std::size_t Rows = 6;
+
+ // This is nn::irsensor::MomentProcessorConfig
+ struct MomentProcessorConfig {
+ Core::IrSensor::CameraConfig camera_config;
+ Core::IrSensor::IrsRect window_of_interest;
+ Core::IrSensor::MomentProcessorPreprocess preprocess;
+ u32 preprocess_intensity_threshold;
+ };
+ static_assert(sizeof(MomentProcessorConfig) == 0x28,
+ "MomentProcessorConfig is an invalid size");
+
+ // This is nn::irsensor::MomentStatistic
+ struct MomentStatistic {
+ f32 average_intensity;
+ Core::IrSensor::IrsCentroid centroid;
+ };
+ static_assert(sizeof(MomentStatistic) == 0xC, "MomentStatistic is an invalid size");
+
+ // This is nn::irsensor::MomentProcessorState
+ struct MomentProcessorState {
+ s64 sampling_number;
+ u64 timestamp;
+ Core::IrSensor::CameraAmbientNoiseLevel ambient_noise_level;
+ INSERT_PADDING_BYTES(4);
+ std::array<MomentStatistic, Columns * Rows> statistic;
+ };
+ static_assert(sizeof(MomentProcessorState) == 0x258, "MomentProcessorState is an invalid size");
+
+ struct MomentSharedMemory {
+ Service::IRS::Lifo<MomentProcessorState, 6> moment_lifo;
+ };
+ static_assert(sizeof(MomentSharedMemory) == 0xE20, "MomentSharedMemory is an invalid size");
+
+ void OnControllerUpdate(Core::HID::ControllerTriggerType type);
+ u8 GetPixel(const std::vector<u8>& data, std::size_t x, std::size_t y) const;
+ MomentStatistic GetStatistic(const std::vector<u8>& data, std::size_t start_x,
+ std::size_t start_y, std::size_t width, std::size_t height) const;
+
+ MomentSharedMemory* shared_memory = nullptr;
+ MomentProcessorState next_state{};
+
+ MomentProcessorConfig current_config{};
+ Core::IrSensor::DeviceFormat& device;
+ Core::HID::EmulatedController* npad_device;
+ int callback_key{};
+
+ Core::System& system;
+};
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/pointing_processor.cpp b/src/hid_core/irsensor/pointing_processor.cpp
new file mode 100644
index 000000000..c1d6c1bb6
--- /dev/null
+++ b/src/hid_core/irsensor/pointing_processor.cpp
@@ -0,0 +1,26 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "hid_core/irsensor/pointing_processor.h"
+
+namespace Service::IRS {
+PointingProcessor::PointingProcessor(Core::IrSensor::DeviceFormat& device_format)
+ : device(device_format) {
+ device.mode = Core::IrSensor::IrSensorMode::PointingProcessorMarker;
+ device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Stopped;
+}
+
+PointingProcessor::~PointingProcessor() = default;
+
+void PointingProcessor::StartProcessor() {}
+
+void PointingProcessor::SuspendProcessor() {}
+
+void PointingProcessor::StopProcessor() {}
+
+void PointingProcessor::SetConfig(Core::IrSensor::PackedPointingProcessorConfig config) {
+ current_config.window_of_interest = config.window_of_interest;
+}
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/pointing_processor.h b/src/hid_core/irsensor/pointing_processor.h
new file mode 100644
index 000000000..968c2e5bd
--- /dev/null
+++ b/src/hid_core/irsensor/pointing_processor.h
@@ -0,0 +1,61 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/common_types.h"
+#include "hid_core/irsensor/irs_types.h"
+#include "hid_core/irsensor/processor_base.h"
+
+namespace Service::IRS {
+class PointingProcessor final : public ProcessorBase {
+public:
+ explicit PointingProcessor(Core::IrSensor::DeviceFormat& device_format);
+ ~PointingProcessor() override;
+
+ // Called when the processor is initialized
+ void StartProcessor() override;
+
+ // Called when the processor is suspended
+ void SuspendProcessor() override;
+
+ // Called when the processor is stopped
+ void StopProcessor() override;
+
+ // Sets config parameters of the camera
+ void SetConfig(Core::IrSensor::PackedPointingProcessorConfig config);
+
+private:
+ // This is nn::irsensor::PointingProcessorConfig
+ struct PointingProcessorConfig {
+ Core::IrSensor::IrsRect window_of_interest;
+ };
+ static_assert(sizeof(PointingProcessorConfig) == 0x8,
+ "PointingProcessorConfig is an invalid size");
+
+ struct PointingProcessorMarkerData {
+ u8 pointing_status;
+ INSERT_PADDING_BYTES(3);
+ u32 unknown;
+ float unknown_float1;
+ float position_x;
+ float position_y;
+ float unknown_float2;
+ Core::IrSensor::IrsRect window_of_interest;
+ };
+ static_assert(sizeof(PointingProcessorMarkerData) == 0x20,
+ "PointingProcessorMarkerData is an invalid size");
+
+ struct PointingProcessorMarkerState {
+ s64 sampling_number;
+ u64 timestamp;
+ std::array<PointingProcessorMarkerData, 0x3> data;
+ };
+ static_assert(sizeof(PointingProcessorMarkerState) == 0x70,
+ "PointingProcessorMarkerState is an invalid size");
+
+ PointingProcessorConfig current_config{};
+ Core::IrSensor::DeviceFormat& device;
+};
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/processor_base.cpp b/src/hid_core/irsensor/processor_base.cpp
new file mode 100644
index 000000000..91a513a70
--- /dev/null
+++ b/src/hid_core/irsensor/processor_base.cpp
@@ -0,0 +1,67 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "hid_core/irsensor/processor_base.h"
+
+namespace Service::IRS {
+
+ProcessorBase::ProcessorBase() {}
+ProcessorBase::~ProcessorBase() = default;
+
+bool ProcessorBase::IsProcessorActive() const {
+ return is_active;
+}
+
+std::size_t ProcessorBase::GetDataSize(Core::IrSensor::ImageTransferProcessorFormat format) const {
+ switch (format) {
+ case Core::IrSensor::ImageTransferProcessorFormat::Size320x240:
+ return 320 * 240;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size160x120:
+ return 160 * 120;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size80x60:
+ return 80 * 60;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size40x30:
+ return 40 * 30;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size20x15:
+ return 20 * 15;
+ default:
+ return 0;
+ }
+}
+
+std::size_t ProcessorBase::GetDataWidth(Core::IrSensor::ImageTransferProcessorFormat format) const {
+ switch (format) {
+ case Core::IrSensor::ImageTransferProcessorFormat::Size320x240:
+ return 320;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size160x120:
+ return 160;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size80x60:
+ return 80;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size40x30:
+ return 40;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size20x15:
+ return 20;
+ default:
+ return 0;
+ }
+}
+
+std::size_t ProcessorBase::GetDataHeight(
+ Core::IrSensor::ImageTransferProcessorFormat format) const {
+ switch (format) {
+ case Core::IrSensor::ImageTransferProcessorFormat::Size320x240:
+ return 240;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size160x120:
+ return 120;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size80x60:
+ return 60;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size40x30:
+ return 30;
+ case Core::IrSensor::ImageTransferProcessorFormat::Size20x15:
+ return 15;
+ default:
+ return 0;
+ }
+}
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/processor_base.h b/src/hid_core/irsensor/processor_base.h
new file mode 100644
index 000000000..48beeafba
--- /dev/null
+++ b/src/hid_core/irsensor/processor_base.h
@@ -0,0 +1,33 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/common_types.h"
+#include "hid_core/irsensor/irs_types.h"
+
+namespace Service::IRS {
+class ProcessorBase {
+public:
+ explicit ProcessorBase();
+ virtual ~ProcessorBase();
+
+ virtual void StartProcessor() = 0;
+ virtual void SuspendProcessor() = 0;
+ virtual void StopProcessor() = 0;
+
+ bool IsProcessorActive() const;
+
+protected:
+ /// Returns the number of bytes the image uses
+ std::size_t GetDataSize(Core::IrSensor::ImageTransferProcessorFormat format) const;
+
+ /// Returns the width of the image
+ std::size_t GetDataWidth(Core::IrSensor::ImageTransferProcessorFormat format) const;
+
+ /// Returns the height of the image
+ std::size_t GetDataHeight(Core::IrSensor::ImageTransferProcessorFormat format) const;
+
+ bool is_active{false};
+};
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/tera_plugin_processor.cpp b/src/hid_core/irsensor/tera_plugin_processor.cpp
new file mode 100644
index 000000000..2382e208a
--- /dev/null
+++ b/src/hid_core/irsensor/tera_plugin_processor.cpp
@@ -0,0 +1,29 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#include "hid_core/irsensor/tera_plugin_processor.h"
+
+namespace Service::IRS {
+TeraPluginProcessor::TeraPluginProcessor(Core::IrSensor::DeviceFormat& device_format)
+ : device(device_format) {
+ device.mode = Core::IrSensor::IrSensorMode::TeraPluginProcessor;
+ device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
+ device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Stopped;
+}
+
+TeraPluginProcessor::~TeraPluginProcessor() = default;
+
+void TeraPluginProcessor::StartProcessor() {}
+
+void TeraPluginProcessor::SuspendProcessor() {}
+
+void TeraPluginProcessor::StopProcessor() {}
+
+void TeraPluginProcessor::SetConfig(Core::IrSensor::PackedTeraPluginProcessorConfig config) {
+ current_config.mode = config.mode;
+ current_config.unknown_1 = config.unknown_1;
+ current_config.unknown_2 = config.unknown_2;
+ current_config.unknown_3 = config.unknown_3;
+}
+
+} // namespace Service::IRS
diff --git a/src/hid_core/irsensor/tera_plugin_processor.h b/src/hid_core/irsensor/tera_plugin_processor.h
new file mode 100644
index 000000000..dc8fe7d07
--- /dev/null
+++ b/src/hid_core/irsensor/tera_plugin_processor.h
@@ -0,0 +1,53 @@
+// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
+// SPDX-License-Identifier: GPL-3.0-or-later
+
+#pragma once
+
+#include "common/bit_field.h"
+#include "common/common_types.h"
+#include "hid_core/irsensor/irs_types.h"
+#include "hid_core/irsensor/processor_base.h"
+
+namespace Service::IRS {
+class TeraPluginProcessor final : public ProcessorBase {
+public:
+ explicit TeraPluginProcessor(Core::IrSensor::DeviceFormat& device_format);
+ ~TeraPluginProcessor() override;
+
+ // Called when the processor is initialized
+ void StartProcessor() override;
+
+ // Called when the processor is suspended
+ void SuspendProcessor() override;
+
+ // Called when the processor is stopped
+ void StopProcessor() override;
+
+ // Sets config parameters of the camera
+ void SetConfig(Core::IrSensor::PackedTeraPluginProcessorConfig config);
+
+private:
+ // This is nn::irsensor::TeraPluginProcessorConfig
+ struct TeraPluginProcessorConfig {
+ u8 mode;
+ u8 unknown_1;
+ u8 unknown_2;
+ u8 unknown_3;
+ };
+ static_assert(sizeof(TeraPluginProcessorConfig) == 0x4,
+ "TeraPluginProcessorConfig is an invalid size");
+
+ struct TeraPluginProcessorState {
+ s64 sampling_number;
+ u64 timestamp;
+ Core::IrSensor::CameraAmbientNoiseLevel ambient_noise_level;
+ std::array<u8, 0x12c> data;
+ };
+ static_assert(sizeof(TeraPluginProcessorState) == 0x140,
+ "TeraPluginProcessorState is an invalid size");
+
+ TeraPluginProcessorConfig current_config{};
+ Core::IrSensor::DeviceFormat& device;
+};
+
+} // namespace Service::IRS