mirror of
https://github.com/yuzu-mirror/yuzu.git
synced 2024-11-19 14:09:59 +00:00
service: irs: Implement moment image processor
This commit is contained in:
parent
2f9487cd38
commit
e588f341ed
6 changed files with 169 additions and 17 deletions
|
@ -138,7 +138,7 @@ void IRS::RunMomentProcessor(HLERequestContext& ctx) {
|
|||
|
||||
if (result.IsSuccess()) {
|
||||
auto& device = GetIrCameraSharedMemoryDeviceEntry(parameters.camera_handle);
|
||||
MakeProcessor<MomentProcessor>(parameters.camera_handle, device);
|
||||
MakeProcessorWithCoreContext<MomentProcessor>(parameters.camera_handle, device);
|
||||
auto& image_transfer_processor = GetProcessor<MomentProcessor>(parameters.camera_handle);
|
||||
image_transfer_processor.SetConfig(parameters.processor_config);
|
||||
npad_device->SetPollingMode(Core::HID::EmulatedDeviceIndex::RightIndex,
|
||||
|
|
|
@ -3,16 +3,18 @@
|
|||
|
||||
#include <queue>
|
||||
|
||||
#include "core/core.h"
|
||||
#include "core/core_timing.h"
|
||||
#include "core/hid/emulated_controller.h"
|
||||
#include "core/hid/hid_core.h"
|
||||
#include "core/hle/service/hid/irsensor/clustering_processor.h"
|
||||
|
||||
namespace Service::IRS {
|
||||
ClusteringProcessor::ClusteringProcessor(Core::HID::HIDCore& hid_core_,
|
||||
ClusteringProcessor::ClusteringProcessor(Core::System& system_,
|
||||
Core::IrSensor::DeviceFormat& device_format,
|
||||
std::size_t npad_index)
|
||||
: device{device_format} {
|
||||
npad_device = hid_core_.GetEmulatedControllerByIndex(npad_index);
|
||||
: device{device_format}, system{system_} {
|
||||
npad_device = system.HIDCore().GetEmulatedControllerByIndex(npad_index);
|
||||
|
||||
device.mode = Core::IrSensor::IrSensorMode::ClusteringProcessor;
|
||||
device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
|
||||
|
@ -48,7 +50,7 @@ void ClusteringProcessor::OnControllerUpdate(Core::HID::ControllerTriggerType ty
|
|||
}
|
||||
|
||||
next_state = {};
|
||||
const auto camera_data = npad_device->GetCamera();
|
||||
const auto& camera_data = npad_device->GetCamera();
|
||||
auto filtered_image = camera_data.data;
|
||||
|
||||
RemoveLowIntensityData(filtered_image);
|
||||
|
@ -83,7 +85,7 @@ void ClusteringProcessor::OnControllerUpdate(Core::HID::ControllerTriggerType ty
|
|||
}
|
||||
|
||||
next_state.sampling_number = camera_data.sample;
|
||||
next_state.timestamp = next_state.timestamp + 131;
|
||||
next_state.timestamp = system.CoreTiming().GetGlobalTimeNs().count();
|
||||
next_state.ambient_noise_level = Core::IrSensor::CameraAmbientNoiseLevel::Low;
|
||||
shared_memory->clustering_lifo.WriteNextEntry(next_state);
|
||||
|
||||
|
@ -202,14 +204,14 @@ ClusteringProcessor::ClusteringData ClusteringProcessor::MergeCluster(
|
|||
}
|
||||
|
||||
u8 ClusteringProcessor::GetPixel(const std::vector<u8>& data, std::size_t x, std::size_t y) const {
|
||||
if ((y * width) + x > data.size()) {
|
||||
if ((y * width) + x >= data.size()) {
|
||||
return 0;
|
||||
}
|
||||
return data[(y * width) + x];
|
||||
}
|
||||
|
||||
void ClusteringProcessor::SetPixel(std::vector<u8>& data, std::size_t x, std::size_t y, u8 value) {
|
||||
if ((y * width) + x > data.size()) {
|
||||
if ((y * width) + x >= data.size()) {
|
||||
return;
|
||||
}
|
||||
data[(y * width) + x] = value;
|
||||
|
|
|
@ -8,6 +8,10 @@
|
|||
#include "core/hle/service/hid/irs_ring_lifo.h"
|
||||
#include "core/hle/service/hid/irsensor/processor_base.h"
|
||||
|
||||
namespace Core {
|
||||
class System;
|
||||
}
|
||||
|
||||
namespace Core::HID {
|
||||
class EmulatedController;
|
||||
} // namespace Core::HID
|
||||
|
@ -15,8 +19,7 @@ class EmulatedController;
|
|||
namespace Service::IRS {
|
||||
class ClusteringProcessor final : public ProcessorBase {
|
||||
public:
|
||||
explicit ClusteringProcessor(Core::HID::HIDCore& hid_core_,
|
||||
Core::IrSensor::DeviceFormat& device_format,
|
||||
explicit ClusteringProcessor(Core::System& system_, Core::IrSensor::DeviceFormat& device_format,
|
||||
std::size_t npad_index);
|
||||
~ClusteringProcessor() override;
|
||||
|
||||
|
@ -106,5 +109,7 @@ private:
|
|||
Core::IrSensor::DeviceFormat& device;
|
||||
Core::HID::EmulatedController* npad_device;
|
||||
int callback_key{};
|
||||
|
||||
Core::System& system;
|
||||
};
|
||||
} // namespace Service::IRS
|
||||
|
|
|
@ -49,7 +49,7 @@ void ImageTransferProcessor::OnControllerUpdate(Core::HID::ControllerTriggerType
|
|||
return;
|
||||
}
|
||||
|
||||
const auto camera_data = npad_device->GetCamera();
|
||||
const auto& camera_data = npad_device->GetCamera();
|
||||
|
||||
// This indicates how much ambient light is present
|
||||
processor_state.ambient_noise_level = Core::IrSensor::CameraAmbientNoiseLevel::Low;
|
||||
|
|
|
@ -1,24 +1,137 @@
|
|||
// SPDX-FileCopyrightText: Copyright 2022 yuzu Emulator Project
|
||||
// SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
#include "core/core.h"
|
||||
#include "core/core_timing.h"
|
||||
#include "core/hid/emulated_controller.h"
|
||||
#include "core/hid/hid_core.h"
|
||||
#include "core/hle/service/hid/irsensor/moment_processor.h"
|
||||
|
||||
namespace Service::IRS {
|
||||
MomentProcessor::MomentProcessor(Core::IrSensor::DeviceFormat& device_format)
|
||||
: device(device_format) {
|
||||
static constexpr auto format = Core::IrSensor::ImageTransferProcessorFormat::Size40x30;
|
||||
static constexpr std::size_t ImageWidth = 40;
|
||||
static constexpr std::size_t ImageHeight = 30;
|
||||
|
||||
MomentProcessor::MomentProcessor(Core::System& system_, Core::IrSensor::DeviceFormat& device_format,
|
||||
std::size_t npad_index)
|
||||
: device(device_format), system{system_} {
|
||||
npad_device = system.HIDCore().GetEmulatedControllerByIndex(npad_index);
|
||||
|
||||
device.mode = Core::IrSensor::IrSensorMode::MomentProcessor;
|
||||
device.camera_status = Core::IrSensor::IrCameraStatus::Unconnected;
|
||||
device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Stopped;
|
||||
|
||||
shared_memory = std::construct_at(
|
||||
reinterpret_cast<MomentSharedMemory*>(&device_format.state.processor_raw_data));
|
||||
|
||||
Core::HID::ControllerUpdateCallback engine_callback{
|
||||
.on_change = [this](Core::HID::ControllerTriggerType type) { OnControllerUpdate(type); },
|
||||
.is_npad_service = true,
|
||||
};
|
||||
callback_key = npad_device->SetCallback(engine_callback);
|
||||
}
|
||||
|
||||
MomentProcessor::~MomentProcessor() = default;
|
||||
MomentProcessor::~MomentProcessor() {
|
||||
npad_device->DeleteCallback(callback_key);
|
||||
};
|
||||
|
||||
void MomentProcessor::StartProcessor() {}
|
||||
void MomentProcessor::StartProcessor() {
|
||||
device.camera_status = Core::IrSensor::IrCameraStatus::Available;
|
||||
device.camera_internal_status = Core::IrSensor::IrCameraInternalStatus::Ready;
|
||||
}
|
||||
|
||||
void MomentProcessor::SuspendProcessor() {}
|
||||
|
||||
void MomentProcessor::StopProcessor() {}
|
||||
|
||||
void MomentProcessor::OnControllerUpdate(Core::HID::ControllerTriggerType type) {
|
||||
if (type != Core::HID::ControllerTriggerType::IrSensor) {
|
||||
return;
|
||||
}
|
||||
|
||||
next_state = {};
|
||||
const auto& camera_data = npad_device->GetCamera();
|
||||
|
||||
const auto window_width = static_cast<std::size_t>(current_config.window_of_interest.width);
|
||||
const auto window_height = static_cast<std::size_t>(current_config.window_of_interest.height);
|
||||
const auto window_start_x = static_cast<std::size_t>(current_config.window_of_interest.x);
|
||||
const auto window_start_y = static_cast<std::size_t>(current_config.window_of_interest.y);
|
||||
|
||||
const std::size_t block_width = window_width / Columns;
|
||||
const std::size_t block_height = window_height / Rows;
|
||||
|
||||
for (std::size_t row = 0; row < Rows; row++) {
|
||||
for (std::size_t column = 0; column < Columns; column++) {
|
||||
const size_t x_pos = (column * block_width) + window_start_x;
|
||||
const size_t y_pos = (row * block_height) + window_start_y;
|
||||
auto& statistic = next_state.statistic[column + (row * Columns)];
|
||||
statistic = GetStatistic(camera_data.data, x_pos, y_pos, block_width, block_height);
|
||||
}
|
||||
}
|
||||
|
||||
next_state.sampling_number = camera_data.sample;
|
||||
next_state.timestamp = system.CoreTiming().GetGlobalTimeNs().count();
|
||||
next_state.ambient_noise_level = Core::IrSensor::CameraAmbientNoiseLevel::Low;
|
||||
shared_memory->moment_lifo.WriteNextEntry(next_state);
|
||||
|
||||
if (!IsProcessorActive()) {
|
||||
StartProcessor();
|
||||
}
|
||||
}
|
||||
|
||||
u8 MomentProcessor::GetPixel(const std::vector<u8>& data, std::size_t x, std::size_t y) const {
|
||||
if ((y * ImageWidth) + x >= data.size()) {
|
||||
return 0;
|
||||
}
|
||||
return data[(y * ImageWidth) + x];
|
||||
}
|
||||
|
||||
MomentProcessor::MomentStatistic MomentProcessor::GetStatistic(const std::vector<u8>& data,
|
||||
std::size_t start_x,
|
||||
std::size_t start_y,
|
||||
std::size_t width,
|
||||
std::size_t height) const {
|
||||
// The actual implementation is always 320x240
|
||||
static constexpr std::size_t RealWidth = 320;
|
||||
static constexpr std::size_t RealHeight = 240;
|
||||
static constexpr std::size_t Threshold = 30;
|
||||
MomentStatistic statistic{};
|
||||
std::size_t active_points{};
|
||||
|
||||
// Sum all data points on the block that meet with the threshold
|
||||
for (std::size_t y = 0; y < width; y++) {
|
||||
for (std::size_t x = 0; x < height; x++) {
|
||||
const size_t x_pos = x + start_x;
|
||||
const size_t y_pos = y + start_y;
|
||||
const auto pixel =
|
||||
GetPixel(data, x_pos * ImageWidth / RealWidth, y_pos * ImageHeight / RealHeight);
|
||||
|
||||
if (pixel < Threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
statistic.average_intensity += pixel;
|
||||
|
||||
statistic.centroid.x += static_cast<float>(x_pos);
|
||||
statistic.centroid.y += static_cast<float>(y_pos);
|
||||
|
||||
active_points++;
|
||||
}
|
||||
}
|
||||
|
||||
// Return an empty field if no points were available
|
||||
if (active_points == 0) {
|
||||
return {};
|
||||
}
|
||||
|
||||
// Finally calculate the actual centroid and average intensity
|
||||
statistic.centroid.x /= static_cast<float>(active_points);
|
||||
statistic.centroid.y /= static_cast<float>(active_points);
|
||||
statistic.average_intensity /= static_cast<f32>(width * height);
|
||||
|
||||
return statistic;
|
||||
}
|
||||
|
||||
void MomentProcessor::SetConfig(Core::IrSensor::PackedMomentProcessorConfig config) {
|
||||
current_config.camera_config.exposure_time = config.camera_config.exposure_time;
|
||||
current_config.camera_config.gain = config.camera_config.gain;
|
||||
|
@ -29,6 +142,8 @@ void MomentProcessor::SetConfig(Core::IrSensor::PackedMomentProcessorConfig conf
|
|||
current_config.preprocess =
|
||||
static_cast<Core::IrSensor::MomentProcessorPreprocess>(config.preprocess);
|
||||
current_config.preprocess_intensity_threshold = config.preprocess_intensity_threshold;
|
||||
|
||||
npad_device->SetCameraFormat(format);
|
||||
}
|
||||
|
||||
} // namespace Service::IRS
|
||||
|
|
|
@ -6,12 +6,22 @@
|
|||
#include "common/bit_field.h"
|
||||
#include "common/common_types.h"
|
||||
#include "core/hid/irs_types.h"
|
||||
#include "core/hle/service/hid/irs_ring_lifo.h"
|
||||
#include "core/hle/service/hid/irsensor/processor_base.h"
|
||||
|
||||
namespace Core {
|
||||
class System;
|
||||
}
|
||||
|
||||
namespace Core::HID {
|
||||
class EmulatedController;
|
||||
} // namespace Core::HID
|
||||
|
||||
namespace Service::IRS {
|
||||
class MomentProcessor final : public ProcessorBase {
|
||||
public:
|
||||
explicit MomentProcessor(Core::IrSensor::DeviceFormat& device_format);
|
||||
explicit MomentProcessor(Core::System& system_, Core::IrSensor::DeviceFormat& device_format,
|
||||
std::size_t npad_index);
|
||||
~MomentProcessor() override;
|
||||
|
||||
// Called when the processor is initialized
|
||||
|
@ -27,6 +37,9 @@ public:
|
|||
void SetConfig(Core::IrSensor::PackedMomentProcessorConfig config);
|
||||
|
||||
private:
|
||||
static constexpr std::size_t Columns = 8;
|
||||
static constexpr std::size_t Rows = 6;
|
||||
|
||||
// This is nn::irsensor::MomentProcessorConfig
|
||||
struct MomentProcessorConfig {
|
||||
Core::IrSensor::CameraConfig camera_config;
|
||||
|
@ -50,12 +63,29 @@ private:
|
|||
u64 timestamp;
|
||||
Core::IrSensor::CameraAmbientNoiseLevel ambient_noise_level;
|
||||
INSERT_PADDING_BYTES(4);
|
||||
std::array<MomentStatistic, 0x30> stadistic;
|
||||
std::array<MomentStatistic, Columns * Rows> statistic;
|
||||
};
|
||||
static_assert(sizeof(MomentProcessorState) == 0x258, "MomentProcessorState is an invalid size");
|
||||
|
||||
struct MomentSharedMemory {
|
||||
Service::IRS::Lifo<MomentProcessorState, 6> moment_lifo;
|
||||
};
|
||||
static_assert(sizeof(MomentSharedMemory) == 0xE20, "MomentSharedMemory is an invalid size");
|
||||
|
||||
void OnControllerUpdate(Core::HID::ControllerTriggerType type);
|
||||
u8 GetPixel(const std::vector<u8>& data, std::size_t x, std::size_t y) const;
|
||||
MomentStatistic GetStatistic(const std::vector<u8>& data, std::size_t start_x,
|
||||
std::size_t start_y, std::size_t width, std::size_t height) const;
|
||||
|
||||
MomentSharedMemory* shared_memory = nullptr;
|
||||
MomentProcessorState next_state{};
|
||||
|
||||
MomentProcessorConfig current_config{};
|
||||
Core::IrSensor::DeviceFormat& device;
|
||||
Core::HID::EmulatedController* npad_device;
|
||||
int callback_key{};
|
||||
|
||||
Core::System& system;
|
||||
};
|
||||
|
||||
} // namespace Service::IRS
|
||||
|
|
Loading…
Reference in a new issue