kernel: convert KConditionVariable, KLightConditionVariable, KLightLock

This commit is contained in:
Liam 2023-03-06 20:53:58 -05:00
parent 097c25b164
commit fdf90c6d75
7 changed files with 77 additions and 75 deletions

View file

@ -45,7 +45,7 @@ Result KCodeMemory::Initialize(Core::DeviceMemory& device_memory, VAddr addr, si
m_is_mapped = false; m_is_mapped = false;
// We succeeded. // We succeeded.
return ResultSuccess; R_SUCCEED();
} }
void KCodeMemory::Finalize() { void KCodeMemory::Finalize() {
@ -80,7 +80,7 @@ Result KCodeMemory::Map(VAddr address, size_t size) {
// Mark ourselves as mapped. // Mark ourselves as mapped.
m_is_mapped = true; m_is_mapped = true;
return ResultSuccess; R_SUCCEED();
} }
Result KCodeMemory::Unmap(VAddr address, size_t size) { Result KCodeMemory::Unmap(VAddr address, size_t size) {
@ -97,7 +97,7 @@ Result KCodeMemory::Unmap(VAddr address, size_t size) {
// Mark ourselves as unmapped. // Mark ourselves as unmapped.
m_is_mapped = false; m_is_mapped = false;
return ResultSuccess; R_SUCCEED();
} }
Result KCodeMemory::MapToOwner(VAddr address, size_t size, Svc::MemoryPermission perm) { Result KCodeMemory::MapToOwner(VAddr address, size_t size, Svc::MemoryPermission perm) {
@ -131,7 +131,7 @@ Result KCodeMemory::MapToOwner(VAddr address, size_t size, Svc::MemoryPermission
// Mark ourselves as mapped. // Mark ourselves as mapped.
m_is_owner_mapped = true; m_is_owner_mapped = true;
return ResultSuccess; R_SUCCEED();
} }
Result KCodeMemory::UnmapFromOwner(VAddr address, size_t size) { Result KCodeMemory::UnmapFromOwner(VAddr address, size_t size) {
@ -147,7 +147,7 @@ Result KCodeMemory::UnmapFromOwner(VAddr address, size_t size) {
// Mark ourselves as unmapped. // Mark ourselves as unmapped.
m_is_owner_mapped = false; m_is_owner_mapped = false;
return ResultSuccess; R_SUCCEED();
} }
} // namespace Kernel } // namespace Kernel

View file

@ -98,17 +98,17 @@ public:
} // namespace } // namespace
KConditionVariable::KConditionVariable(Core::System& system_) KConditionVariable::KConditionVariable(Core::System& system)
: system{system_}, kernel{system.Kernel()} {} : m_system{system}, m_kernel{system.Kernel()} {}
KConditionVariable::~KConditionVariable() = default; KConditionVariable::~KConditionVariable() = default;
Result KConditionVariable::SignalToAddress(VAddr addr) { Result KConditionVariable::SignalToAddress(VAddr addr) {
KThread* owner_thread = GetCurrentThreadPointer(kernel); KThread* owner_thread = GetCurrentThreadPointer(m_kernel);
// Signal the address. // Signal the address.
{ {
KScopedSchedulerLock sl(kernel); KScopedSchedulerLock sl(m_kernel);
// Remove waiter thread. // Remove waiter thread.
bool has_waiters{}; bool has_waiters{};
@ -129,7 +129,7 @@ Result KConditionVariable::SignalToAddress(VAddr addr) {
// Write the value to userspace. // Write the value to userspace.
Result result{ResultSuccess}; Result result{ResultSuccess};
if (WriteToUser(system, addr, std::addressof(next_value))) [[likely]] { if (WriteToUser(m_system, addr, std::addressof(next_value))) [[likely]] {
result = ResultSuccess; result = ResultSuccess;
} else { } else {
result = ResultInvalidCurrentMemory; result = ResultInvalidCurrentMemory;
@ -145,26 +145,27 @@ Result KConditionVariable::SignalToAddress(VAddr addr) {
} }
Result KConditionVariable::WaitForAddress(Handle handle, VAddr addr, u32 value) { Result KConditionVariable::WaitForAddress(Handle handle, VAddr addr, u32 value) {
KThread* cur_thread = GetCurrentThreadPointer(kernel); KThread* cur_thread = GetCurrentThreadPointer(m_kernel);
ThreadQueueImplForKConditionVariableWaitForAddress wait_queue(kernel); ThreadQueueImplForKConditionVariableWaitForAddress wait_queue(m_kernel);
// Wait for the address. // Wait for the address.
KThread* owner_thread{}; KThread* owner_thread{};
{ {
KScopedSchedulerLock sl(kernel); KScopedSchedulerLock sl(m_kernel);
// Check if the thread should terminate. // Check if the thread should terminate.
R_UNLESS(!cur_thread->IsTerminationRequested(), ResultTerminationRequested); R_UNLESS(!cur_thread->IsTerminationRequested(), ResultTerminationRequested);
// Read the tag from userspace. // Read the tag from userspace.
u32 test_tag{}; u32 test_tag{};
R_UNLESS(ReadFromUser(system, std::addressof(test_tag), addr), ResultInvalidCurrentMemory); R_UNLESS(ReadFromUser(m_system, std::addressof(test_tag), addr),
ResultInvalidCurrentMemory);
// If the tag isn't the handle (with wait mask), we're done. // If the tag isn't the handle (with wait mask), we're done.
R_SUCCEED_IF(test_tag != (handle | Svc::HandleWaitMask)); R_SUCCEED_IF(test_tag != (handle | Svc::HandleWaitMask));
// Get the lock owner thread. // Get the lock owner thread.
owner_thread = GetCurrentProcess(kernel) owner_thread = GetCurrentProcess(m_kernel)
.GetHandleTable() .GetHandleTable()
.GetObjectWithoutPseudoHandle<KThread>(handle) .GetObjectWithoutPseudoHandle<KThread>(handle)
.ReleasePointerUnsafe(); .ReleasePointerUnsafe();
@ -184,12 +185,12 @@ Result KConditionVariable::WaitForAddress(Handle handle, VAddr addr, u32 value)
owner_thread->Close(); owner_thread->Close();
// Get the wait result. // Get the wait result.
return cur_thread->GetWaitResult(); R_RETURN(cur_thread->GetWaitResult());
} }
void KConditionVariable::SignalImpl(KThread* thread) { void KConditionVariable::SignalImpl(KThread* thread) {
// Check pre-conditions. // Check pre-conditions.
ASSERT(kernel.GlobalSchedulerContext().IsLocked()); ASSERT(KScheduler::IsSchedulerLockedByCurrentThread(m_kernel));
// Update the tag. // Update the tag.
VAddr address = thread->GetAddressKey(); VAddr address = thread->GetAddressKey();
@ -204,7 +205,7 @@ void KConditionVariable::SignalImpl(KThread* thread) {
// TODO(bunnei): We should call CanAccessAtomic(..) here. // TODO(bunnei): We should call CanAccessAtomic(..) here.
can_access = true; can_access = true;
if (can_access) [[likely]] { if (can_access) [[likely]] {
UpdateLockAtomic(system, std::addressof(prev_tag), address, own_tag, UpdateLockAtomic(m_system, std::addressof(prev_tag), address, own_tag,
Svc::HandleWaitMask); Svc::HandleWaitMask);
} }
} }
@ -215,7 +216,7 @@ void KConditionVariable::SignalImpl(KThread* thread) {
thread->EndWait(ResultSuccess); thread->EndWait(ResultSuccess);
} else { } else {
// Get the previous owner. // Get the previous owner.
KThread* owner_thread = GetCurrentProcess(kernel) KThread* owner_thread = GetCurrentProcess(m_kernel)
.GetHandleTable() .GetHandleTable()
.GetObjectWithoutPseudoHandle<KThread>( .GetObjectWithoutPseudoHandle<KThread>(
static_cast<Handle>(prev_tag & ~Svc::HandleWaitMask)) static_cast<Handle>(prev_tag & ~Svc::HandleWaitMask))
@ -240,14 +241,14 @@ void KConditionVariable::Signal(u64 cv_key, s32 count) {
// Perform signaling. // Perform signaling.
s32 num_waiters{}; s32 num_waiters{};
{ {
KScopedSchedulerLock sl(kernel); KScopedSchedulerLock sl(m_kernel);
auto it = thread_tree.nfind_key({cv_key, -1}); auto it = m_tree.nfind_key({cv_key, -1});
while ((it != thread_tree.end()) && (count <= 0 || num_waiters < count) && while ((it != m_tree.end()) && (count <= 0 || num_waiters < count) &&
(it->GetConditionVariableKey() == cv_key)) { (it->GetConditionVariableKey() == cv_key)) {
KThread* target_thread = std::addressof(*it); KThread* target_thread = std::addressof(*it);
it = thread_tree.erase(it); it = m_tree.erase(it);
target_thread->ClearConditionVariable(); target_thread->ClearConditionVariable();
this->SignalImpl(target_thread); this->SignalImpl(target_thread);
@ -256,27 +257,27 @@ void KConditionVariable::Signal(u64 cv_key, s32 count) {
} }
// If we have no waiters, clear the has waiter flag. // If we have no waiters, clear the has waiter flag.
if (it == thread_tree.end() || it->GetConditionVariableKey() != cv_key) { if (it == m_tree.end() || it->GetConditionVariableKey() != cv_key) {
const u32 has_waiter_flag{}; const u32 has_waiter_flag{};
WriteToUser(system, cv_key, std::addressof(has_waiter_flag)); WriteToUser(m_system, cv_key, std::addressof(has_waiter_flag));
} }
} }
} }
Result KConditionVariable::Wait(VAddr addr, u64 key, u32 value, s64 timeout) { Result KConditionVariable::Wait(VAddr addr, u64 key, u32 value, s64 timeout) {
// Prepare to wait. // Prepare to wait.
KThread* cur_thread = GetCurrentThreadPointer(kernel); KThread* cur_thread = GetCurrentThreadPointer(m_kernel);
KHardwareTimer* timer{}; KHardwareTimer* timer{};
ThreadQueueImplForKConditionVariableWaitConditionVariable wait_queue( ThreadQueueImplForKConditionVariableWaitConditionVariable wait_queue(m_kernel,
kernel, std::addressof(thread_tree)); std::addressof(m_tree));
{ {
KScopedSchedulerLockAndSleep slp(kernel, std::addressof(timer), cur_thread, timeout); KScopedSchedulerLockAndSleep slp(m_kernel, std::addressof(timer), cur_thread, timeout);
// Check that the thread isn't terminating. // Check that the thread isn't terminating.
if (cur_thread->IsTerminationRequested()) { if (cur_thread->IsTerminationRequested()) {
slp.CancelSleep(); slp.CancelSleep();
return ResultTerminationRequested; R_THROW(ResultTerminationRequested);
} }
// Update the value and process for the next owner. // Update the value and process for the next owner.
@ -302,14 +303,14 @@ Result KConditionVariable::Wait(VAddr addr, u64 key, u32 value, s64 timeout) {
// Write to the cv key. // Write to the cv key.
{ {
const u32 has_waiter_flag = 1; const u32 has_waiter_flag = 1;
WriteToUser(system, key, std::addressof(has_waiter_flag)); WriteToUser(m_system, key, std::addressof(has_waiter_flag));
// TODO(bunnei): We should call DataMemoryBarrier(..) here. std::atomic_thread_fence(std::memory_order_seq_cst);
} }
// Write the value to userspace. // Write the value to userspace.
if (!WriteToUser(system, addr, std::addressof(next_value))) { if (!WriteToUser(m_system, addr, std::addressof(next_value))) {
slp.CancelSleep(); slp.CancelSleep();
return ResultInvalidCurrentMemory; R_THROW(ResultInvalidCurrentMemory);
} }
} }
@ -317,8 +318,8 @@ Result KConditionVariable::Wait(VAddr addr, u64 key, u32 value, s64 timeout) {
R_UNLESS(timeout != 0, ResultTimedOut); R_UNLESS(timeout != 0, ResultTimedOut);
// Update condition variable tracking. // Update condition variable tracking.
cur_thread->SetConditionVariable(std::addressof(thread_tree), addr, key, value); cur_thread->SetConditionVariable(std::addressof(m_tree), addr, key, value);
thread_tree.insert(*cur_thread); m_tree.insert(*cur_thread);
// Begin waiting. // Begin waiting.
wait_queue.SetHardwareTimer(timer); wait_queue.SetHardwareTimer(timer);
@ -328,7 +329,7 @@ Result KConditionVariable::Wait(VAddr addr, u64 key, u32 value, s64 timeout) {
} }
// Get the wait result. // Get the wait result.
return cur_thread->GetWaitResult(); R_RETURN(cur_thread->GetWaitResult());
} }
} // namespace Kernel } // namespace Kernel

View file

@ -21,24 +21,24 @@ class KConditionVariable {
public: public:
using ThreadTree = typename KThread::ConditionVariableThreadTreeType; using ThreadTree = typename KThread::ConditionVariableThreadTreeType;
explicit KConditionVariable(Core::System& system_); explicit KConditionVariable(Core::System& system);
~KConditionVariable(); ~KConditionVariable();
// Arbitration // Arbitration
[[nodiscard]] Result SignalToAddress(VAddr addr); Result SignalToAddress(VAddr addr);
[[nodiscard]] Result WaitForAddress(Handle handle, VAddr addr, u32 value); Result WaitForAddress(Handle handle, VAddr addr, u32 value);
// Condition variable // Condition variable
void Signal(u64 cv_key, s32 count); void Signal(u64 cv_key, s32 count);
[[nodiscard]] Result Wait(VAddr addr, u64 key, u32 value, s64 timeout); Result Wait(VAddr addr, u64 key, u32 value, s64 timeout);
private: private:
void SignalImpl(KThread* thread); void SignalImpl(KThread* thread);
ThreadTree thread_tree; private:
Core::System& m_system;
Core::System& system; KernelCore& m_kernel;
KernelCore& kernel; ThreadTree m_tree{};
}; };
inline void BeforeUpdatePriority(const KernelCore& kernel, KConditionVariable::ThreadTree* tree, inline void BeforeUpdatePriority(const KernelCore& kernel, KConditionVariable::ThreadTree* tree,

View file

@ -13,9 +13,9 @@ namespace {
class ThreadQueueImplForKLightConditionVariable final : public KThreadQueue { class ThreadQueueImplForKLightConditionVariable final : public KThreadQueue {
public: public:
ThreadQueueImplForKLightConditionVariable(KernelCore& kernel_, KThread::WaiterList* wl, ThreadQueueImplForKLightConditionVariable(KernelCore& kernel, KThread::WaiterList* wl,
bool term) bool term)
: KThreadQueue(kernel_), m_wait_list(wl), m_allow_terminating_thread(term) {} : KThreadQueue(kernel), m_wait_list(wl), m_allow_terminating_thread(term) {}
void CancelWait(KThread* waiting_thread, Result wait_result, bool cancel_timer_task) override { void CancelWait(KThread* waiting_thread, Result wait_result, bool cancel_timer_task) override {
// Only process waits if we're allowed to. // Only process waits if we're allowed to.
@ -39,15 +39,15 @@ private:
void KLightConditionVariable::Wait(KLightLock* lock, s64 timeout, bool allow_terminating_thread) { void KLightConditionVariable::Wait(KLightLock* lock, s64 timeout, bool allow_terminating_thread) {
// Create thread queue. // Create thread queue.
KThread* owner = GetCurrentThreadPointer(kernel); KThread* owner = GetCurrentThreadPointer(m_kernel);
KHardwareTimer* timer{}; KHardwareTimer* timer{};
ThreadQueueImplForKLightConditionVariable wait_queue(kernel, std::addressof(wait_list), ThreadQueueImplForKLightConditionVariable wait_queue(m_kernel, std::addressof(m_wait_list),
allow_terminating_thread); allow_terminating_thread);
// Sleep the thread. // Sleep the thread.
{ {
KScopedSchedulerLockAndSleep lk(kernel, std::addressof(timer), owner, timeout); KScopedSchedulerLockAndSleep lk(m_kernel, std::addressof(timer), owner, timeout);
if (!allow_terminating_thread && owner->IsTerminationRequested()) { if (!allow_terminating_thread && owner->IsTerminationRequested()) {
lk.CancelSleep(); lk.CancelSleep();
@ -57,7 +57,7 @@ void KLightConditionVariable::Wait(KLightLock* lock, s64 timeout, bool allow_ter
lock->Unlock(); lock->Unlock();
// Add the thread to the queue. // Add the thread to the queue.
wait_list.push_back(*owner); m_wait_list.push_back(*owner);
// Begin waiting. // Begin waiting.
wait_queue.SetHardwareTimer(timer); wait_queue.SetHardwareTimer(timer);
@ -69,10 +69,10 @@ void KLightConditionVariable::Wait(KLightLock* lock, s64 timeout, bool allow_ter
} }
void KLightConditionVariable::Broadcast() { void KLightConditionVariable::Broadcast() {
KScopedSchedulerLock lk(kernel); KScopedSchedulerLock lk(m_kernel);
// Signal all threads. // Signal all threads.
for (auto it = wait_list.begin(); it != wait_list.end(); it = wait_list.erase(it)) { for (auto it = m_wait_list.begin(); it != m_wait_list.end(); it = m_wait_list.erase(it)) {
it->EndWait(ResultSuccess); it->EndWait(ResultSuccess);
} }
} }

View file

@ -13,13 +13,13 @@ class KLightLock;
class KLightConditionVariable { class KLightConditionVariable {
public: public:
explicit KLightConditionVariable(KernelCore& kernel_) : kernel{kernel_} {} explicit KLightConditionVariable(KernelCore& kernel) : m_kernel{kernel} {}
void Wait(KLightLock* lock, s64 timeout = -1, bool allow_terminating_thread = true); void Wait(KLightLock* lock, s64 timeout = -1, bool allow_terminating_thread = true);
void Broadcast(); void Broadcast();
private: private:
KernelCore& kernel; KernelCore& m_kernel;
KThread::WaiterList wait_list{}; KThread::WaiterList m_wait_list{};
}; };
} // namespace Kernel } // namespace Kernel

View file

@ -13,7 +13,7 @@ namespace {
class ThreadQueueImplForKLightLock final : public KThreadQueue { class ThreadQueueImplForKLightLock final : public KThreadQueue {
public: public:
explicit ThreadQueueImplForKLightLock(KernelCore& kernel_) : KThreadQueue(kernel_) {} explicit ThreadQueueImplForKLightLock(KernelCore& kernel) : KThreadQueue(kernel) {}
void CancelWait(KThread* waiting_thread, Result wait_result, bool cancel_timer_task) override { void CancelWait(KThread* waiting_thread, Result wait_result, bool cancel_timer_task) override {
// Remove the thread as a waiter from its owner. // Remove the thread as a waiter from its owner.
@ -29,12 +29,12 @@ public:
} // namespace } // namespace
void KLightLock::Lock() { void KLightLock::Lock() {
const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer(kernel)); const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer(m_kernel));
while (true) { while (true) {
uintptr_t old_tag = tag.load(std::memory_order_relaxed); uintptr_t old_tag = m_tag.load(std::memory_order_relaxed);
while (!tag.compare_exchange_weak(old_tag, (old_tag == 0) ? cur_thread : (old_tag | 1), while (!m_tag.compare_exchange_weak(old_tag, (old_tag == 0) ? cur_thread : (old_tag | 1),
std::memory_order_acquire)) { std::memory_order_acquire)) {
} }
@ -45,30 +45,30 @@ void KLightLock::Lock() {
} }
void KLightLock::Unlock() { void KLightLock::Unlock() {
const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer(kernel)); const uintptr_t cur_thread = reinterpret_cast<uintptr_t>(GetCurrentThreadPointer(m_kernel));
uintptr_t expected = cur_thread; uintptr_t expected = cur_thread;
if (!tag.compare_exchange_strong(expected, 0, std::memory_order_release)) { if (!m_tag.compare_exchange_strong(expected, 0, std::memory_order_release)) {
this->UnlockSlowPath(cur_thread); this->UnlockSlowPath(cur_thread);
} }
} }
bool KLightLock::LockSlowPath(uintptr_t _owner, uintptr_t _cur_thread) { bool KLightLock::LockSlowPath(uintptr_t _owner, uintptr_t _cur_thread) {
KThread* cur_thread = reinterpret_cast<KThread*>(_cur_thread); KThread* cur_thread = reinterpret_cast<KThread*>(_cur_thread);
ThreadQueueImplForKLightLock wait_queue(kernel); ThreadQueueImplForKLightLock wait_queue(m_kernel);
// Pend the current thread waiting on the owner thread. // Pend the current thread waiting on the owner thread.
{ {
KScopedSchedulerLock sl{kernel}; KScopedSchedulerLock sl{m_kernel};
// Ensure we actually have locking to do. // Ensure we actually have locking to do.
if (tag.load(std::memory_order_relaxed) != _owner) { if (m_tag.load(std::memory_order_relaxed) != _owner) {
return false; return false;
} }
// Add the current thread as a waiter on the owner. // Add the current thread as a waiter on the owner.
KThread* owner_thread = reinterpret_cast<KThread*>(_owner & ~1ULL); KThread* owner_thread = reinterpret_cast<KThread*>(_owner & ~1ULL);
cur_thread->SetKernelAddressKey(reinterpret_cast<uintptr_t>(std::addressof(tag))); cur_thread->SetKernelAddressKey(reinterpret_cast<uintptr_t>(std::addressof(m_tag)));
owner_thread->AddWaiter(cur_thread); owner_thread->AddWaiter(cur_thread);
// Begin waiting to hold the lock. // Begin waiting to hold the lock.
@ -87,12 +87,12 @@ void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) {
// Unlock. // Unlock.
{ {
KScopedSchedulerLock sl(kernel); KScopedSchedulerLock sl(m_kernel);
// Get the next owner. // Get the next owner.
bool has_waiters; bool has_waiters;
KThread* next_owner = owner_thread->RemoveKernelWaiterByKey( KThread* next_owner = owner_thread->RemoveKernelWaiterByKey(
std::addressof(has_waiters), reinterpret_cast<uintptr_t>(std::addressof(tag))); std::addressof(has_waiters), reinterpret_cast<uintptr_t>(std::addressof(m_tag)));
// Pass the lock to the next owner. // Pass the lock to the next owner.
uintptr_t next_tag = 0; uintptr_t next_tag = 0;
@ -114,12 +114,13 @@ void KLightLock::UnlockSlowPath(uintptr_t _cur_thread) {
} }
// Write the new tag value. // Write the new tag value.
tag.store(next_tag, std::memory_order_release); m_tag.store(next_tag, std::memory_order_release);
} }
} }
bool KLightLock::IsLockedByCurrentThread() const { bool KLightLock::IsLockedByCurrentThread() const {
return (tag | 1ULL) == (reinterpret_cast<uintptr_t>(GetCurrentThreadPointer(kernel)) | 1ULL); return (m_tag.load() | 1ULL) ==
(reinterpret_cast<uintptr_t>(GetCurrentThreadPointer(m_kernel)) | 1ULL);
} }
} // namespace Kernel } // namespace Kernel

View file

@ -13,7 +13,7 @@ class KernelCore;
class KLightLock { class KLightLock {
public: public:
explicit KLightLock(KernelCore& kernel_) : kernel{kernel_} {} explicit KLightLock(KernelCore& kernel) : m_kernel{kernel} {}
void Lock(); void Lock();
@ -24,14 +24,14 @@ public:
void UnlockSlowPath(uintptr_t cur_thread); void UnlockSlowPath(uintptr_t cur_thread);
bool IsLocked() const { bool IsLocked() const {
return tag != 0; return m_tag.load() != 0;
} }
bool IsLockedByCurrentThread() const; bool IsLockedByCurrentThread() const;
private: private:
std::atomic<uintptr_t> tag{}; std::atomic<uintptr_t> m_tag{};
KernelCore& kernel; KernelCore& m_kernel;
}; };
using KScopedLightLock = KScopedLock<KLightLock>; using KScopedLightLock = KScopedLock<KLightLock>;