diff --git a/src/core/hle/kernel/scheduler.cpp b/src/core/hle/kernel/scheduler.cpp
index e5892727e..f2664ce65 100644
--- a/src/core/hle/kernel/scheduler.cpp
+++ b/src/core/hle/kernel/scheduler.cpp
@@ -530,10 +530,11 @@ SchedulerLockAndSleep::SchedulerLockAndSleep(KernelCore& kernel, Handle& event_h
 }
 
 SchedulerLockAndSleep::~SchedulerLockAndSleep() {
-    if (!sleep_cancelled) {
-        auto& time_manager = kernel.TimeManager();
-        time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds);
+    if (sleep_cancelled) {
+        return;
     }
+    auto& time_manager = kernel.TimeManager();
+    time_manager.ScheduleTimeEvent(event_handle, time_task, nanoseconds);
 }
 
 } // namespace Kernel