diff --git a/src/core/arm/dynarmic/arm_dynarmic.cpp b/src/core/arm/dynarmic/arm_dynarmic.cpp
index 0762321a9..4d2491870 100644
--- a/src/core/arm/dynarmic/arm_dynarmic.cpp
+++ b/src/core/arm/dynarmic/arm_dynarmic.cpp
@@ -144,7 +144,7 @@ std::unique_ptr<Dynarmic::A64::Jit> ARM_Dynarmic::MakeJit() const {
 
     // Multi-process state
     config.processor_id = core_index;
-    config.global_monitor = &exclusive_monitor->monitor;
+    config.global_monitor = &exclusive_monitor.monitor;
 
     // System registers
     config.tpidrro_el0 = &cb->tpidrro_el0;
@@ -171,10 +171,9 @@ void ARM_Dynarmic::Step() {
     cb->InterpreterFallback(jit->GetPC(), 1);
 }
 
-ARM_Dynarmic::ARM_Dynarmic(std::shared_ptr<ExclusiveMonitor> exclusive_monitor,
-                           std::size_t core_index)
+ARM_Dynarmic::ARM_Dynarmic(ExclusiveMonitor& exclusive_monitor, std::size_t core_index)
     : cb(std::make_unique<ARM_Dynarmic_Callbacks>(*this)), core_index{core_index},
-      exclusive_monitor{std::dynamic_pointer_cast<DynarmicExclusiveMonitor>(exclusive_monitor)} {
+      exclusive_monitor{dynamic_cast<DynarmicExclusiveMonitor&>(exclusive_monitor)} {
     ThreadContext ctx{};
     inner_unicorn.SaveContext(ctx);
     PageTableChanged();
diff --git a/src/core/arm/dynarmic/arm_dynarmic.h b/src/core/arm/dynarmic/arm_dynarmic.h
index 4ee92ee27..512bf8ce9 100644
--- a/src/core/arm/dynarmic/arm_dynarmic.h
+++ b/src/core/arm/dynarmic/arm_dynarmic.h
@@ -23,7 +23,7 @@ class DynarmicExclusiveMonitor;
 
 class ARM_Dynarmic final : public ARM_Interface {
 public:
-    ARM_Dynarmic(std::shared_ptr<ExclusiveMonitor> exclusive_monitor, std::size_t core_index);
+    ARM_Dynarmic(ExclusiveMonitor& exclusive_monitor, std::size_t core_index);
     ~ARM_Dynarmic();
 
     void MapBackingMemory(VAddr address, std::size_t size, u8* memory,
@@ -62,7 +62,7 @@ private:
     ARM_Unicorn inner_unicorn;
 
     std::size_t core_index;
-    std::shared_ptr<DynarmicExclusiveMonitor> exclusive_monitor;
+    DynarmicExclusiveMonitor& exclusive_monitor;
 
     Memory::PageTable* current_page_table = nullptr;
 };
diff --git a/src/core/core.cpp b/src/core/core.cpp
index 32baa40dc..3c57a62ec 100644
--- a/src/core/core.cpp
+++ b/src/core/core.cpp
@@ -71,9 +71,9 @@ FileSys::VirtualFile GetGameFileFromPath(const FileSys::VirtualFilesystem& vfs,
 }
 
 /// Runs a CPU core while the system is powered on
-void RunCpuCore(std::shared_ptr<Cpu> cpu_state) {
+void RunCpuCore(Cpu& cpu_state) {
     while (Core::System::GetInstance().IsPoweredOn()) {
-        cpu_state->RunLoop(true);
+        cpu_state.RunLoop(true);
     }
 }
 } // Anonymous namespace
@@ -95,7 +95,7 @@ struct System::Impl {
         status = ResultStatus::Success;
 
         // Update thread_to_cpu in case Core 0 is run from a different host thread
-        thread_to_cpu[std::this_thread::get_id()] = cpu_cores[0];
+        thread_to_cpu[std::this_thread::get_id()] = cpu_cores[0].get();
 
         if (GDBStub::IsServerEnabled()) {
             GDBStub::HandlePacket();
@@ -139,10 +139,10 @@ struct System::Impl {
         auto main_process = Kernel::Process::Create(kernel, "main");
         kernel.MakeCurrentProcess(main_process.get());
 
-        cpu_barrier = std::make_shared<CpuBarrier>();
+        cpu_barrier = std::make_unique<CpuBarrier>();
         cpu_exclusive_monitor = Cpu::MakeExclusiveMonitor(cpu_cores.size());
         for (std::size_t index = 0; index < cpu_cores.size(); ++index) {
-            cpu_cores[index] = std::make_shared<Cpu>(cpu_exclusive_monitor, cpu_barrier, index);
+            cpu_cores[index] = std::make_unique<Cpu>(*cpu_exclusive_monitor, *cpu_barrier, index);
         }
 
         telemetry_session = std::make_unique<Core::TelemetrySession>();
@@ -160,12 +160,12 @@ struct System::Impl {
 
         // Create threads for CPU cores 1-3, and build thread_to_cpu map
         // CPU core 0 is run on the main thread
-        thread_to_cpu[std::this_thread::get_id()] = cpu_cores[0];
+        thread_to_cpu[std::this_thread::get_id()] = cpu_cores[0].get();
         if (Settings::values.use_multi_core) {
             for (std::size_t index = 0; index < cpu_core_threads.size(); ++index) {
                 cpu_core_threads[index] =
-                    std::make_unique<std::thread>(RunCpuCore, cpu_cores[index + 1]);
-                thread_to_cpu[cpu_core_threads[index]->get_id()] = cpu_cores[index + 1];
+                    std::make_unique<std::thread>(RunCpuCore, std::ref(*cpu_cores[index + 1]));
+                thread_to_cpu[cpu_core_threads[index]->get_id()] = cpu_cores[index + 1].get();
             }
         }
 
@@ -245,6 +245,7 @@ struct System::Impl {
         for (auto& cpu_core : cpu_cores) {
             cpu_core.reset();
         }
+        cpu_exclusive_monitor.reset();
         cpu_barrier.reset();
 
         // Shutdown kernel and core timing
@@ -282,9 +283,9 @@ struct System::Impl {
     std::unique_ptr<VideoCore::RendererBase> renderer;
     std::unique_ptr<Tegra::GPU> gpu_core;
     std::shared_ptr<Tegra::DebugContext> debug_context;
-    std::shared_ptr<ExclusiveMonitor> cpu_exclusive_monitor;
-    std::shared_ptr<CpuBarrier> cpu_barrier;
-    std::array<std::shared_ptr<Cpu>, NUM_CPU_CORES> cpu_cores;
+    std::unique_ptr<ExclusiveMonitor> cpu_exclusive_monitor;
+    std::unique_ptr<CpuBarrier> cpu_barrier;
+    std::array<std::unique_ptr<Cpu>, NUM_CPU_CORES> cpu_cores;
     std::array<std::unique_ptr<std::thread>, NUM_CPU_CORES - 1> cpu_core_threads;
     std::size_t active_core{}; ///< Active core, only used in single thread mode
 
@@ -298,7 +299,7 @@ struct System::Impl {
     std::string status_details = "";
 
     /// Map of guest threads to CPU cores
-    std::map<std::thread::id, std::shared_ptr<Cpu>> thread_to_cpu;
+    std::map<std::thread::id, Cpu*> thread_to_cpu;
 
     Core::PerfStats perf_stats;
     Core::FrameLimiter frame_limiter;
@@ -354,12 +355,15 @@ std::size_t System::CurrentCoreIndex() {
 }
 
 Kernel::Scheduler& System::CurrentScheduler() {
-    return *CurrentCpuCore().Scheduler();
+    return CurrentCpuCore().Scheduler();
 }
 
-const std::shared_ptr<Kernel::Scheduler>& System::Scheduler(std::size_t core_index) {
-    ASSERT(core_index < NUM_CPU_CORES);
-    return impl->cpu_cores[core_index]->Scheduler();
+Kernel::Scheduler& System::Scheduler(std::size_t core_index) {
+    return CpuCore(core_index).Scheduler();
+}
+
+const Kernel::Scheduler& System::Scheduler(std::size_t core_index) const {
+    return CpuCore(core_index).Scheduler();
 }
 
 Kernel::Process* System::CurrentProcess() {
@@ -380,6 +384,11 @@ Cpu& System::CpuCore(std::size_t core_index) {
     return *impl->cpu_cores[core_index];
 }
 
+const Cpu& System::CpuCore(std::size_t core_index) const {
+    ASSERT(core_index < NUM_CPU_CORES);
+    return *impl->cpu_cores[core_index];
+}
+
 ExclusiveMonitor& System::Monitor() {
     return *impl->cpu_exclusive_monitor;
 }
diff --git a/src/core/core.h b/src/core/core.h
index ea4d53914..173be45f8 100644
--- a/src/core/core.h
+++ b/src/core/core.h
@@ -156,6 +156,9 @@ public:
     /// Gets a CPU interface to the CPU core with the specified index
     Cpu& CpuCore(std::size_t core_index);
 
+    /// Gets a CPU interface to the CPU core with the specified index
+    const Cpu& CpuCore(std::size_t core_index) const;
+
     /// Gets the exclusive monitor
     ExclusiveMonitor& Monitor();
 
@@ -172,7 +175,10 @@ public:
     const VideoCore::RendererBase& Renderer() const;
 
     /// Gets the scheduler for the CPU core with the specified index
-    const std::shared_ptr<Kernel::Scheduler>& Scheduler(std::size_t core_index);
+    Kernel::Scheduler& Scheduler(std::size_t core_index);
+
+    /// Gets the scheduler for the CPU core with the specified index
+    const Kernel::Scheduler& Scheduler(std::size_t core_index) const;
 
     /// Provides a pointer to the current process
     Kernel::Process* CurrentProcess();
diff --git a/src/core/core_cpu.cpp b/src/core/core_cpu.cpp
index 265f8ed9c..fffda8a99 100644
--- a/src/core/core_cpu.cpp
+++ b/src/core/core_cpu.cpp
@@ -49,10 +49,8 @@ bool CpuBarrier::Rendezvous() {
     return false;
 }
 
-Cpu::Cpu(std::shared_ptr<ExclusiveMonitor> exclusive_monitor,
-         std::shared_ptr<CpuBarrier> cpu_barrier, std::size_t core_index)
-    : cpu_barrier{std::move(cpu_barrier)}, core_index{core_index} {
-
+Cpu::Cpu(ExclusiveMonitor& exclusive_monitor, CpuBarrier& cpu_barrier, std::size_t core_index)
+    : cpu_barrier{cpu_barrier}, core_index{core_index} {
     if (Settings::values.use_cpu_jit) {
 #ifdef ARCHITECTURE_x86_64
         arm_interface = std::make_unique<ARM_Dynarmic>(exclusive_monitor, core_index);
@@ -64,15 +62,15 @@ Cpu::Cpu(std::shared_ptr<ExclusiveMonitor> exclusive_monitor,
         arm_interface = std::make_unique<ARM_Unicorn>();
     }
 
-    scheduler = std::make_shared<Kernel::Scheduler>(*arm_interface);
+    scheduler = std::make_unique<Kernel::Scheduler>(*arm_interface);
 }
 
 Cpu::~Cpu() = default;
 
-std::shared_ptr<ExclusiveMonitor> Cpu::MakeExclusiveMonitor(std::size_t num_cores) {
+std::unique_ptr<ExclusiveMonitor> Cpu::MakeExclusiveMonitor(std::size_t num_cores) {
     if (Settings::values.use_cpu_jit) {
 #ifdef ARCHITECTURE_x86_64
-        return std::make_shared<DynarmicExclusiveMonitor>(num_cores);
+        return std::make_unique<DynarmicExclusiveMonitor>(num_cores);
 #else
         return nullptr; // TODO(merry): Passthrough exclusive monitor
 #endif
@@ -83,7 +81,7 @@ std::shared_ptr<ExclusiveMonitor> Cpu::MakeExclusiveMonitor(std::size_t num_core
 
 void Cpu::RunLoop(bool tight_loop) {
     // Wait for all other CPU cores to complete the previous slice, such that they run in lock-step
-    if (!cpu_barrier->Rendezvous()) {
+    if (!cpu_barrier.Rendezvous()) {
         // If rendezvous failed, session has been killed
         return;
     }
diff --git a/src/core/core_cpu.h b/src/core/core_cpu.h
index ee7e04abc..1d2bdc6cd 100644
--- a/src/core/core_cpu.h
+++ b/src/core/core_cpu.h
@@ -41,8 +41,7 @@ private:
 
 class Cpu {
 public:
-    Cpu(std::shared_ptr<ExclusiveMonitor> exclusive_monitor,
-        std::shared_ptr<CpuBarrier> cpu_barrier, std::size_t core_index);
+    Cpu(ExclusiveMonitor& exclusive_monitor, CpuBarrier& cpu_barrier, std::size_t core_index);
     ~Cpu();
 
     void RunLoop(bool tight_loop = true);
@@ -59,8 +58,12 @@ public:
         return *arm_interface;
     }
 
-    const std::shared_ptr<Kernel::Scheduler>& Scheduler() const {
-        return scheduler;
+    Kernel::Scheduler& Scheduler() {
+        return *scheduler;
+    }
+
+    const Kernel::Scheduler& Scheduler() const {
+        return *scheduler;
     }
 
     bool IsMainCore() const {
@@ -71,14 +74,14 @@ public:
         return core_index;
     }
 
-    static std::shared_ptr<ExclusiveMonitor> MakeExclusiveMonitor(std::size_t num_cores);
+    static std::unique_ptr<ExclusiveMonitor> MakeExclusiveMonitor(std::size_t num_cores);
 
 private:
     void Reschedule();
 
     std::unique_ptr<ARM_Interface> arm_interface;
-    std::shared_ptr<CpuBarrier> cpu_barrier;
-    std::shared_ptr<Kernel::Scheduler> scheduler;
+    CpuBarrier& cpu_barrier;
+    std::unique_ptr<Kernel::Scheduler> scheduler;
 
     std::atomic<bool> reschedule_pending = false;
     std::size_t core_index;
diff --git a/src/core/gdbstub/gdbstub.cpp b/src/core/gdbstub/gdbstub.cpp
index e961ef121..bdcc889e0 100644
--- a/src/core/gdbstub/gdbstub.cpp
+++ b/src/core/gdbstub/gdbstub.cpp
@@ -207,7 +207,7 @@ void RegisterModule(std::string name, VAddr beg, VAddr end, bool add_elf_ext) {
 
 static Kernel::Thread* FindThreadById(int id) {
     for (u32 core = 0; core < Core::NUM_CPU_CORES; core++) {
-        const auto& threads = Core::System::GetInstance().Scheduler(core)->GetThreadList();
+        const auto& threads = Core::System::GetInstance().Scheduler(core).GetThreadList();
         for (auto& thread : threads) {
             if (thread->GetThreadID() == static_cast<u32>(id)) {
                 current_core = core;
@@ -597,7 +597,7 @@ static void HandleQuery() {
     } else if (strncmp(query, "fThreadInfo", strlen("fThreadInfo")) == 0) {
         std::string val = "m";
         for (u32 core = 0; core < Core::NUM_CPU_CORES; core++) {
-            const auto& threads = Core::System::GetInstance().Scheduler(core)->GetThreadList();
+            const auto& threads = Core::System::GetInstance().Scheduler(core).GetThreadList();
             for (const auto& thread : threads) {
                 val += fmt::format("{:x}", thread->GetThreadID());
                 val += ",";
@@ -612,7 +612,7 @@ static void HandleQuery() {
         buffer += "l<?xml version=\"1.0\"?>";
         buffer += "<threads>";
         for (u32 core = 0; core < Core::NUM_CPU_CORES; core++) {
-            const auto& threads = Core::System::GetInstance().Scheduler(core)->GetThreadList();
+            const auto& threads = Core::System::GetInstance().Scheduler(core).GetThreadList();
             for (const auto& thread : threads) {
                 buffer +=
                     fmt::format(R"*(<thread id="{:x}" core="{:d}" name="Thread {:x}"></thread>)*",
diff --git a/src/core/hle/kernel/address_arbiter.cpp b/src/core/hle/kernel/address_arbiter.cpp
index ebf193930..57157beb4 100644
--- a/src/core/hle/kernel/address_arbiter.cpp
+++ b/src/core/hle/kernel/address_arbiter.cpp
@@ -39,7 +39,7 @@ static std::vector<SharedPtr<Thread>> GetThreadsWaitingOnAddress(VAddr address)
                                            std::vector<SharedPtr<Thread>>& waiting_threads,
                                            VAddr arb_addr) {
         const auto& scheduler = Core::System::GetInstance().Scheduler(core_index);
-        const auto& thread_list = scheduler->GetThreadList();
+        const auto& thread_list = scheduler.GetThreadList();
 
         for (const auto& thread : thread_list) {
             if (thread->GetArbiterWaitAddress() == arb_addr)
diff --git a/src/core/hle/kernel/process.cpp b/src/core/hle/kernel/process.cpp
index c80b2c507..073dd5a7d 100644
--- a/src/core/hle/kernel/process.cpp
+++ b/src/core/hle/kernel/process.cpp
@@ -153,11 +153,11 @@ void Process::PrepareForTermination() {
         }
     };
 
-    auto& system = Core::System::GetInstance();
-    stop_threads(system.Scheduler(0)->GetThreadList());
-    stop_threads(system.Scheduler(1)->GetThreadList());
-    stop_threads(system.Scheduler(2)->GetThreadList());
-    stop_threads(system.Scheduler(3)->GetThreadList());
+    const auto& system = Core::System::GetInstance();
+    stop_threads(system.Scheduler(0).GetThreadList());
+    stop_threads(system.Scheduler(1).GetThreadList());
+    stop_threads(system.Scheduler(2).GetThreadList());
+    stop_threads(system.Scheduler(3).GetThreadList());
 }
 
 /**
diff --git a/src/core/hle/kernel/svc.cpp b/src/core/hle/kernel/svc.cpp
index 7a053da1e..3e5f11f2b 100644
--- a/src/core/hle/kernel/svc.cpp
+++ b/src/core/hle/kernel/svc.cpp
@@ -809,7 +809,7 @@ static ResultCode SignalProcessWideKey(VAddr condition_variable_addr, s32 target
                                            std::vector<SharedPtr<Thread>>& waiting_threads,
                                            VAddr condvar_addr) {
         const auto& scheduler = Core::System::GetInstance().Scheduler(core_index);
-        const auto& thread_list = scheduler->GetThreadList();
+        const auto& thread_list = scheduler.GetThreadList();
 
         for (const auto& thread : thread_list) {
             if (thread->GetCondVarWaitAddress() == condvar_addr)
diff --git a/src/core/hle/kernel/thread.cpp b/src/core/hle/kernel/thread.cpp
index 352ce1725..35ec98c1a 100644
--- a/src/core/hle/kernel/thread.cpp
+++ b/src/core/hle/kernel/thread.cpp
@@ -97,7 +97,7 @@ void Thread::CancelWakeupTimer() {
 static boost::optional<s32> GetNextProcessorId(u64 mask) {
     for (s32 index = 0; index < Core::NUM_CPU_CORES; ++index) {
         if (mask & (1ULL << index)) {
-            if (!Core::System::GetInstance().Scheduler(index)->GetCurrentThread()) {
+            if (!Core::System::GetInstance().Scheduler(index).GetCurrentThread()) {
                 // Core is enabled and not running any threads, use this one
                 return index;
             }
@@ -147,14 +147,14 @@ void Thread::ResumeFromWait() {
         new_processor_id = processor_id;
     }
     if (ideal_core != -1 &&
-        Core::System::GetInstance().Scheduler(ideal_core)->GetCurrentThread() == nullptr) {
+        Core::System::GetInstance().Scheduler(ideal_core).GetCurrentThread() == nullptr) {
         new_processor_id = ideal_core;
     }
 
     ASSERT(*new_processor_id < 4);
 
     // Add thread to new core's scheduler
-    auto& next_scheduler = Core::System::GetInstance().Scheduler(*new_processor_id);
+    auto* next_scheduler = &Core::System::GetInstance().Scheduler(*new_processor_id);
 
     if (*new_processor_id != processor_id) {
         // Remove thread from previous core's scheduler
@@ -169,7 +169,7 @@ void Thread::ResumeFromWait() {
     next_scheduler->ScheduleThread(this, current_priority);
 
     // Change thread's scheduler
-    scheduler = next_scheduler.get();
+    scheduler = next_scheduler;
 
     Core::System::GetInstance().CpuCore(processor_id).PrepareReschedule();
 }
@@ -230,7 +230,7 @@ ResultVal<SharedPtr<Thread>> Thread::Create(KernelCore& kernel, std::string name
     thread->name = std::move(name);
     thread->callback_handle = kernel.ThreadWakeupCallbackHandleTable().Create(thread).Unwrap();
     thread->owner_process = &owner_process;
-    thread->scheduler = Core::System::GetInstance().Scheduler(processor_id).get();
+    thread->scheduler = &Core::System::GetInstance().Scheduler(processor_id);
     thread->scheduler->AddThread(thread, priority);
     thread->tls_address = thread->owner_process->MarkNextAvailableTLSSlotAsUsed(*thread);
 
@@ -375,14 +375,14 @@ void Thread::ChangeCore(u32 core, u64 mask) {
         new_processor_id = processor_id;
     }
     if (ideal_core != -1 &&
-        Core::System::GetInstance().Scheduler(ideal_core)->GetCurrentThread() == nullptr) {
+        Core::System::GetInstance().Scheduler(ideal_core).GetCurrentThread() == nullptr) {
         new_processor_id = ideal_core;
     }
 
     ASSERT(*new_processor_id < 4);
 
     // Add thread to new core's scheduler
-    auto& next_scheduler = Core::System::GetInstance().Scheduler(*new_processor_id);
+    auto* next_scheduler = &Core::System::GetInstance().Scheduler(*new_processor_id);
 
     if (*new_processor_id != processor_id) {
         // Remove thread from previous core's scheduler
@@ -397,7 +397,7 @@ void Thread::ChangeCore(u32 core, u64 mask) {
     next_scheduler->ScheduleThread(this, current_priority);
 
     // Change thread's scheduler
-    scheduler = next_scheduler.get();
+    scheduler = next_scheduler;
 
     Core::System::GetInstance().CpuCore(processor_id).PrepareReschedule();
 }
diff --git a/src/yuzu/debugger/wait_tree.cpp b/src/yuzu/debugger/wait_tree.cpp
index 4a09da685..7403e9ccd 100644
--- a/src/yuzu/debugger/wait_tree.cpp
+++ b/src/yuzu/debugger/wait_tree.cpp
@@ -66,10 +66,11 @@ std::vector<std::unique_ptr<WaitTreeThread>> WaitTreeItem::MakeThreadItemList()
         }
     };
 
-    add_threads(Core::System::GetInstance().Scheduler(0)->GetThreadList());
-    add_threads(Core::System::GetInstance().Scheduler(1)->GetThreadList());
-    add_threads(Core::System::GetInstance().Scheduler(2)->GetThreadList());
-    add_threads(Core::System::GetInstance().Scheduler(3)->GetThreadList());
+    const auto& system = Core::System::GetInstance();
+    add_threads(system.Scheduler(0).GetThreadList());
+    add_threads(system.Scheduler(1).GetThreadList());
+    add_threads(system.Scheduler(2).GetThreadList());
+    add_threads(system.Scheduler(3).GetThreadList());
 
     return item_list;
 }