diff --git a/src/core/core_cpu.cpp b/src/core/core_cpu.cpp
index 21568ad50..265f8ed9c 100644
--- a/src/core/core_cpu.cpp
+++ b/src/core/core_cpu.cpp
@@ -55,16 +55,16 @@ Cpu::Cpu(std::shared_ptr<ExclusiveMonitor> exclusive_monitor,
 
     if (Settings::values.use_cpu_jit) {
 #ifdef ARCHITECTURE_x86_64
-        arm_interface = std::make_shared<ARM_Dynarmic>(exclusive_monitor, core_index);
+        arm_interface = std::make_unique<ARM_Dynarmic>(exclusive_monitor, core_index);
 #else
-        arm_interface = std::make_shared<ARM_Unicorn>();
+        arm_interface = std::make_unique<ARM_Unicorn>();
         LOG_WARNING(Core, "CPU JIT requested, but Dynarmic not available");
 #endif
     } else {
-        arm_interface = std::make_shared<ARM_Unicorn>();
+        arm_interface = std::make_unique<ARM_Unicorn>();
     }
 
-    scheduler = std::make_shared<Kernel::Scheduler>(arm_interface.get());
+    scheduler = std::make_shared<Kernel::Scheduler>(*arm_interface);
 }
 
 Cpu::~Cpu() = default;
diff --git a/src/core/core_cpu.h b/src/core/core_cpu.h
index 685532965..ee7e04abc 100644
--- a/src/core/core_cpu.h
+++ b/src/core/core_cpu.h
@@ -76,7 +76,7 @@ public:
 private:
     void Reschedule();
 
-    std::shared_ptr<ARM_Interface> arm_interface;
+    std::unique_ptr<ARM_Interface> arm_interface;
     std::shared_ptr<CpuBarrier> cpu_barrier;
     std::shared_ptr<Kernel::Scheduler> scheduler;
 
diff --git a/src/core/hle/kernel/scheduler.cpp b/src/core/hle/kernel/scheduler.cpp
index 69c812f16..9faf903cf 100644
--- a/src/core/hle/kernel/scheduler.cpp
+++ b/src/core/hle/kernel/scheduler.cpp
@@ -17,7 +17,7 @@ namespace Kernel {
 
 std::mutex Scheduler::scheduler_mutex;
 
-Scheduler::Scheduler(Core::ARM_Interface* cpu_core) : cpu_core(cpu_core) {}
+Scheduler::Scheduler(Core::ARM_Interface& cpu_core) : cpu_core(cpu_core) {}
 
 Scheduler::~Scheduler() {
     for (auto& thread : thread_list) {
@@ -59,9 +59,9 @@ void Scheduler::SwitchContext(Thread* new_thread) {
     // Save context for previous thread
     if (previous_thread) {
         previous_thread->last_running_ticks = CoreTiming::GetTicks();
-        cpu_core->SaveContext(previous_thread->context);
+        cpu_core.SaveContext(previous_thread->context);
         // Save the TPIDR_EL0 system register in case it was modified.
-        previous_thread->tpidr_el0 = cpu_core->GetTPIDR_EL0();
+        previous_thread->tpidr_el0 = cpu_core.GetTPIDR_EL0();
 
         if (previous_thread->status == ThreadStatus::Running) {
             // This is only the case when a reschedule is triggered without the current thread
@@ -91,10 +91,10 @@ void Scheduler::SwitchContext(Thread* new_thread) {
             SetCurrentPageTable(&Core::CurrentProcess()->vm_manager.page_table);
         }
 
-        cpu_core->LoadContext(new_thread->context);
-        cpu_core->SetTlsAddress(new_thread->GetTLSAddress());
-        cpu_core->SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
-        cpu_core->ClearExclusiveState();
+        cpu_core.LoadContext(new_thread->context);
+        cpu_core.SetTlsAddress(new_thread->GetTLSAddress());
+        cpu_core.SetTPIDR_EL0(new_thread->GetTPIDR_EL0());
+        cpu_core.ClearExclusiveState();
     } else {
         current_thread = nullptr;
         // Note: We do not reset the current process and current page table when idling because
diff --git a/src/core/hle/kernel/scheduler.h b/src/core/hle/kernel/scheduler.h
index 744990c9b..2c94641ec 100644
--- a/src/core/hle/kernel/scheduler.h
+++ b/src/core/hle/kernel/scheduler.h
@@ -19,7 +19,7 @@ namespace Kernel {
 
 class Scheduler final {
 public:
-    explicit Scheduler(Core::ARM_Interface* cpu_core);
+    explicit Scheduler(Core::ARM_Interface& cpu_core);
     ~Scheduler();
 
     /// Returns whether there are any threads that are ready to run.
@@ -72,7 +72,7 @@ private:
 
     SharedPtr<Thread> current_thread = nullptr;
 
-    Core::ARM_Interface* cpu_core;
+    Core::ARM_Interface& cpu_core;
 
     static std::mutex scheduler_mutex;
 };