diff --git a/src/core/hle/kernel/code_set.h b/src/core/hle/kernel/code_set.h
index 834fd23d2..879957dcb 100644
--- a/src/core/hle/kernel/code_set.h
+++ b/src/core/hle/kernel/code_set.h
@@ -5,7 +5,6 @@
 #pragma once
 
 #include <cstddef>
-#include <memory>
 #include <vector>
 
 #include "common/common_types.h"
@@ -78,7 +77,7 @@ struct CodeSet final {
     }
 
     /// The overall data that backs this code set.
-    std::shared_ptr<std::vector<u8>> memory;
+    std::vector<u8> memory;
 
     /// The segments that comprise this code set.
     std::array<Segment, 3> segments;
diff --git a/src/core/hle/kernel/process.cpp b/src/core/hle/kernel/process.cpp
index 0d782e4ba..87779a71c 100644
--- a/src/core/hle/kernel/process.cpp
+++ b/src/core/hle/kernel/process.cpp
@@ -210,11 +210,13 @@ void Process::FreeTLSSlot(VAddr tls_address) {
 }
 
 void Process::LoadModule(CodeSet module_, VAddr base_addr) {
+    const auto memory = std::make_shared<std::vector<u8>>(std::move(module_.memory));
+
     const auto MapSegment = [&](const CodeSet::Segment& segment, VMAPermission permissions,
                                 MemoryState memory_state) {
         const auto vma = vm_manager
-                             .MapMemoryBlock(segment.addr + base_addr, module_.memory,
-                                             segment.offset, segment.size, memory_state)
+                             .MapMemoryBlock(segment.addr + base_addr, memory, segment.offset,
+                                             segment.size, memory_state)
                              .Unwrap();
         vm_manager.Reprotect(vma, permissions);
     };
diff --git a/src/core/loader/elf.cpp b/src/core/loader/elf.cpp
index 8b1920f22..46ac372f6 100644
--- a/src/core/loader/elf.cpp
+++ b/src/core/loader/elf.cpp
@@ -341,7 +341,7 @@ Kernel::CodeSet ElfReader::LoadInto(VAddr vaddr) {
     }
 
     codeset.entrypoint = base_addr + header->e_entry;
-    codeset.memory = std::make_shared<std::vector<u8>>(std::move(program_image));
+    codeset.memory = std::move(program_image);
 
     LOG_DEBUG(Loader, "Done loading.");
 
diff --git a/src/core/loader/nro.cpp b/src/core/loader/nro.cpp
index 5de02a94b..31e4a0c84 100644
--- a/src/core/loader/nro.cpp
+++ b/src/core/loader/nro.cpp
@@ -187,7 +187,7 @@ static bool LoadNroImpl(Kernel::Process& process, const std::vector<u8>& data,
     program_image.resize(static_cast<u32>(program_image.size()) + bss_size);
 
     // Load codeset for current process
-    codeset.memory = std::make_shared<std::vector<u8>>(std::move(program_image));
+    codeset.memory = std::move(program_image);
     process.LoadModule(std::move(codeset), load_base);
 
     // Register module with GDBStub
diff --git a/src/core/loader/nso.cpp b/src/core/loader/nso.cpp
index 0eb9fd7f7..5caceb267 100644
--- a/src/core/loader/nso.cpp
+++ b/src/core/loader/nso.cpp
@@ -178,7 +178,7 @@ std::optional<VAddr> AppLoader_NSO::LoadModule(Kernel::Process& process,
     }
 
     // Load codeset for current process
-    codeset.memory = std::make_shared<std::vector<u8>>(std::move(program_image));
+    codeset.memory = std::move(program_image);
     process.LoadModule(std::move(codeset), load_base);
 
     // Register module with GDBStub