Mirrors/yuzu-fork
1
0
Fork 0
mirror of https://github.com/yuzu-mirror/yuzu.git synced 2024-09-07 03:59:10 +01:00
Code Issues Projects Releases Packages Wiki Activity

core: hle: kernel: k_memory_manager: Refresh.

Browse source
This commit is contained in:
bunnei 2022-10-29 14:53:28 -07:00
parent 32d7faafa8
commit ba21ba0c5c
4 changed files with 461 additions and 370 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

242
src/core/hle/kernel/k_memory_manager.cpp
View file

@ -29,43 +29,44 @@ constexpr KMemoryManager::Pool GetPoolFromMemoryRegionType(u32 type) {
} else if ((type | KMemoryRegionType_DramSystemNonSecurePool) == type) { } else if ((type | KMemoryRegionType_DramSystemNonSecurePool) == type) {
return KMemoryManager::Pool::SystemNonSecure; return KMemoryManager::Pool::SystemNonSecure;
} else { } else {
ASSERT_MSG(false, "InvalidMemoryRegionType for conversion to Pool"); UNREACHABLE_MSG("InvalidMemoryRegionType for conversion to Pool");
return {};
} }
} }
} // namespace } // namespace
KMemoryManager::KMemoryManager(Core::System& system_) KMemoryManager::KMemoryManager(Core::System& system)
: system{system_}, pool_locks{ : m_system{system}, m_memory_layout{system.Kernel().MemoryLayout()},
KLightLock{system_.Kernel()}, m_pool_locks{
KLightLock{system_.Kernel()}, KLightLock{system.Kernel()},
KLightLock{system_.Kernel()}, KLightLock{system.Kernel()},
KLightLock{system_.Kernel()}, KLightLock{system.Kernel()},
KLightLock{system.Kernel()},
} {} } {}
void KMemoryManager::Initialize(VAddr management_region, size_t management_region_size) { void KMemoryManager::Initialize(VAddr management_region, size_t management_region_size) {
// Clear the management region to zero. // Clear the management region to zero.
const VAddr management_region_end = management_region + management_region_size; const VAddr management_region_end = management_region + management_region_size;
// std::memset(GetVoidPointer(management_region), 0, management_region_size);
// Reset our manager count. // Reset our manager count.
num_managers = 0; m_num_managers = 0;
// Traverse the virtual memory layout tree, initializing each manager as appropriate. // Traverse the virtual memory layout tree, initializing each manager as appropriate.
while (num_managers != MaxManagerCount) { while (m_num_managers != MaxManagerCount) {
// Locate the region that should initialize the current manager. // Locate the region that should initialize the current manager.
PAddr region_address = 0; PAddr region_address = 0;
size_t region_size = 0; size_t region_size = 0;
Pool region_pool = Pool::Count; Pool region_pool = Pool::Count;
for (const auto& it : system.Kernel().MemoryLayout().GetPhysicalMemoryRegionTree()) { for (const auto& it : m_system.Kernel().MemoryLayout().GetPhysicalMemoryRegionTree()) {
// We only care about regions that we need to create managers for. // We only care about regions that we need to create managers for.
if (!it.IsDerivedFrom(KMemoryRegionType_DramUserPool)) { if (!it.IsDerivedFrom(KMemoryRegionType_DramUserPool)) {
continue; continue;
} }
// We want to initialize the managers in order. // We want to initialize the managers in order.
if (it.GetAttributes() != num_managers) { if (it.GetAttributes() != m_num_managers) {
continue; continue;
} }
@ -97,8 +98,8 @@ void KMemoryManager::Initialize(VAddr management_region, size_t management_regio
} }
// Initialize a new manager for the region. // Initialize a new manager for the region.
Impl* manager = std::addressof(managers[num_managers++]); Impl* manager = std::addressof(m_managers[m_num_managers++]);
ASSERT(num_managers <= managers.size()); ASSERT(m_num_managers <= m_managers.size());
const size_t cur_size = manager->Initialize(region_address, region_size, management_region, const size_t cur_size = manager->Initialize(region_address, region_size, management_region,
management_region_end, region_pool); management_region_end, region_pool);
@ -107,13 +108,13 @@ void KMemoryManager::Initialize(VAddr management_region, size_t management_regio
// Insert the manager into the pool list. // Insert the manager into the pool list.
const auto region_pool_index = static_cast<u32>(region_pool); const auto region_pool_index = static_cast<u32>(region_pool);
if (pool_managers_tail[region_pool_index] == nullptr) { if (m_pool_managers_tail[region_pool_index] == nullptr) {
pool_managers_head[region_pool_index] = manager; m_pool_managers_head[region_pool_index] = manager;
} else { } else {
pool_managers_tail[region_pool_index]->SetNext(manager); m_pool_managers_tail[region_pool_index]->SetNext(manager);
manager->SetPrev(pool_managers_tail[region_pool_index]); manager->SetPrev(m_pool_managers_tail[region_pool_index]);
} }
pool_managers_tail[region_pool_index] = manager; m_pool_managers_tail[region_pool_index] = manager;
} }
// Free each region to its corresponding heap. // Free each region to its corresponding heap.
@ -121,11 +122,10 @@ void KMemoryManager::Initialize(VAddr management_region, size_t management_regio
const PAddr ini_start = GetInitialProcessBinaryPhysicalAddress(); const PAddr ini_start = GetInitialProcessBinaryPhysicalAddress();
const PAddr ini_end = ini_start + InitialProcessBinarySizeMax; const PAddr ini_end = ini_start + InitialProcessBinarySizeMax;
const PAddr ini_last = ini_end - 1; const PAddr ini_last = ini_end - 1;
for (const auto& it : system.Kernel().MemoryLayout().GetPhysicalMemoryRegionTree()) { for (const auto& it : m_system.Kernel().MemoryLayout().GetPhysicalMemoryRegionTree()) {
if (it.IsDerivedFrom(KMemoryRegionType_DramUserPool)) { if (it.IsDerivedFrom(KMemoryRegionType_DramUserPool)) {
// Get the manager for the region. // Get the manager for the region.
auto index = it.GetAttributes(); auto& manager = m_managers[it.GetAttributes()];
auto& manager = managers[index];
const PAddr cur_start = it.GetAddress(); const PAddr cur_start = it.GetAddress();
const PAddr cur_last = it.GetLastAddress(); const PAddr cur_last = it.GetLastAddress();
@ -162,11 +162,19 @@ void KMemoryManager::Initialize(VAddr management_region, size_t management_regio
} }
// Update the used size for all managers. // Update the used size for all managers.
for (size_t i = 0; i < num_managers; ++i) { for (size_t i = 0; i < m_num_managers; ++i) {
managers[i].SetInitialUsedHeapSize(reserved_sizes[i]); m_managers[i].SetInitialUsedHeapSize(reserved_sizes[i]);
} }
} }
Result KMemoryManager::InitializeOptimizedMemory(u64 process_id, Pool pool) {
UNREACHABLE();
}
void KMemoryManager::FinalizeOptimizedMemory(u64 process_id, Pool pool) {
UNREACHABLE();
}
PAddr KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option) { PAddr KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option) {
// Early return if we're allocating no pages. // Early return if we're allocating no pages.
if (num_pages == 0) { if (num_pages == 0) {
@ -175,7 +183,7 @@ PAddr KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, size_t align_p
// Lock the pool that we're allocating from. // Lock the pool that we're allocating from.
const auto [pool, dir] = DecodeOption(option); const auto [pool, dir] = DecodeOption(option);
KScopedLightLock lk(pool_locks[static_cast<std::size_t>(pool)]); KScopedLightLock lk(m_pool_locks[static_cast<std::size_t>(pool)]);
// Choose a heap based on our page size request. // Choose a heap based on our page size request.
const s32 heap_index = KPageHeap::GetAlignedBlockIndex(num_pages, align_pages); const s32 heap_index = KPageHeap::GetAlignedBlockIndex(num_pages, align_pages);
@ -185,7 +193,7 @@ PAddr KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, size_t align_p
PAddr allocated_block = 0; PAddr allocated_block = 0;
for (chosen_manager = this->GetFirstManager(pool, dir); chosen_manager != nullptr; for (chosen_manager = this->GetFirstManager(pool, dir); chosen_manager != nullptr;
chosen_manager = this->GetNextManager(chosen_manager, dir)) { chosen_manager = this->GetNextManager(chosen_manager, dir)) {
allocated_block = chosen_manager->AllocateBlock(heap_index, true); allocated_block = chosen_manager->AllocateAligned(heap_index, num_pages, align_pages);
if (allocated_block != 0) { if (allocated_block != 0) {
break; break;
} }
@ -196,10 +204,9 @@ PAddr KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, size_t align_p
return 0; return 0;
} }
// If we allocated more than we need, free some. // Maintain the optimized memory bitmap, if we should.
const size_t allocated_pages = KPageHeap::GetBlockNumPages(heap_index); if (m_has_optimized_process[static_cast<size_t>(pool)]) {
if (allocated_pages > num_pages) { UNIMPLEMENTED();
chosen_manager->Free(allocated_block + num_pages * PageSize, allocated_pages - num_pages);
} }
// Open the first reference to the pages. // Open the first reference to the pages.
@ -209,20 +216,21 @@ PAddr KMemoryManager::AllocateAndOpenContinuous(size_t num_pages, size_t align_p
} }
Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, Pool pool, Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, Pool pool,
Direction dir, bool random) { Direction dir, bool unoptimized, bool random) {
// Choose a heap based on our page size request. // Choose a heap based on our page size request.
const s32 heap_index = KPageHeap::GetBlockIndex(num_pages); const s32 heap_index = KPageHeap::GetBlockIndex(num_pages);
R_UNLESS(0 <= heap_index, ResultOutOfMemory); R_UNLESS(0 <= heap_index, ResultOutOfMemory);
// Ensure that we don't leave anything un-freed. // Ensure that we don't leave anything un-freed.
auto group_guard = SCOPE_GUARD({ ON_RESULT_FAILURE {
for (const auto& it : out->Nodes()) { for (const auto& it : out->Nodes()) {
auto& manager = this->GetManager(system.Kernel().MemoryLayout(), it.GetAddress()); auto& manager = this->GetManager(it.GetAddress());
const size_t num_pages_to_free = const size_t node_num_pages =
std::min(it.GetNumPages(), (manager.GetEndAddress() - it.GetAddress()) / PageSize); std::min(it.GetNumPages(), (manager.GetEndAddress() - it.GetAddress()) / PageSize);
manager.Free(it.GetAddress(), num_pages_to_free); manager.Free(it.GetAddress(), node_num_pages);
} }
}); out->Finalize();
};
// Keep allocating until we've allocated all our pages. // Keep allocating until we've allocated all our pages.
for (s32 index = heap_index; index >= 0 && num_pages > 0; index--) { for (s32 index = heap_index; index >= 0 && num_pages > 0; index--) {
@ -236,12 +244,17 @@ Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages,
break; break;
} }
// Safely add it to our group. // Ensure we don't leak the block if we fail.
{ ON_RESULT_FAILURE_2 {
auto block_guard = cur_manager->Free(allocated_block, pages_per_alloc);
SCOPE_GUARD({ cur_manager->Free(allocated_block, pages_per_alloc); }); };
// Add the block to our group.
R_TRY(out->AddBlock(allocated_block, pages_per_alloc)); R_TRY(out->AddBlock(allocated_block, pages_per_alloc));
block_guard.Cancel();
// Maintain the optimized memory bitmap, if we should.
if (unoptimized) {
UNIMPLEMENTED();
} }
num_pages -= pages_per_alloc; num_pages -= pages_per_alloc;
@ -253,8 +266,7 @@ Result KMemoryManager::AllocatePageGroupImpl(KPageGroup* out, size_t num_pages,
R_UNLESS(num_pages == 0, ResultOutOfMemory); R_UNLESS(num_pages == 0, ResultOutOfMemory);
// We succeeded! // We succeeded!
group_guard.Cancel(); R_SUCCEED();
return ResultSuccess;
} }
Result KMemoryManager::AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 option) { Result KMemoryManager::AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 option) {
@ -266,10 +278,11 @@ Result KMemoryManager::AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 op
// Lock the pool that we're allocating from. // Lock the pool that we're allocating from.
const auto [pool, dir] = DecodeOption(option); const auto [pool, dir] = DecodeOption(option);
KScopedLightLock lk(pool_locks[static_cast<size_t>(pool)]); KScopedLightLock lk(m_pool_locks[static_cast<size_t>(pool)]);
// Allocate the page group. // Allocate the page group.
R_TRY(this->AllocatePageGroupImpl(out, num_pages, pool, dir, false)); R_TRY(this->AllocatePageGroupImpl(out, num_pages, pool, dir,
m_has_optimized_process[static_cast<size_t>(pool)], true));
// Open the first reference to the pages. // Open the first reference to the pages.
for (const auto& block : out->Nodes()) { for (const auto& block : out->Nodes()) {
@ -277,7 +290,7 @@ Result KMemoryManager::AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 op
size_t remaining_pages = block.GetNumPages(); size_t remaining_pages = block.GetNumPages();
while (remaining_pages > 0) { while (remaining_pages > 0) {
// Get the manager for the current address. // Get the manager for the current address.
auto& manager = this->GetManager(system.Kernel().MemoryLayout(), cur_address); auto& manager = this->GetManager(cur_address);
// Process part or all of the block. // Process part or all of the block.
const size_t cur_pages = const size_t cur_pages =
@ -290,10 +303,10 @@ Result KMemoryManager::AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 op
} }
} }
return ResultSuccess; R_SUCCEED();
} }
Result KMemoryManager::AllocateAndOpenForProcess(KPageGroup* out, size_t num_pages, u32 option, Result KMemoryManager::AllocateForProcess(KPageGroup* out, size_t num_pages, u32 option,
u64 process_id, u8 fill_pattern) { u64 process_id, u8 fill_pattern) {
ASSERT(out != nullptr); ASSERT(out != nullptr);
ASSERT(out->GetNumPages() == 0); ASSERT(out->GetNumPages() == 0);
@ -302,25 +315,73 @@ Result KMemoryManager::AllocateAndOpenForProcess(KPageGroup* out, size_t num_pag
const auto [pool, dir] = DecodeOption(option); const auto [pool, dir] = DecodeOption(option);
// Allocate the memory. // Allocate the memory.
bool optimized;
{ {
// Lock the pool that we're allocating from. // Lock the pool that we're allocating from.
KScopedLightLock lk(pool_locks[static_cast<size_t>(pool)]); KScopedLightLock lk(m_pool_locks[static_cast<size_t>(pool)]);
// Check if we have an optimized process.
const bool has_optimized = m_has_optimized_process[static_cast<size_t>(pool)];
const bool is_optimized = m_optimized_process_ids[static_cast<size_t>(pool)] == process_id;
// Allocate the page group. // Allocate the page group.
R_TRY(this->AllocatePageGroupImpl(out, num_pages, pool, dir, false)); R_TRY(this->AllocatePageGroupImpl(out, num_pages, pool, dir, has_optimized && !is_optimized,
false));
// Open the first reference to the pages. // Set whether we should optimize.
optimized = has_optimized && is_optimized;
}
// Perform optimized memory tracking, if we should.
if (optimized) {
// Iterate over the allocated blocks.
for (const auto& block : out->Nodes()) { for (const auto& block : out->Nodes()) {
PAddr cur_address = block.GetAddress(); // Get the block extents.
size_t remaining_pages = block.GetNumPages(); const PAddr block_address = block.GetAddress();
const size_t block_pages = block.GetNumPages();
// If it has no pages, we don't need to do anything.
if (block_pages == 0) {
continue;
}
// Fill all the pages that we need to fill.
bool any_new = false;
{
PAddr cur_address = block_address;
size_t remaining_pages = block_pages;
while (remaining_pages > 0) { while (remaining_pages > 0) {
// Get the manager for the current address. // Get the manager for the current address.
auto& manager = this->GetManager(system.Kernel().MemoryLayout(), cur_address); auto& manager = this->GetManager(cur_address);
// Process part or all of the block. // Process part or all of the block.
const size_t cur_pages = const size_t cur_pages =
std::min(remaining_pages, manager.GetPageOffsetToEnd(cur_address)); std::min(remaining_pages, manager.GetPageOffsetToEnd(cur_address));
manager.OpenFirst(cur_address, cur_pages); any_new =
manager.ProcessOptimizedAllocation(cur_address, cur_pages, fill_pattern);
// Advance.
cur_address += cur_pages * PageSize;
remaining_pages -= cur_pages;
}
}
// If there are new pages, update tracking for the allocation.
if (any_new) {
// Update tracking for the allocation.
PAddr cur_address = block_address;
size_t remaining_pages = block_pages;
while (remaining_pages > 0) {
// Get the manager for the current address.
auto& manager = this->GetManager(cur_address);
// Lock the pool for the manager.
KScopedLightLock lk(m_pool_locks[static_cast<size_t>(manager.GetPool())]);
// Track some or all of the current pages.
const size_t cur_pages =
std::min(remaining_pages, manager.GetPageOffsetToEnd(cur_address));
manager.TrackOptimizedAllocation(cur_address, cur_pages);
// Advance. // Advance.
cur_address += cur_pages * PageSize; cur_address += cur_pages * PageSize;
@ -328,57 +389,15 @@ Result KMemoryManager::AllocateAndOpenForProcess(KPageGroup* out, size_t num_pag
} }
} }
} }
} else {
// Set all the allocated memory. // Set all the allocated memory.
for (const auto& block : out->Nodes()) { for (const auto& block : out->Nodes()) {
std::memset(system.DeviceMemory().GetPointer<void>(block.GetAddress()), fill_pattern, std::memset(m_system.DeviceMemory().GetPointer<void>(block.GetAddress()), fill_pattern,
block.GetSize()); block.GetSize());
} }
return ResultSuccess;
}
void KMemoryManager::Open(PAddr address, size_t num_pages) {
// Repeatedly open references until we've done so for all pages.
while (num_pages) {
auto& manager = this->GetManager(system.Kernel().MemoryLayout(), address);
const size_t cur_pages = std::min(num_pages, manager.GetPageOffsetToEnd(address));
{
KScopedLightLock lk(pool_locks[static_cast<size_t>(manager.GetPool())]);
manager.Open(address, cur_pages);
} }
num_pages -= cur_pages; R_SUCCEED();
address += cur_pages * PageSize;
}
}
void KMemoryManager::Close(PAddr address, size_t num_pages) {
// Repeatedly close references until we've done so for all pages.
while (num_pages) {
auto& manager = this->GetManager(system.Kernel().MemoryLayout(), address);
const size_t cur_pages = std::min(num_pages, manager.GetPageOffsetToEnd(address));
{
KScopedLightLock lk(pool_locks[static_cast<size_t>(manager.GetPool())]);
manager.Close(address, cur_pages);
}
num_pages -= cur_pages;
address += cur_pages * PageSize;
}
}
void KMemoryManager::Close(const KPageGroup& pg) {
for (const auto& node : pg.Nodes()) {
Close(node.GetAddress(), node.GetNumPages());
}
}
void KMemoryManager::Open(const KPageGroup& pg) {
for (const auto& node : pg.Nodes()) {
Open(node.GetAddress(), node.GetNumPages());
}
} }
size_t KMemoryManager::Impl::Initialize(PAddr address, size_t size, VAddr management, size_t KMemoryManager::Impl::Initialize(PAddr address, size_t size, VAddr management,
@ -394,18 +413,31 @@ size_t KMemoryManager::Impl::Initialize(PAddr address, size_t size, VAddr manage
ASSERT(Common::IsAligned(total_management_size, PageSize)); ASSERT(Common::IsAligned(total_management_size, PageSize));
// Setup region. // Setup region.
pool = p; m_pool = p;
management_region = management; m_management_region = management;
page_reference_counts.resize( m_page_reference_counts.resize(
Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize() / PageSize); Kernel::Board::Nintendo::Nx::KSystemControl::Init::GetIntendedMemorySize() / PageSize);
ASSERT(Common::IsAligned(management_region, PageSize)); ASSERT(Common::IsAligned(m_management_region, PageSize));
// Initialize the manager's KPageHeap. // Initialize the manager's KPageHeap.
heap.Initialize(address, size, management + manager_size, page_heap_size); m_heap.Initialize(address, size, management + manager_size, page_heap_size);
return total_management_size; return total_management_size;
} }
void KMemoryManager::Impl::TrackUnoptimizedAllocation(PAddr block, size_t num_pages) {
UNREACHABLE();
}
void KMemoryManager::Impl::TrackOptimizedAllocation(PAddr block, size_t num_pages) {
UNREACHABLE();
}
bool KMemoryManager::Impl::ProcessOptimizedAllocation(PAddr block, size_t num_pages,
u8 fill_pattern) {
UNREACHABLE();
}
size_t KMemoryManager::Impl::CalculateManagementOverheadSize(size_t region_size) { size_t KMemoryManager::Impl::CalculateManagementOverheadSize(size_t region_size) {
const size_t ref_count_size = (region_size / PageSize) * sizeof(u16); const size_t ref_count_size = (region_size / PageSize) * sizeof(u16);
const size_t optimize_map_size = const size_t optimize_map_size =

257
src/core/hle/kernel/k_memory_manager.h
View file

@ -21,11 +21,8 @@ namespace Kernel {
class KPageGroup; class KPageGroup;
class KMemoryManager final { class KMemoryManager {
public: public:
YUZU_NON_COPYABLE(KMemoryManager);
YUZU_NON_MOVEABLE(KMemoryManager);
enum class Pool : u32 { enum class Pool : u32 {
Application = 0, Application = 0,
Applet = 1, Applet = 1,
@ -45,16 +42,85 @@ public:
enum class Direction : u32 { enum class Direction : u32 {
FromFront = 0, FromFront = 0,
FromBack = 1, FromBack = 1,
Shift = 0, Shift = 0,
Mask = (0xF << Shift), Mask = (0xF << Shift),
}; };
explicit KMemoryManager(Core::System& system_); static constexpr size_t MaxManagerCount = 10;
explicit KMemoryManager(Core::System& system);
void Initialize(VAddr management_region, size_t management_region_size); void Initialize(VAddr management_region, size_t management_region_size);
constexpr size_t GetSize(Pool pool) const { Result InitializeOptimizedMemory(u64 process_id, Pool pool);
void FinalizeOptimizedMemory(u64 process_id, Pool pool);
PAddr AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option);
Result AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 option);
Result AllocateForProcess(KPageGroup* out, size_t num_pages, u32 option, u64 process_id,
u8 fill_pattern);
Pool GetPool(PAddr address) const {
return this->GetManager(address).GetPool();
}
void Open(PAddr address, size_t num_pages) {
// Repeatedly open references until we've done so for all pages.
while (num_pages) {
auto& manager = this->GetManager(address);
const size_t cur_pages = std::min(num_pages, manager.GetPageOffsetToEnd(address));
{
KScopedLightLock lk(m_pool_locks[static_cast<size_t>(manager.GetPool())]);
manager.Open(address, cur_pages);
}
num_pages -= cur_pages;
address += cur_pages * PageSize;
}
}
void OpenFirst(PAddr address, size_t num_pages) {
// Repeatedly open references until we've done so for all pages.
while (num_pages) {
auto& manager = this->GetManager(address);
const size_t cur_pages = std::min(num_pages, manager.GetPageOffsetToEnd(address));
{
KScopedLightLock lk(m_pool_locks[static_cast<size_t>(manager.GetPool())]);
manager.OpenFirst(address, cur_pages);
}
num_pages -= cur_pages;
address += cur_pages * PageSize;
}
}
void Close(PAddr address, size_t num_pages) {
// Repeatedly close references until we've done so for all pages.
while (num_pages) {
auto& manager = this->GetManager(address);
const size_t cur_pages = std::min(num_pages, manager.GetPageOffsetToEnd(address));
{
KScopedLightLock lk(m_pool_locks[static_cast<size_t>(manager.GetPool())]);
manager.Close(address, cur_pages);
}
num_pages -= cur_pages;
address += cur_pages * PageSize;
}
}
size_t GetSize() {
size_t total = 0;
for (size_t i = 0; i < m_num_managers; i++) {
total += m_managers[i].GetSize();
}
return total;
}
size_t GetSize(Pool pool) {
constexpr Direction GetSizeDirection = Direction::FromFront; constexpr Direction GetSizeDirection = Direction::FromFront;
size_t total = 0; size_t total = 0;
for (auto* manager = this->GetFirstManager(pool, GetSizeDirection); manager != nullptr; for (auto* manager = this->GetFirstManager(pool, GetSizeDirection); manager != nullptr;
@ -64,18 +130,36 @@ public:
return total; return total;
} }
PAddr AllocateAndOpenContinuous(size_t num_pages, size_t align_pages, u32 option); size_t GetFreeSize() {
Result AllocateAndOpen(KPageGroup* out, size_t num_pages, u32 option); size_t total = 0;
Result AllocateAndOpenForProcess(KPageGroup* out, size_t num_pages, u32 option, u64 process_id, for (size_t i = 0; i < m_num_managers; i++) {
u8 fill_pattern); KScopedLightLock lk(m_pool_locks[static_cast<size_t>(m_managers[i].GetPool())]);
total += m_managers[i].GetFreeSize();
}
return total;
}
static constexpr size_t MaxManagerCount = 10; size_t GetFreeSize(Pool pool) {
KScopedLightLock lk(m_pool_locks[static_cast<size_t>(pool)]);
void Close(PAddr address, size_t num_pages); constexpr Direction GetSizeDirection = Direction::FromFront;
void Close(const KPageGroup& pg); size_t total = 0;
for (auto* manager = this->GetFirstManager(pool, GetSizeDirection); manager != nullptr;
manager = this->GetNextManager(manager, GetSizeDirection)) {
total += manager->GetFreeSize();
}
return total;
}
void Open(PAddr address, size_t num_pages); void DumpFreeList(Pool pool) {
void Open(const KPageGroup& pg); KScopedLightLock lk(m_pool_locks[static_cast<size_t>(pool)]);
constexpr Direction DumpDirection = Direction::FromFront;
for (auto* manager = this->GetFirstManager(pool, DumpDirection); manager != nullptr;
manager = this->GetNextManager(manager, DumpDirection)) {
manager->DumpFreeList();
}
}
public: public:
static size_t CalculateManagementOverheadSize(size_t region_size) { static size_t CalculateManagementOverheadSize(size_t region_size) {
@ -88,13 +172,12 @@ public:
} }
static constexpr Pool GetPool(u32 option) { static constexpr Pool GetPool(u32 option) {
return static_cast<Pool>((static_cast<u32>(option) & static_cast<u32>(Pool::Mask)) >> return static_cast<Pool>((option & static_cast<u32>(Pool::Mask)) >>
static_cast<u32>(Pool::Shift)); static_cast<u32>(Pool::Shift));
} }
static constexpr Direction GetDirection(u32 option) { static constexpr Direction GetDirection(u32 option) {
return static_cast<Direction>( return static_cast<Direction>((option & static_cast<u32>(Direction::Mask)) >>
(static_cast<u32>(option) & static_cast<u32>(Direction::Mask)) >>
static_cast<u32>(Direction::Shift)); static_cast<u32>(Direction::Shift));
} }
@ -103,74 +186,88 @@ public:
} }
private: private:
class Impl final { class Impl {
public: public:
YUZU_NON_COPYABLE(Impl); static size_t CalculateManagementOverheadSize(size_t region_size);
YUZU_NON_MOVEABLE(Impl);
static constexpr size_t CalculateOptimizedProcessOverheadSize(size_t region_size) {
return (Common::AlignUp((region_size / PageSize), Common::BitSize<u64>()) /
Common::BitSize<u64>()) *
sizeof(u64);
}
public:
Impl() = default; Impl() = default;
~Impl() = default;
size_t Initialize(PAddr address, size_t size, VAddr management, VAddr management_end, size_t Initialize(PAddr address, size_t size, VAddr management, VAddr management_end,
Pool p); Pool p);
VAddr AllocateBlock(s32 index, bool random) { PAddr AllocateBlock(s32 index, bool random) {
return heap.AllocateBlock(index, random); return m_heap.AllocateBlock(index, random);
} }
PAddr AllocateAligned(s32 index, size_t num_pages, size_t align_pages) {
void Free(VAddr addr, size_t num_pages) { return m_heap.AllocateAligned(index, num_pages, align_pages);
heap.Free(addr, num_pages); }
void Free(PAddr addr, size_t num_pages) {
m_heap.Free(addr, num_pages);
} }
void SetInitialUsedHeapSize(size_t reserved_size) { void SetInitialUsedHeapSize(size_t reserved_size) {
heap.SetInitialUsedSize(reserved_size); m_heap.SetInitialUsedSize(reserved_size);
} }
void InitializeOptimizedMemory() {
UNIMPLEMENTED();
}
void TrackUnoptimizedAllocation(PAddr block, size_t num_pages);
void TrackOptimizedAllocation(PAddr block, size_t num_pages);
bool ProcessOptimizedAllocation(PAddr block, size_t num_pages, u8 fill_pattern);
constexpr Pool GetPool() const { constexpr Pool GetPool() const {
return pool; return m_pool;
} }
constexpr size_t GetSize() const { constexpr size_t GetSize() const {
return heap.GetSize(); return m_heap.GetSize();
}
constexpr PAddr GetEndAddress() const {
return m_heap.GetEndAddress();
} }
constexpr VAddr GetAddress() const { size_t GetFreeSize() const {
return heap.GetAddress(); return m_heap.GetFreeSize();
} }
constexpr VAddr GetEndAddress() const { void DumpFreeList() const {
return heap.GetEndAddress(); UNIMPLEMENTED();
} }
constexpr size_t GetPageOffset(PAddr address) const { constexpr size_t GetPageOffset(PAddr address) const {
return heap.GetPageOffset(address); return m_heap.GetPageOffset(address);
} }
constexpr size_t GetPageOffsetToEnd(PAddr address) const { constexpr size_t GetPageOffsetToEnd(PAddr address) const {
return heap.GetPageOffsetToEnd(address); return m_heap.GetPageOffsetToEnd(address);
} }
constexpr void SetNext(Impl* n) { constexpr void SetNext(Impl* n) {
next = n; m_next = n;
} }
constexpr void SetPrev(Impl* n) { constexpr void SetPrev(Impl* n) {
prev = n; m_prev = n;
} }
constexpr Impl* GetNext() const { constexpr Impl* GetNext() const {
return next; return m_next;
} }
constexpr Impl* GetPrev() const { constexpr Impl* GetPrev() const {
return prev; return m_prev;
} }
void OpenFirst(PAddr address, size_t num_pages) { void OpenFirst(PAddr address, size_t num_pages) {
size_t index = this->GetPageOffset(address); size_t index = this->GetPageOffset(address);
const size_t end = index + num_pages; const size_t end = index + num_pages;
while (index < end) { while (index < end) {
const RefCount ref_count = (++page_reference_counts[index]); const RefCount ref_count = (++m_page_reference_counts[index]);
ASSERT(ref_count == 1); ASSERT(ref_count == 1);
index++; index++;
@ -181,7 +278,7 @@ private:
size_t index = this->GetPageOffset(address); size_t index = this->GetPageOffset(address);
const size_t end = index + num_pages; const size_t end = index + num_pages;
while (index < end) { while (index < end) {
const RefCount ref_count = (++page_reference_counts[index]); const RefCount ref_count = (++m_page_reference_counts[index]);
ASSERT(ref_count > 1); ASSERT(ref_count > 1);
index++; index++;
@ -195,8 +292,8 @@ private:
size_t free_start = 0; size_t free_start = 0;
size_t free_count = 0; size_t free_count = 0;
while (index < end) { while (index < end) {
ASSERT(page_reference_counts[index] > 0); ASSERT(m_page_reference_counts[index] > 0);
const RefCount ref_count = (--page_reference_counts[index]); const RefCount ref_count = (--m_page_reference_counts[index]);
// Keep track of how many zero refcounts we see in a row, to minimize calls to free. // Keep track of how many zero refcounts we see in a row, to minimize calls to free.
if (ref_count == 0) { if (ref_count == 0) {
@ -208,7 +305,7 @@ private:
} }
} else { } else {
if (free_count > 0) { if (free_count > 0) {
this->Free(heap.GetAddress() + free_start * PageSize, free_count); this->Free(m_heap.GetAddress() + free_start * PageSize, free_count);
free_count = 0; free_count = 0;
} }
} }
@ -217,44 +314,36 @@ private:
} }
if (free_count > 0) { if (free_count > 0) {
this->Free(heap.GetAddress() + free_start * PageSize, free_count); this->Free(m_heap.GetAddress() + free_start * PageSize, free_count);
} }
} }
static size_t CalculateManagementOverheadSize(size_t region_size);
static constexpr size_t CalculateOptimizedProcessOverheadSize(size_t region_size) {
return (Common::AlignUp((region_size / PageSize), Common::BitSize<u64>()) /
Common::BitSize<u64>()) *
sizeof(u64);
}
private: private:
using RefCount = u16; using RefCount = u16;
KPageHeap heap; KPageHeap m_heap;
std::vector<RefCount> page_reference_counts; std::vector<RefCount> m_page_reference_counts;
VAddr management_region{}; VAddr m_management_region{};
Pool pool{}; Pool m_pool{};
Impl* next{}; Impl* m_next{};
Impl* prev{}; Impl* m_prev{};
}; };
private: private:
Impl& GetManager(const KMemoryLayout& memory_layout, PAddr address) { Impl& GetManager(PAddr address) {
return managers[memory_layout.GetPhysicalLinearRegion(address).GetAttributes()]; return m_managers[m_memory_layout.GetPhysicalLinearRegion(address).GetAttributes()];
} }
const Impl& GetManager(const KMemoryLayout& memory_layout, PAddr address) const { const Impl& GetManager(PAddr address) const {
return managers[memory_layout.GetPhysicalLinearRegion(address).GetAttributes()]; return m_managers[m_memory_layout.GetPhysicalLinearRegion(address).GetAttributes()];
} }
constexpr Impl* GetFirstManager(Pool pool, Direction dir) const { constexpr Impl* GetFirstManager(Pool pool, Direction dir) {
return dir == Direction::FromBack ? pool_managers_tail[static_cast<size_t>(pool)] return dir == Direction::FromBack ? m_pool_managers_tail[static_cast<size_t>(pool)]
: pool_managers_head[static_cast<size_t>(pool)]; : m_pool_managers_head[static_cast<size_t>(pool)];
} }
constexpr Impl* GetNextManager(Impl* cur, Direction dir) const { constexpr Impl* GetNextManager(Impl* cur, Direction dir) {
if (dir == Direction::FromBack) { if (dir == Direction::FromBack) {
return cur->GetPrev(); return cur->GetPrev();
} else { } else {
@ -263,15 +352,21 @@ private:
} }
Result AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, Pool pool, Direction dir, Result AllocatePageGroupImpl(KPageGroup* out, size_t num_pages, Pool pool, Direction dir,
bool random); bool unoptimized, bool random);
private: private:
Core::System& system; template <typename T>
std::array<KLightLock, static_cast<size_t>(Pool::Count)> pool_locks; using PoolArray = std::array<T, static_cast<size_t>(Pool::Count)>;
std::array<Impl*, MaxManagerCount> pool_managers_head{};
std::array<Impl*, MaxManagerCount> pool_managers_tail{}; Core::System& m_system;
std::array<Impl, MaxManagerCount> managers; const KMemoryLayout& m_memory_layout;
size_t num_managers{}; PoolArray<KLightLock> m_pool_locks;
std::array<Impl*, MaxManagerCount> m_pool_managers_head{};
std::array<Impl*, MaxManagerCount> m_pool_managers_tail{};
std::array<Impl, MaxManagerCount> m_managers;
size_t m_num_managers{};
PoolArray<u64> m_optimized_process_ids{};
PoolArray<bool> m_has_optimized_process{};
}; };
} // namespace Kernel } // namespace Kernel

294
src/core/hle/kernel/k_page_table.cpp
View file

@ -114,7 +114,7 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type
// Set other basic fields // Set other basic fields
m_enable_aslr = enable_aslr; m_enable_aslr = enable_aslr;
m_enable_device_address_space_merge = false; m_enable_device_address_space_merge = enable_das_merge;
m_address_space_start = start; m_address_space_start = start;
m_address_space_end = end; m_address_space_end = end;
m_is_kernel = false; m_is_kernel = false;
@ -219,10 +219,22 @@ Result KPageTable::InitializeForProcess(FileSys::ProgramAddressSpaceType as_type
} }
} }
// Set heap members // Set heap and fill members.
m_current_heap_end = m_heap_region_start; m_current_heap_end = m_heap_region_start;
m_max_heap_size = 0; m_max_heap_size = 0;
m_max_physical_memory_size = 0; m_mapped_physical_memory_size = 0;
m_mapped_unsafe_physical_memory = 0;
m_mapped_insecure_memory = 0;
m_mapped_ipc_server_memory = 0;
m_heap_fill_value = 0;
m_ipc_fill_value = 0;
m_stack_fill_value = 0;
// Set allocation option.
m_allocate_option =
KMemoryManager::EncodeOption(pool, from_back ? KMemoryManager::Direction::FromBack
: KMemoryManager::Direction::FromFront);
// Ensure that we regions inside our address space // Ensure that we regions inside our address space
auto IsInAddressSpace = [&](VAddr addr) { auto IsInAddressSpace = [&](VAddr addr) {
@ -271,6 +283,16 @@ void KPageTable::Finalize() {
m_system.Memory().UnmapRegion(*m_page_table_impl, addr, size); m_system.Memory().UnmapRegion(*m_page_table_impl, addr, size);
}); });
// Release any insecure mapped memory.
if (m_mapped_insecure_memory) {
UNIMPLEMENTED();
}
// Release any ipc server memory.
if (m_mapped_ipc_server_memory) {
UNIMPLEMENTED();
}
// Close the backing page table, as the destructor is not called for guest objects. // Close the backing page table, as the destructor is not called for guest objects.
m_page_table_impl.reset(); m_page_table_impl.reset();
} }
@ -690,9 +712,20 @@ Result KPageTable::UnmapProcessMemory(VAddr dst_addr, size_t size, KPageTable& s
R_SUCCEED(); R_SUCCEED();
} }
void KPageTable::HACK_OpenPages(PAddr phys_addr, size_t num_pages) {
m_system.Kernel().MemoryManager().OpenFirst(phys_addr, num_pages);
}
void KPageTable::HACK_ClosePages(VAddr virt_addr, size_t num_pages) {
for (size_t index = 0; index < num_pages; ++index) {
const auto paddr = GetPhysicalAddr(virt_addr + (index * PageSize));
m_system.Kernel().MemoryManager().Close(paddr, 1);
}
}
Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) { Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
// Lock the physical memory lock. // Lock the physical memory lock.
KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); KScopedLightLock phys_lk(m_map_physical_memory_lock);
// Calculate the last address for convenience. // Calculate the last address for convenience.
const VAddr last_address = address + size - 1; const VAddr last_address = address + size - 1;
@ -746,15 +779,19 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
{ {
// Reserve the memory from the process resource limit. // Reserve the memory from the process resource limit.
KScopedResourceReservation memory_reservation( KScopedResourceReservation memory_reservation(
m_system.Kernel().CurrentProcess()->GetResourceLimit(), m_resource_limit, LimitableResource::PhysicalMemory, size - mapped_size);
LimitableResource::PhysicalMemory, size - mapped_size);
R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached);
// Allocate pages for the new memory. // Allocate pages for the new memory.
KPageGroup pg; KPageGroup pg;
R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpenForProcess( R_TRY(m_system.Kernel().MemoryManager().AllocateForProcess(
&pg, (size - mapped_size) / PageSize, &pg, (size - mapped_size) / PageSize, m_allocate_option, 0, 0));
KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0));
// If we fail in the next bit (or retry), we need to cleanup the pages.
// auto pg_guard = SCOPE_GUARD {
// pg.OpenFirst();
// pg.Close();
//};
// Map the memory. // Map the memory.
{ {
@ -814,15 +851,24 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
// Create an update allocator. // Create an update allocator.
ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks);
Result allocator_result{ResultSuccess}; Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
m_memory_block_slab_manager, m_memory_block_slab_manager,
num_allocator_blocks); num_allocator_blocks);
R_TRY(allocator_result); R_TRY(allocator_result);
// We're going to perform an update, so create a helper.
// KScopedPageTableUpdater updater(this);
// Prepare to iterate over the memory.
auto pg_it = pg.Nodes().begin();
PAddr pg_phys_addr = pg_it->GetAddress();
size_t pg_pages = pg_it->GetNumPages();
// Reset the current tracking address, and make sure we clean up on failure. // Reset the current tracking address, and make sure we clean up on failure.
// pg_guard.Cancel();
cur_address = address; cur_address = address;
auto unmap_guard = detail::ScopeExit([&] { ON_RESULT_FAILURE {
if (cur_address > address) { if (cur_address > address) {
const VAddr last_unmap_address = cur_address - 1; const VAddr last_unmap_address = cur_address - 1;
@ -845,6 +891,9 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
last_unmap_address + 1 - cur_address) / last_unmap_address + 1 - cur_address) /
PageSize; PageSize;
// HACK: Manually close the pages.
HACK_ClosePages(cur_address, cur_pages);
// Unmap. // Unmap.
ASSERT(Operate(cur_address, cur_pages, KMemoryPermission::None, ASSERT(Operate(cur_address, cur_pages, KMemoryPermission::None,
OperationType::Unmap) OperationType::Unmap)
@ -861,12 +910,17 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
++it; ++it;
} }
} }
});
// Iterate over the memory. // Release any remaining unmapped memory.
auto pg_it = pg.Nodes().begin(); m_system.Kernel().MemoryManager().OpenFirst(pg_phys_addr, pg_pages);
PAddr pg_phys_addr = pg_it->GetAddress(); m_system.Kernel().MemoryManager().Close(pg_phys_addr, pg_pages);
size_t pg_pages = pg_it->GetNumPages(); for (++pg_it; pg_it != pg.Nodes().end(); ++pg_it) {
m_system.Kernel().MemoryManager().OpenFirst(pg_it->GetAddress(),
pg_it->GetNumPages());
m_system.Kernel().MemoryManager().Close(pg_it->GetAddress(),
pg_it->GetNumPages());
}
};
auto it = m_memory_block_manager.FindIterator(cur_address); auto it = m_memory_block_manager.FindIterator(cur_address);
while (true) { while (true) {
@ -901,6 +955,9 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
R_TRY(Operate(cur_address, cur_pages, KMemoryPermission::UserReadWrite, R_TRY(Operate(cur_address, cur_pages, KMemoryPermission::UserReadWrite,
OperationType::Map, pg_phys_addr)); OperationType::Map, pg_phys_addr));
// HACK: Manually open the pages.
HACK_OpenPages(pg_phys_addr, cur_pages);
// Advance. // Advance.
cur_address += cur_pages * PageSize; cur_address += cur_pages * PageSize;
map_pages -= cur_pages; map_pages -= cur_pages;
@ -932,9 +989,6 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
KMemoryPermission::None, KMemoryAttribute::None, KMemoryState::Normal, KMemoryPermission::None, KMemoryAttribute::None, KMemoryState::Normal,
KMemoryPermission::UserReadWrite, KMemoryAttribute::None); KMemoryPermission::UserReadWrite, KMemoryAttribute::None);
// Cancel our guard.
unmap_guard.Cancel();
R_SUCCEED(); R_SUCCEED();
} }
} }
@ -943,7 +997,7 @@ Result KPageTable::MapPhysicalMemory(VAddr address, size_t size) {
Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) { Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
// Lock the physical memory lock. // Lock the physical memory lock.
KScopedLightLock map_phys_mem_lk(m_map_physical_memory_lock); KScopedLightLock phys_lk(m_map_physical_memory_lock);
// Lock the table. // Lock the table.
KScopedLightLock lk(m_general_lock); KScopedLightLock lk(m_general_lock);
@ -952,8 +1006,11 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
const VAddr last_address = address + size - 1; const VAddr last_address = address + size - 1;
// Define iteration variables. // Define iteration variables.
VAddr cur_address = 0; VAddr map_start_address = 0;
size_t mapped_size = 0; VAddr map_last_address = 0;
VAddr cur_address;
size_t mapped_size;
size_t num_allocator_blocks = 0; size_t num_allocator_blocks = 0;
// Check if the memory is mapped. // Check if the memory is mapped.
@ -979,27 +1036,27 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
if (is_normal) { if (is_normal) {
R_UNLESS(info.GetAttribute() == KMemoryAttribute::None, ResultInvalidCurrentMemory); R_UNLESS(info.GetAttribute() == KMemoryAttribute::None, ResultInvalidCurrentMemory);
if (map_start_address == 0) {
map_start_address = cur_address;
}
map_last_address =
(last_address >= info.GetLastAddress()) ? info.GetLastAddress() : last_address;
if (info.GetAddress() < address) { if (info.GetAddress() < address) {
++num_allocator_blocks; ++num_allocator_blocks;
} }
if (last_address < info.GetLastAddress()) { if (last_address < info.GetLastAddress()) {
++num_allocator_blocks; ++num_allocator_blocks;
} }
mapped_size += (map_last_address + 1 - cur_address);
} }
// Check if we're done. // Check if we're done.
if (last_address <= info.GetLastAddress()) { if (last_address <= info.GetLastAddress()) {
if (is_normal) {
mapped_size += (last_address + 1 - cur_address);
}
break; break;
} }
// Track the memory if it's mapped.
if (is_normal) {
mapped_size += VAddr(info.GetEndAddress()) - cur_address;
}
// Advance. // Advance.
cur_address = info.GetEndAddress(); cur_address = info.GetEndAddress();
++it; ++it;
@ -1009,125 +1066,22 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
R_SUCCEED_IF(mapped_size == 0); R_SUCCEED_IF(mapped_size == 0);
} }
// Make a page group for the unmap region.
KPageGroup pg;
{
auto& impl = this->PageTableImpl();
// Begin traversal.
Common::PageTable::TraversalContext context;
Common::PageTable::TraversalEntry cur_entry = {.phys_addr = 0, .block_size = 0};
bool cur_valid = false;
Common::PageTable::TraversalEntry next_entry;
bool next_valid = false;
size_t tot_size = 0;
cur_address = address;
next_valid = impl.BeginTraversal(next_entry, context, cur_address);
next_entry.block_size =
(next_entry.block_size - (next_entry.phys_addr & (next_entry.block_size - 1)));
// Iterate, building the group.
while (true) {
if ((!next_valid && !cur_valid) ||
(next_valid && cur_valid &&
next_entry.phys_addr == cur_entry.phys_addr + cur_entry.block_size)) {
cur_entry.block_size += next_entry.block_size;
} else {
if (cur_valid) {
// ASSERT(IsHeapPhysicalAddress(cur_entry.phys_addr));
R_TRY(pg.AddBlock(cur_entry.phys_addr, cur_entry.block_size / PageSize));
}
// Update tracking variables.
tot_size += cur_entry.block_size;
cur_entry = next_entry;
cur_valid = next_valid;
}
if (cur_entry.block_size + tot_size >= size) {
break;
}
next_valid = impl.ContinueTraversal(next_entry, context);
}
// Add the last block.
if (cur_valid) {
// ASSERT(IsHeapPhysicalAddress(cur_entry.phys_addr));
R_TRY(pg.AddBlock(cur_entry.phys_addr, (size - tot_size) / PageSize));
}
}
ASSERT(pg.GetNumPages() == mapped_size / PageSize);
// Create an update allocator. // Create an update allocator.
ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks); ASSERT(num_allocator_blocks <= KMemoryBlockManagerUpdateAllocator::MaxBlocks);
Result allocator_result{ResultSuccess}; Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
m_memory_block_slab_manager, num_allocator_blocks); m_memory_block_slab_manager, num_allocator_blocks);
R_TRY(allocator_result); R_TRY(allocator_result);
// We're going to perform an update, so create a helper.
// KScopedPageTableUpdater updater(this);
// Separate the mapping.
R_TRY(Operate(map_start_address, (map_last_address + 1 - map_start_address) / PageSize,
KMemoryPermission::None, OperationType::Separate));
// Reset the current tracking address, and make sure we clean up on failure. // Reset the current tracking address, and make sure we clean up on failure.
cur_address = address; cur_address = address;
auto remap_guard = detail::ScopeExit([&] {
if (cur_address > address) {
const VAddr last_map_address = cur_address - 1;
cur_address = address;
// Iterate over the memory we unmapped.
auto it = m_memory_block_manager.FindIterator(cur_address);
auto pg_it = pg.Nodes().begin();
PAddr pg_phys_addr = pg_it->GetAddress();
size_t pg_pages = pg_it->GetNumPages();
while (true) {
// Get the memory info for the pages we unmapped, convert to property.
const KMemoryInfo info = it->GetMemoryInfo();
// If the memory is normal, we unmapped it and need to re-map it.
if (info.GetState() == KMemoryState::Normal) {
// Determine the range to map.
size_t map_pages = std::min(VAddr(info.GetEndAddress()) - cur_address,
last_map_address + 1 - cur_address) /
PageSize;
// While we have pages to map, map them.
while (map_pages > 0) {
// Check if we're at the end of the physical block.
if (pg_pages == 0) {
// Ensure there are more pages to map.
ASSERT(pg_it != pg.Nodes().end());
// Advance our physical block.
++pg_it;
pg_phys_addr = pg_it->GetAddress();
pg_pages = pg_it->GetNumPages();
}
// Map whatever we can.
const size_t cur_pages = std::min(pg_pages, map_pages);
ASSERT(this->Operate(cur_address, cur_pages, info.GetPermission(),
OperationType::Map, pg_phys_addr) == ResultSuccess);
// Advance.
cur_address += cur_pages * PageSize;
map_pages -= cur_pages;
pg_phys_addr += cur_pages * PageSize;
pg_pages -= cur_pages;
}
}
// Check if we're done.
if (last_map_address <= info.GetLastAddress()) {
break;
}
// Advance.
++it;
}
}
});
// Iterate over the memory, unmapping as we go. // Iterate over the memory, unmapping as we go.
auto it = m_memory_block_manager.FindIterator(cur_address); auto it = m_memory_block_manager.FindIterator(cur_address);
@ -1145,8 +1099,12 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
last_address + 1 - cur_address) / last_address + 1 - cur_address) /
PageSize; PageSize;
// HACK: Manually close the pages.
HACK_ClosePages(cur_address, cur_pages);
// Unmap. // Unmap.
R_TRY(Operate(cur_address, cur_pages, KMemoryPermission::None, OperationType::Unmap)); ASSERT(Operate(cur_address, cur_pages, KMemoryPermission::None, OperationType::Unmap)
.IsSuccess());
} }
// Check if we're done. // Check if we're done.
@ -1161,8 +1119,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
// Release the memory resource. // Release the memory resource.
m_mapped_physical_memory_size -= mapped_size; m_mapped_physical_memory_size -= mapped_size;
auto process{m_system.Kernel().CurrentProcess()}; m_resource_limit->Release(LimitableResource::PhysicalMemory, mapped_size);
process->GetResourceLimit()->Release(LimitableResource::PhysicalMemory, mapped_size);
// Update memory blocks. // Update memory blocks.
m_memory_block_manager.Update(std::addressof(allocator), address, size / PageSize, m_memory_block_manager.Update(std::addressof(allocator), address, size / PageSize,
@ -1170,14 +1127,7 @@ Result KPageTable::UnmapPhysicalMemory(VAddr address, size_t size) {
KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None, KMemoryAttribute::None, KMemoryBlockDisableMergeAttribute::None,
KMemoryBlockDisableMergeAttribute::None); KMemoryBlockDisableMergeAttribute::None);
// TODO(bunnei): This is a workaround until the next set of changes, where we add reference
// counting for mapped pages. Until then, we must manually close the reference to the page
// group.
m_system.Kernel().MemoryManager().Close(pg);
// We succeeded. // We succeeded.
remap_guard.Cancel();
R_SUCCEED(); R_SUCCEED();
} }
@ -1753,8 +1703,7 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) {
OperationType::Unmap)); OperationType::Unmap));
// Release the memory from the resource limit. // Release the memory from the resource limit.
m_system.Kernel().CurrentProcess()->GetResourceLimit()->Release( m_resource_limit->Release(LimitableResource::PhysicalMemory, num_pages * PageSize);
LimitableResource::PhysicalMemory, num_pages * PageSize);
// Apply the memory block update. // Apply the memory block update.
m_memory_block_manager.Update(std::addressof(allocator), m_heap_region_start + size, m_memory_block_manager.Update(std::addressof(allocator), m_heap_region_start + size,
@ -1784,8 +1733,7 @@ Result KPageTable::SetHeapSize(VAddr* out, size_t size) {
// Reserve memory for the heap extension. // Reserve memory for the heap extension.
KScopedResourceReservation memory_reservation( KScopedResourceReservation memory_reservation(
m_system.Kernel().CurrentProcess()->GetResourceLimit(), LimitableResource::PhysicalMemory, m_resource_limit, LimitableResource::PhysicalMemory, allocation_size);
allocation_size);
R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached); R_UNLESS(memory_reservation.Succeeded(), ResultLimitReached);
// Allocate pages for the heap extension. // Allocate pages for the heap extension.
@ -1873,7 +1821,7 @@ ResultVal<VAddr> KPageTable::AllocateAndMapMemory(size_t needed_num_pages, size_
R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr)); R_TRY(Operate(addr, needed_num_pages, perm, OperationType::Map, map_addr));
} else { } else {
KPageGroup page_group; KPageGroup page_group;
R_TRY(m_system.Kernel().MemoryManager().AllocateAndOpenForProcess( R_TRY(m_system.Kernel().MemoryManager().AllocateForProcess(
&page_group, needed_num_pages, &page_group, needed_num_pages,
KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0)); KMemoryManager::EncodeOption(m_memory_pool, m_allocation_option), 0, 0));
R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup)); R_TRY(Operate(addr, needed_num_pages, page_group, OperationType::MapGroup));
@ -1887,8 +1835,9 @@ ResultVal<VAddr> KPageTable::AllocateAndMapMemory(size_t needed_num_pages, size_
return addr; return addr;
} }
Result KPageTable::LockForMapDeviceAddressSpace(VAddr address, size_t size, KMemoryPermission perm, Result KPageTable::LockForMapDeviceAddressSpace(bool* out_is_io, VAddr address, size_t size,
bool is_aligned) { KMemoryPermission perm, bool is_aligned,
bool check_heap) {
// Lightly validate the range before doing anything else. // Lightly validate the range before doing anything else.
const size_t num_pages = size / PageSize; const size_t num_pages = size / PageSize;
R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory);
@ -1898,15 +1847,18 @@ Result KPageTable::LockForMapDeviceAddressSpace(VAddr address, size_t size, KMem
// Check the memory state. // Check the memory state.
const auto test_state = const auto test_state =
(is_aligned ? KMemoryState::FlagCanAlignedDeviceMap : KMemoryState::FlagCanDeviceMap); (is_aligned ? KMemoryState::FlagCanAlignedDeviceMap : KMemoryState::FlagCanDeviceMap) |
(check_heap ? KMemoryState::FlagReferenceCounted : KMemoryState::None);
size_t num_allocator_blocks; size_t num_allocator_blocks;
R_TRY(this->CheckMemoryState(std::addressof(num_allocator_blocks), address, size, test_state, KMemoryState old_state;
R_TRY(this->CheckMemoryState(std::addressof(old_state), nullptr, nullptr,
std::addressof(num_allocator_blocks), address, size, test_state,
test_state, perm, perm, test_state, perm, perm,
KMemoryAttribute::IpcLocked | KMemoryAttribute::Locked, KMemoryAttribute::IpcLocked | KMemoryAttribute::Locked,
KMemoryAttribute::None, KMemoryAttribute::DeviceShared)); KMemoryAttribute::None, KMemoryAttribute::DeviceShared));
// Create an update allocator. // Create an update allocator.
Result allocator_result{ResultSuccess}; Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
m_memory_block_slab_manager, num_allocator_blocks); m_memory_block_slab_manager, num_allocator_blocks);
R_TRY(allocator_result); R_TRY(allocator_result);
@ -1915,10 +1867,13 @@ Result KPageTable::LockForMapDeviceAddressSpace(VAddr address, size_t size, KMem
m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages, m_memory_block_manager.UpdateLock(std::addressof(allocator), address, num_pages,
&KMemoryBlock::ShareToDevice, KMemoryPermission::None); &KMemoryBlock::ShareToDevice, KMemoryPermission::None);
// Set whether the locked memory was io.
*out_is_io = old_state == KMemoryState::Io;
R_SUCCEED(); R_SUCCEED();
} }
Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size) { Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size, bool check_heap) {
// Lightly validate the range before doing anything else. // Lightly validate the range before doing anything else.
const size_t num_pages = size / PageSize; const size_t num_pages = size / PageSize;
R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory); R_UNLESS(this->Contains(address, size), ResultInvalidCurrentMemory);
@ -1927,16 +1882,16 @@ Result KPageTable::LockForUnmapDeviceAddressSpace(VAddr address, size_t size) {
KScopedLightLock lk(m_general_lock); KScopedLightLock lk(m_general_lock);
// Check the memory state. // Check the memory state.
const auto test_state = KMemoryState::FlagCanDeviceMap |
(check_heap ? KMemoryState::FlagReferenceCounted : KMemoryState::None);
size_t num_allocator_blocks; size_t num_allocator_blocks;
R_TRY(this->CheckMemoryStateContiguous( R_TRY(this->CheckMemoryStateContiguous(
std::addressof(num_allocator_blocks), address, size, std::addressof(num_allocator_blocks), address, size, test_state, test_state,
KMemoryState::FlagReferenceCounted | KMemoryState::FlagCanDeviceMap,
KMemoryState::FlagReferenceCounted | KMemoryState::FlagCanDeviceMap,
KMemoryPermission::None, KMemoryPermission::None, KMemoryPermission::None, KMemoryPermission::None,
KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked, KMemoryAttribute::DeviceShared)); KMemoryAttribute::DeviceShared | KMemoryAttribute::Locked, KMemoryAttribute::DeviceShared));
// Create an update allocator. // Create an update allocator.
Result allocator_result{ResultSuccess}; Result allocator_result;
KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result), KMemoryBlockManagerUpdateAllocator allocator(std::addressof(allocator_result),
m_memory_block_slab_manager, num_allocator_blocks); m_memory_block_slab_manager, num_allocator_blocks);
R_TRY(allocator_result); R_TRY(allocator_result);
@ -2070,6 +2025,10 @@ Result KPageTable::Operate(VAddr addr, size_t num_pages, KMemoryPermission perm,
m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, num_pages * PageSize, map_addr); m_system.Memory().MapMemoryRegion(*m_page_table_impl, addr, num_pages * PageSize, map_addr);
break; break;
} }
case OperationType::Separate: {
// HACK: Unimplemented.
break;
}
case OperationType::ChangePermissions: case OperationType::ChangePermissions:
case OperationType::ChangePermissionsAndRefresh: case OperationType::ChangePermissionsAndRefresh:
break; break;
@ -2105,6 +2064,7 @@ VAddr KPageTable::GetRegionAddress(KMemoryState state) const {
case KMemoryState::GeneratedCode: case KMemoryState::GeneratedCode:
case KMemoryState::CodeOut: case KMemoryState::CodeOut:
case KMemoryState::Coverage: case KMemoryState::Coverage:
case KMemoryState::Insecure:
return m_alias_code_region_start; return m_alias_code_region_start;
case KMemoryState::Code: case KMemoryState::Code:
case KMemoryState::CodeData: case KMemoryState::CodeData:
@ -2140,6 +2100,7 @@ size_t KPageTable::GetRegionSize(KMemoryState state) const {
case KMemoryState::GeneratedCode: case KMemoryState::GeneratedCode:
case KMemoryState::CodeOut: case KMemoryState::CodeOut:
case KMemoryState::Coverage: case KMemoryState::Coverage:
case KMemoryState::Insecure:
return m_alias_code_region_end - m_alias_code_region_start; return m_alias_code_region_end - m_alias_code_region_start;
case KMemoryState::Code: case KMemoryState::Code:
case KMemoryState::CodeData: case KMemoryState::CodeData:
@ -2181,6 +2142,7 @@ bool KPageTable::CanContain(VAddr addr, size_t size, KMemoryState state) const {
case KMemoryState::GeneratedCode: case KMemoryState::GeneratedCode:
case KMemoryState::CodeOut: case KMemoryState::CodeOut:
case KMemoryState::Coverage: case KMemoryState::Coverage:
case KMemoryState::Insecure:
return is_in_region && !is_in_heap && !is_in_alias; return is_in_region && !is_in_heap && !is_in_alias;
case KMemoryState::Normal: case KMemoryState::Normal:
ASSERT(is_in_heap); ASSERT(is_in_heap);

6
src/core/hle/service/nvdrv/devices/nvmap.cpp
View file

@ -126,10 +126,12 @@ NvResult nvmap::IocAlloc(const std::vector<u8>& input, std::vector<u8>& output)
LOG_CRITICAL(Service_NVDRV, "Object failed to allocate, handle={:08X}", params.handle); LOG_CRITICAL(Service_NVDRV, "Object failed to allocate, handle={:08X}", params.handle);
return result; return result;
} }
bool is_out_io{};
ASSERT(system.CurrentProcess() ASSERT(system.CurrentProcess()
->PageTable() ->PageTable()
.LockForMapDeviceAddressSpace(handle_description->address, handle_description->size, .LockForMapDeviceAddressSpace(&is_out_io, handle_description->address,
Kernel::KMemoryPermission::None, true) handle_description->size,
Kernel::KMemoryPermission::None, true, false)
.IsSuccess()); .IsSuccess());
std::memcpy(output.data(), &params, sizeof(params)); std::memcpy(output.data(), &params, sizeof(params));
return result; return result;