core: hle: kernel: Update init_slab_heap, use device memory, and add KThreadLocalPage and KPageBuffer.

- Refreshes our slab initialization code to latest known behavior.
- Moves all guest kernel slabs into emulated device memory.
- Adds KThreadLocalPage and KPageBuffer, which we will use for accurate TLS management.
This commit is contained in:
bunnei 2022-03-11 16:11:57 -08:00
parent 91819726b1
commit a25cd4bb4b
4 changed files with 92 additions and 55 deletions

View File

@ -7,19 +7,23 @@
#include "common/common_funcs.h" #include "common/common_funcs.h"
#include "common/common_types.h" #include "common/common_types.h"
#include "core/core.h" #include "core/core.h"
#include "core/device_memory.h"
#include "core/hardware_properties.h" #include "core/hardware_properties.h"
#include "core/hle/kernel/init/init_slab_setup.h" #include "core/hle/kernel/init/init_slab_setup.h"
#include "core/hle/kernel/k_code_memory.h" #include "core/hle/kernel/k_code_memory.h"
#include "core/hle/kernel/k_event.h" #include "core/hle/kernel/k_event.h"
#include "core/hle/kernel/k_memory_layout.h" #include "core/hle/kernel/k_memory_layout.h"
#include "core/hle/kernel/k_memory_manager.h" #include "core/hle/kernel/k_memory_manager.h"
#include "core/hle/kernel/k_page_buffer.h"
#include "core/hle/kernel/k_port.h" #include "core/hle/kernel/k_port.h"
#include "core/hle/kernel/k_process.h" #include "core/hle/kernel/k_process.h"
#include "core/hle/kernel/k_resource_limit.h" #include "core/hle/kernel/k_resource_limit.h"
#include "core/hle/kernel/k_session.h" #include "core/hle/kernel/k_session.h"
#include "core/hle/kernel/k_shared_memory.h" #include "core/hle/kernel/k_shared_memory.h"
#include "core/hle/kernel/k_shared_memory_info.h"
#include "core/hle/kernel/k_system_control.h" #include "core/hle/kernel/k_system_control.h"
#include "core/hle/kernel/k_thread.h" #include "core/hle/kernel/k_thread.h"
#include "core/hle/kernel/k_thread_local_page.h"
#include "core/hle/kernel/k_transfer_memory.h" #include "core/hle/kernel/k_transfer_memory.h"
namespace Kernel::Init { namespace Kernel::Init {
@ -32,9 +36,13 @@ namespace Kernel::Init {
HANDLER(KEvent, (SLAB_COUNT(KEvent)), ##__VA_ARGS__) \ HANDLER(KEvent, (SLAB_COUNT(KEvent)), ##__VA_ARGS__) \
HANDLER(KPort, (SLAB_COUNT(KPort)), ##__VA_ARGS__) \ HANDLER(KPort, (SLAB_COUNT(KPort)), ##__VA_ARGS__) \
HANDLER(KSharedMemory, (SLAB_COUNT(KSharedMemory)), ##__VA_ARGS__) \ HANDLER(KSharedMemory, (SLAB_COUNT(KSharedMemory)), ##__VA_ARGS__) \
HANDLER(KSharedMemoryInfo, (SLAB_COUNT(KSharedMemory) * 8), ##__VA_ARGS__) \
HANDLER(KTransferMemory, (SLAB_COUNT(KTransferMemory)), ##__VA_ARGS__) \ HANDLER(KTransferMemory, (SLAB_COUNT(KTransferMemory)), ##__VA_ARGS__) \
HANDLER(KCodeMemory, (SLAB_COUNT(KCodeMemory)), ##__VA_ARGS__) \ HANDLER(KCodeMemory, (SLAB_COUNT(KCodeMemory)), ##__VA_ARGS__) \
HANDLER(KSession, (SLAB_COUNT(KSession)), ##__VA_ARGS__) \ HANDLER(KSession, (SLAB_COUNT(KSession)), ##__VA_ARGS__) \
HANDLER(KThreadLocalPage, \
(SLAB_COUNT(KProcess) + (SLAB_COUNT(KProcess) + SLAB_COUNT(KThread)) / 8), \
##__VA_ARGS__) \
HANDLER(KResourceLimit, (SLAB_COUNT(KResourceLimit)), ##__VA_ARGS__) HANDLER(KResourceLimit, (SLAB_COUNT(KResourceLimit)), ##__VA_ARGS__)
namespace { namespace {
@ -50,38 +58,46 @@ enum KSlabType : u32 {
// Constexpr counts. // Constexpr counts.
constexpr size_t SlabCountKProcess = 80; constexpr size_t SlabCountKProcess = 80;
constexpr size_t SlabCountKThread = 800; constexpr size_t SlabCountKThread = 800;
constexpr size_t SlabCountKEvent = 700; constexpr size_t SlabCountKEvent = 900;
constexpr size_t SlabCountKInterruptEvent = 100; constexpr size_t SlabCountKInterruptEvent = 100;
constexpr size_t SlabCountKPort = 256 + 0x20; // Extra 0x20 ports over Nintendo for homebrew. constexpr size_t SlabCountKPort = 384;
constexpr size_t SlabCountKSharedMemory = 80; constexpr size_t SlabCountKSharedMemory = 80;
constexpr size_t SlabCountKTransferMemory = 200; constexpr size_t SlabCountKTransferMemory = 200;
constexpr size_t SlabCountKCodeMemory = 10; constexpr size_t SlabCountKCodeMemory = 10;
constexpr size_t SlabCountKDeviceAddressSpace = 300; constexpr size_t SlabCountKDeviceAddressSpace = 300;
constexpr size_t SlabCountKSession = 933; constexpr size_t SlabCountKSession = 1133;
constexpr size_t SlabCountKLightSession = 100; constexpr size_t SlabCountKLightSession = 100;
constexpr size_t SlabCountKObjectName = 7; constexpr size_t SlabCountKObjectName = 7;
constexpr size_t SlabCountKResourceLimit = 5; constexpr size_t SlabCountKResourceLimit = 5;
constexpr size_t SlabCountKDebug = Core::Hardware::NUM_CPU_CORES; constexpr size_t SlabCountKDebug = Core::Hardware::NUM_CPU_CORES;
constexpr size_t SlabCountKAlpha = 1; constexpr size_t SlabCountKIoPool = 1;
constexpr size_t SlabCountKBeta = 6; constexpr size_t SlabCountKIoRegion = 6;
constexpr size_t SlabCountExtraKThread = 160; constexpr size_t SlabCountExtraKThread = 160;
/// Helper function to translate from the slab virtual address to the reserved location in physical
/// memory.
static PAddr TranslateSlabAddrToPhysical(KMemoryLayout& memory_layout, VAddr slab_addr) {
slab_addr -= memory_layout.GetSlabRegionAddress();
return slab_addr + Core::DramMemoryMap::SlabHeapBase;
}
template <typename T> template <typename T>
VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAddr address, VAddr InitializeSlabHeap(Core::System& system, KMemoryLayout& memory_layout, VAddr address,
size_t num_objects) { size_t num_objects) {
// TODO(bunnei): This is just a place holder. We should initialize the appropriate KSlabHeap for
// kernel object type T with the backing kernel memory pointer once we emulate kernel memory.
const size_t size = Common::AlignUp(sizeof(T) * num_objects, alignof(void*)); const size_t size = Common::AlignUp(sizeof(T) * num_objects, alignof(void*));
VAddr start = Common::AlignUp(address, alignof(T)); VAddr start = Common::AlignUp(address, alignof(T));
// This is intentionally empty. Once KSlabHeap is fully implemented, we can replace this with // This should use the virtual memory address passed in, but currently, we do not setup the
// the pointer to emulated memory to pass along. Until then, KSlabHeap will just allocate/free // kernel virtual memory layout. Instead, we simply map these at a region of physical memory
// host memory. // that we reserve for the slab heaps.
void* backing_kernel_memory{}; // TODO(bunnei): Fix this once we support the kernel virtual memory layout.
if (size > 0) { if (size > 0) {
void* backing_kernel_memory{
system.DeviceMemory().GetPointer(TranslateSlabAddrToPhysical(memory_layout, start))};
const KMemoryRegion* region = memory_layout.FindVirtual(start + size - 1); const KMemoryRegion* region = memory_layout.FindVirtual(start + size - 1);
ASSERT(region != nullptr); ASSERT(region != nullptr);
ASSERT(region->IsDerivedFrom(KMemoryRegionType_KernelSlab)); ASSERT(region->IsDerivedFrom(KMemoryRegionType_KernelSlab));
@ -109,8 +125,8 @@ KSlabResourceCounts KSlabResourceCounts::CreateDefault() {
.num_KObjectName = SlabCountKObjectName, .num_KObjectName = SlabCountKObjectName,
.num_KResourceLimit = SlabCountKResourceLimit, .num_KResourceLimit = SlabCountKResourceLimit,
.num_KDebug = SlabCountKDebug, .num_KDebug = SlabCountKDebug,
.num_KAlpha = SlabCountKAlpha, .num_KIoPool = SlabCountKIoPool,
.num_KBeta = SlabCountKBeta, .num_KIoRegion = SlabCountKIoRegion,
}; };
} }
@ -121,6 +137,12 @@ void InitializeSlabResourceCounts(KernelCore& kernel) {
} }
} }
size_t CalculateSlabHeapGapSize() {
constexpr size_t KernelSlabHeapGapSize = 2_MiB - 296_KiB;
static_assert(KernelSlabHeapGapSize <= KernelSlabHeapGapsSizeMax);
return KernelSlabHeapGapSize;
}
size_t CalculateTotalSlabHeapSize(const KernelCore& kernel) { size_t CalculateTotalSlabHeapSize(const KernelCore& kernel) {
size_t size = 0; size_t size = 0;
@ -136,11 +158,34 @@ size_t CalculateTotalSlabHeapSize(const KernelCore& kernel) {
#undef ADD_SLAB_SIZE #undef ADD_SLAB_SIZE
// Add the reserved size. // Add the reserved size.
size += KernelSlabHeapGapsSize; size += CalculateSlabHeapGapSize();
return size; return size;
} }
void InitializeKPageBufferSlabHeap(Core::System& system) {
auto& kernel = system.Kernel();
const auto& counts = kernel.SlabResourceCounts();
const size_t num_pages =
counts.num_KProcess + counts.num_KThread + (counts.num_KProcess + counts.num_KThread) / 8;
const size_t slab_size = num_pages * PageSize;
// Reserve memory from the system resource limit.
ASSERT(kernel.GetSystemResourceLimit()->Reserve(LimitableResource::PhysicalMemory, slab_size));
// Allocate memory for the slab.
constexpr auto AllocateOption = KMemoryManager::EncodeOption(
KMemoryManager::Pool::System, KMemoryManager::Direction::FromFront);
const PAddr slab_address =
kernel.MemoryManager().AllocateAndOpenContinuous(num_pages, 1, AllocateOption);
ASSERT(slab_address != 0);
// Initialize the slabheap.
KPageBuffer::InitializeSlabHeap(kernel, system.DeviceMemory().GetPointer(slab_address),
slab_size);
}
void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) { void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
auto& kernel = system.Kernel(); auto& kernel = system.Kernel();
@ -160,13 +205,13 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
} }
// Create an array to represent the gaps between the slabs. // Create an array to represent the gaps between the slabs.
const size_t total_gap_size = KernelSlabHeapGapsSize; const size_t total_gap_size = CalculateSlabHeapGapSize();
std::array<size_t, slab_types.size()> slab_gaps; std::array<size_t, slab_types.size()> slab_gaps;
for (size_t i = 0; i < slab_gaps.size(); i++) { for (auto& slab_gap : slab_gaps) {
// Note: This is an off-by-one error from Nintendo's intention, because GenerateRandomRange // Note: This is an off-by-one error from Nintendo's intention, because GenerateRandomRange
// is inclusive. However, Nintendo also has the off-by-one error, and it's "harmless", so we // is inclusive. However, Nintendo also has the off-by-one error, and it's "harmless", so we
// will include it ourselves. // will include it ourselves.
slab_gaps[i] = KSystemControl::GenerateRandomRange(0, total_gap_size); slab_gap = KSystemControl::GenerateRandomRange(0, total_gap_size);
} }
// Sort the array, so that we can treat differences between values as offsets to the starts of // Sort the array, so that we can treat differences between values as offsets to the starts of
@ -177,13 +222,21 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
} }
} }
for (size_t i = 0; i < slab_types.size(); i++) { // Track the gaps, so that we can free them to the unused slab tree.
VAddr gap_start = address;
size_t gap_size = 0;
for (size_t i = 0; i < slab_gaps.size(); i++) {
// Add the random gap to the address. // Add the random gap to the address.
address += (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1]; const auto cur_gap = (i == 0) ? slab_gaps[0] : slab_gaps[i] - slab_gaps[i - 1];
address += cur_gap;
gap_size += cur_gap;
#define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...) \ #define INITIALIZE_SLAB_HEAP(NAME, COUNT, ...) \
case KSlabType_##NAME: \ case KSlabType_##NAME: \
if (COUNT > 0) { \
address = InitializeSlabHeap<NAME>(system, memory_layout, address, COUNT); \ address = InitializeSlabHeap<NAME>(system, memory_layout, address, COUNT); \
} \
break; break;
// Initialize the slabheap. // Initialize the slabheap.
@ -192,7 +245,13 @@ void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout) {
FOREACH_SLAB_TYPE(INITIALIZE_SLAB_HEAP) FOREACH_SLAB_TYPE(INITIALIZE_SLAB_HEAP)
// If we somehow get an invalid type, abort. // If we somehow get an invalid type, abort.
default: default:
UNREACHABLE(); UNREACHABLE_MSG("Unknown slab type: {}", slab_types[i]);
}
// If we've hit the end of a gap, free it.
if (gap_start + gap_size != address) {
gap_start = address;
gap_size = 0;
} }
} }
} }

View File

@ -32,12 +32,13 @@ struct KSlabResourceCounts {
size_t num_KObjectName; size_t num_KObjectName;
size_t num_KResourceLimit; size_t num_KResourceLimit;
size_t num_KDebug; size_t num_KDebug;
size_t num_KAlpha; size_t num_KIoPool;
size_t num_KBeta; size_t num_KIoRegion;
}; };
void InitializeSlabResourceCounts(KernelCore& kernel); void InitializeSlabResourceCounts(KernelCore& kernel);
size_t CalculateTotalSlabHeapSize(const KernelCore& kernel); size_t CalculateTotalSlabHeapSize(const KernelCore& kernel);
void InitializeKPageBufferSlabHeap(Core::System& system);
void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout); void InitializeSlabHeaps(Core::System& system, KMemoryLayout& memory_layout);
} // namespace Kernel::Init } // namespace Kernel::Init

View File

@ -76,7 +76,7 @@ struct KernelCore::Impl {
// Initialize kernel memory and resources. // Initialize kernel memory and resources.
InitializeSystemResourceLimit(kernel, system.CoreTiming()); InitializeSystemResourceLimit(kernel, system.CoreTiming());
InitializeMemoryLayout(); InitializeMemoryLayout();
InitializePageSlab(); Init::InitializeKPageBufferSlabHeap(system);
InitializeSchedulers(); InitializeSchedulers();
InitializeSuspendThreads(); InitializeSuspendThreads();
InitializePreemption(kernel); InitializePreemption(kernel);
@ -660,22 +660,6 @@ struct KernelCore::Impl {
time_phys_addr, time_size, "Time:SharedMemory"); time_phys_addr, time_size, "Time:SharedMemory");
} }
void InitializePageSlab() {
// Allocate slab heaps
user_slab_heap_pages =
std::make_unique<KSlabHeap<Page>>(KSlabHeap<Page>::AllocationType::Guest);
// TODO(ameerj): This should be derived, not hardcoded within the kernel
constexpr u64 user_slab_heap_size{0x3de000};
// Reserve slab heaps
ASSERT(
system_resource_limit->Reserve(LimitableResource::PhysicalMemory, user_slab_heap_size));
// Initialize slab heap
user_slab_heap_pages->Initialize(
system.DeviceMemory().GetPointer(Core::DramMemoryMap::SlabHeapBase),
user_slab_heap_size);
}
KClientPort* CreateNamedServicePort(std::string name) { KClientPort* CreateNamedServicePort(std::string name) {
auto search = service_interface_factory.find(name); auto search = service_interface_factory.find(name);
if (search == service_interface_factory.end()) { if (search == service_interface_factory.end()) {
@ -756,7 +740,6 @@ struct KernelCore::Impl {
// Kernel memory management // Kernel memory management
std::unique_ptr<KMemoryManager> memory_manager; std::unique_ptr<KMemoryManager> memory_manager;
std::unique_ptr<KSlabHeap<Page>> user_slab_heap_pages;
// Shared memory for services // Shared memory for services
Kernel::KSharedMemory* hid_shared_mem{}; Kernel::KSharedMemory* hid_shared_mem{};
@ -1031,14 +1014,6 @@ const KMemoryManager& KernelCore::MemoryManager() const {
return *impl->memory_manager; return *impl->memory_manager;
} }
KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() {
return *impl->user_slab_heap_pages;
}
const KSlabHeap<Page>& KernelCore::GetUserSlabHeapPages() const {
return *impl->user_slab_heap_pages;
}
Kernel::KSharedMemory& KernelCore::GetHidSharedMem() { Kernel::KSharedMemory& KernelCore::GetHidSharedMem() {
return *impl->hid_shared_mem; return *impl->hid_shared_mem;
} }

View File

@ -43,6 +43,7 @@ class KHandleTable;
class KLinkedListNode; class KLinkedListNode;
class KMemoryLayout; class KMemoryLayout;
class KMemoryManager; class KMemoryManager;
class KPageBuffer;
class KPort; class KPort;
class KProcess; class KProcess;
class KResourceLimit; class KResourceLimit;
@ -52,6 +53,7 @@ class KSession;
class KSharedMemory; class KSharedMemory;
class KSharedMemoryInfo; class KSharedMemoryInfo;
class KThread; class KThread;
class KThreadLocalPage;
class KTransferMemory; class KTransferMemory;
class KWorkerTaskManager; class KWorkerTaskManager;
class KWritableEvent; class KWritableEvent;
@ -239,12 +241,6 @@ public:
/// Gets the virtual memory manager for the kernel. /// Gets the virtual memory manager for the kernel.
const KMemoryManager& MemoryManager() const; const KMemoryManager& MemoryManager() const;
/// Gets the slab heap allocated for user space pages.
KSlabHeap<Page>& GetUserSlabHeapPages();
/// Gets the slab heap allocated for user space pages.
const KSlabHeap<Page>& GetUserSlabHeapPages() const;
/// Gets the shared memory object for HID services. /// Gets the shared memory object for HID services.
Kernel::KSharedMemory& GetHidSharedMem(); Kernel::KSharedMemory& GetHidSharedMem();
@ -336,6 +332,10 @@ public:
return slab_heap_container->writeable_event; return slab_heap_container->writeable_event;
} else if constexpr (std::is_same_v<T, KCodeMemory>) { } else if constexpr (std::is_same_v<T, KCodeMemory>) {
return slab_heap_container->code_memory; return slab_heap_container->code_memory;
} else if constexpr (std::is_same_v<T, KPageBuffer>) {
return slab_heap_container->page_buffer;
} else if constexpr (std::is_same_v<T, KThreadLocalPage>) {
return slab_heap_container->thread_local_page;
} }
} }
@ -397,6 +397,8 @@ private:
KSlabHeap<KTransferMemory> transfer_memory; KSlabHeap<KTransferMemory> transfer_memory;
KSlabHeap<KWritableEvent> writeable_event; KSlabHeap<KWritableEvent> writeable_event;
KSlabHeap<KCodeMemory> code_memory; KSlabHeap<KCodeMemory> code_memory;
KSlabHeap<KPageBuffer> page_buffer;
KSlabHeap<KThreadLocalPage> thread_local_page;
}; };
std::unique_ptr<SlabHeapContainer> slab_heap_container; std::unique_ptr<SlabHeapContainer> slab_heap_container;