From 03d0fd0f628b6ad0c38a1e270668a0bd9b3758e8 Mon Sep 17 00:00:00 2001 From: Michael Scire Date: Tue, 21 Feb 2023 10:38:48 -0700 Subject: [PATCH] kern: refactor init (kill identity map, merge cpu on logic) --- .../arch/arm64/init/kern_k_init_arguments.hpp | 4 - .../arm64/init/kern_k_init_page_table.hpp | 157 ++++++++++--- .../arch/arm64/kern_assembly_offsets.h | 4 +- .../mesosphere/arch/arm64/kern_cpu.hpp | 8 + .../arch/arm64/kern_k_page_table_entry.hpp | 9 +- .../nintendo/nx/kern_k_system_control.hpp | 5 +- .../init/kern_init_arguments_select.hpp | 2 +- .../mesosphere/kern_k_memory_layout.hpp | 6 +- .../mesosphere/kern_k_system_control_base.hpp | 10 +- .../arch/arm64/kern_k_page_table_impl.cpp | 18 +- .../nintendo/nx/kern_k_sleep_manager.cpp | 53 +++-- .../nintendo/nx/kern_k_sleep_manager.hpp | 4 +- .../nintendo/nx/kern_k_sleep_manager_asm.s | 218 ++---------------- .../nintendo/nx/kern_k_system_control.cpp | 2 +- .../board/nintendo/nx/kern_lps_driver.cpp | 4 +- .../board/nintendo/nx/kern_lps_driver.hpp | 2 +- .../source/kern_k_system_control_base.cpp | 25 +- 17 files changed, 241 insertions(+), 290 deletions(-) diff --git a/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp b/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp index 49475051..94206bfa 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp +++ b/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp @@ -29,8 +29,6 @@ namespace ams::kern::init { u64 sp; u64 entrypoint; u64 argument; - u64 setup_function; - u64 exception_stack; }; static_assert(alignof(KInitArguments) == util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE)); static_assert(sizeof(KInitArguments) == std::max(INIT_ARGUMENTS_SIZE, util::CeilingPowerOfTwo(INIT_ARGUMENTS_SIZE))); @@ -45,7 +43,5 @@ namespace ams::kern::init { static_assert(AMS_OFFSETOF(KInitArguments, sp) == INIT_ARGUMENTS_SP); static_assert(AMS_OFFSETOF(KInitArguments, entrypoint) == INIT_ARGUMENTS_ENTRYPOINT); static_assert(AMS_OFFSETOF(KInitArguments, argument) == INIT_ARGUMENTS_ARGUMENT); - static_assert(AMS_OFFSETOF(KInitArguments, setup_function) == INIT_ARGUMENTS_SETUP_FUNCTION); - static_assert(AMS_OFFSETOF(KInitArguments, exception_stack) == INIT_ARGUMENTS_EXCEPTION_STACK); } \ No newline at end of file diff --git a/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp b/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp index e0613fbd..f47407c8 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp +++ b/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_page_table.hpp @@ -23,17 +23,6 @@ namespace ams::kern::arch::arm64::init { - inline void ClearPhysicalMemory(KPhysicalAddress address, size_t size) { - MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(size, sizeof(u64))); - - /* This Physical Address -> void * conversion is valid, because this is init page table code. */ - /* The MMU is necessarily not yet turned on, if we are creating an initial page table. */ - volatile u64 *ptr = reinterpret_cast(GetInteger(address)); - for (size_t i = 0; i < size / sizeof(u64); ++i) { - ptr[i] = 0; - } - } - /* NOTE: Nintendo uses virtual functions, rather than a concept + template. */ template concept IsInitialPageAllocator = requires (T &t, KPhysicalAddress phys_addr, size_t size) { @@ -41,25 +30,23 @@ namespace ams::kern::arch::arm64::init { { t.Free(phys_addr, size) } -> std::same_as; }; - template - class KInitialPageTableTemplate { - public: - using PageAllocator = _PageAllocator; + class KInitialPageTable { private: KPhysicalAddress m_l1_tables[2]; u32 m_num_entries[2]; public: - KInitialPageTableTemplate(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) { + template + KInitialPageTable(KVirtualAddress start_address, KVirtualAddress end_address, PageAllocator &allocator) { /* Set tables. */ - m_l1_tables[0] = AllocateNewPageTable(allocator); - m_l1_tables[1] = AllocateNewPageTable(allocator); + m_l1_tables[0] = AllocateNewPageTable(allocator, 0); + m_l1_tables[1] = AllocateNewPageTable(allocator, 0); /* Set counts. */ m_num_entries[0] = MaxPageTableEntries; m_num_entries[1] = ((end_address / L1BlockSize) & (MaxPageTableEntries - 1)) - ((start_address / L1BlockSize) & (MaxPageTableEntries - 1)) + 1; } - KInitialPageTableTemplate() { + KInitialPageTable() { /* Set tables. */ m_l1_tables[0] = util::AlignDown(cpu::GetTtbr0El1(), PageSize); m_l1_tables[1] = util::AlignDown(cpu::GetTtbr1El1(), PageSize); @@ -82,30 +69,35 @@ namespace ams::kern::arch::arm64::init { return GetInteger(m_l1_tables[1]); } private: - constexpr ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KVirtualAddress address) const { + constexpr ALWAYS_INLINE L1PageTableEntry *GetL1Entry(KVirtualAddress address, u64 phys_to_virt_offset = 0) const { const size_t index = (GetInteger(address) >> (BITSIZEOF(address) - 1)) & 1; - L1PageTableEntry *l1_table = reinterpret_cast(GetInteger(m_l1_tables[index])); + L1PageTableEntry *l1_table = reinterpret_cast(GetInteger(m_l1_tables[index]) + phys_to_virt_offset); return l1_table + ((GetInteger(address) / L1BlockSize) & (m_num_entries[index] - 1)); } - static constexpr ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KVirtualAddress address) { - L2PageTableEntry *l2_table = reinterpret_cast(GetInteger(entry->GetTable())); + static constexpr ALWAYS_INLINE L2PageTableEntry *GetL2Entry(const L1PageTableEntry *entry, KVirtualAddress address, u64 phys_to_virt_offset = 0) { + L2PageTableEntry *l2_table = reinterpret_cast(GetInteger(entry->GetTable()) + phys_to_virt_offset); return l2_table + ((GetInteger(address) / L2BlockSize) & (MaxPageTableEntries - 1)); } - static constexpr ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KVirtualAddress address) { - L3PageTableEntry *l3_table = reinterpret_cast(GetInteger(entry->GetTable())); + static constexpr ALWAYS_INLINE L3PageTableEntry *GetL3Entry(const L2PageTableEntry *entry, KVirtualAddress address, u64 phys_to_virt_offset = 0) { + L3PageTableEntry *l3_table = reinterpret_cast(GetInteger(entry->GetTable()) + phys_to_virt_offset); return l3_table + ((GetInteger(address) / L3BlockSize) & (MaxPageTableEntries - 1)); } - static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator) { + template + static ALWAYS_INLINE KPhysicalAddress AllocateNewPageTable(PageAllocator &allocator, u64 phys_to_virt_offset) { auto address = allocator.Allocate(PageSize); - ClearNewPageTable(address); + ClearNewPageTable(address, phys_to_virt_offset); return address; } - static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address) { - ClearPhysicalMemory(address, PageSize); + static ALWAYS_INLINE void ClearNewPageTable(KPhysicalAddress address, u64 phys_to_virt_offset) { + /* Convert to a deferenceable address, and clear. */ + volatile u64 *ptr = reinterpret_cast(GetInteger(address) + phys_to_virt_offset); + for (size_t i = 0; i < PageSize / sizeof(u64); ++i) { + ptr[i] = 0; + } } public: static consteval size_t GetMaximumOverheadSize(size_t size) { @@ -327,7 +319,8 @@ namespace ams::kern::arch::arm64::init { } } public: - void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator) { + template + void NOINLINE Map(KVirtualAddress virt_addr, size_t size, KPhysicalAddress phys_addr, const PageTableEntry &attr, PageAllocator &allocator, u64 phys_to_virt_offset) { /* Ensure that addresses and sizes are page aligned. */ MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(virt_addr), PageSize)); MESOSPHERE_INIT_ABORT_UNLESS(util::IsAligned(GetInteger(phys_addr), PageSize)); @@ -335,7 +328,7 @@ namespace ams::kern::arch::arm64::init { /* Iteratively map pages until the requested region is mapped. */ while (size > 0) { - L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); + L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr, phys_to_virt_offset); /* Can we make an L1 block? */ if (util::IsAligned(GetInteger(virt_addr), L1BlockSize) && util::IsAligned(GetInteger(phys_addr), L1BlockSize) && size >= L1BlockSize) { @@ -349,12 +342,12 @@ namespace ams::kern::arch::arm64::init { /* If we don't already have an L2 table, we need to make a new one. */ if (!l1_entry->IsTable()) { - KPhysicalAddress new_table = AllocateNewPageTable(allocator); + KPhysicalAddress new_table = AllocateNewPageTable(allocator, phys_to_virt_offset); cpu::DataSynchronizationBarrierInnerShareable(); *l1_entry = L1PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever()); } - L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr); + L2PageTableEntry *l2_entry = GetL2Entry(l1_entry, virt_addr, phys_to_virt_offset); /* Can we make a contiguous L2 block? */ if (util::IsAligned(GetInteger(virt_addr), L2ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L2ContiguousBlockSize) && size >= L2ContiguousBlockSize) { @@ -380,12 +373,12 @@ namespace ams::kern::arch::arm64::init { /* If we don't already have an L3 table, we need to make a new one. */ if (!l2_entry->IsTable()) { - KPhysicalAddress new_table = AllocateNewPageTable(allocator); + KPhysicalAddress new_table = AllocateNewPageTable(allocator, phys_to_virt_offset); cpu::DataSynchronizationBarrierInnerShareable(); *l2_entry = L2PageTableEntry(PageTableEntry::TableTag{}, new_table, attr.IsPrivilegedExecuteNever()); } - L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr); + L3PageTableEntry *l3_entry = GetL3Entry(l2_entry, virt_addr, phys_to_virt_offset); /* Can we make a contiguous L3 block? */ if (util::IsAligned(GetInteger(virt_addr), L3ContiguousBlockSize) && util::IsAligned(GetInteger(phys_addr), L3ContiguousBlockSize) && size >= L3ContiguousBlockSize) { @@ -410,6 +403,98 @@ namespace ams::kern::arch::arm64::init { cpu::DataSynchronizationBarrierInnerShareable(); } + void UnmapTtbr0Entries(u64 phys_to_virt_offset) { + /* Ensure data consistency before we unmap. */ + cpu::DataSynchronizationBarrierInnerShareable(); + + /* Define helper, as we only want to clear non-nGnRE pages. */ + constexpr auto ShouldUnmap = [](const PageTableEntry *entry) ALWAYS_INLINE_LAMBDA -> bool { + return entry->GetPageAttribute() != PageTableEntry::PageAttribute_Device_nGnRE; + }; + + /* Iterate all L1 entries. */ + L1PageTableEntry * const l1_table = reinterpret_cast(GetInteger(m_l1_tables[0]) + phys_to_virt_offset); + for (size_t l1_index = 0; l1_index < m_num_entries[0]; l1_index++) { + /* Get L1 entry. */ + L1PageTableEntry * const l1_entry = l1_table + l1_index; + if (l1_entry->IsBlock()) { + /* Unmap the L1 entry, if we should. */ + if (ShouldUnmap(l1_entry)) { + *static_cast(l1_entry) = InvalidPageTableEntry; + } + } else if (l1_entry->IsTable()) { + /* Get the L2 table. */ + L2PageTableEntry * const l2_table = reinterpret_cast(GetInteger(l1_entry->GetTable()) + phys_to_virt_offset); + + /* Unmap all L2 entries, as relevant. */ + size_t remaining_l2_entries = 0; + for (size_t l2_index = 0; l2_index < MaxPageTableEntries; ++l2_index) { + /* Get L2 entry. */ + L2PageTableEntry * const l2_entry = l2_table + l2_index; + if (l2_entry->IsBlock()) { + const size_t num_to_clear = (l2_entry->IsContiguous() ? L2ContiguousBlockSize : L2BlockSize) / L2BlockSize; + + if (ShouldUnmap(l2_entry)) { + for (size_t i = 0; i < num_to_clear; ++i) { + static_cast(l2_entry)[i] = InvalidPageTableEntry; + } + } else { + remaining_l2_entries += num_to_clear; + } + + l2_index = l2_index + num_to_clear - 1; + } else if (l2_entry->IsTable()) { + /* Get the L3 table. */ + L3PageTableEntry * const l3_table = reinterpret_cast(GetInteger(l2_entry->GetTable()) + phys_to_virt_offset); + + /* Unmap all L3 entries, as relevant. */ + size_t remaining_l3_entries = 0; + for (size_t l3_index = 0; l3_index < MaxPageTableEntries; ++l3_index) { + /* Get L3 entry. */ + if (L3PageTableEntry * const l3_entry = l3_table + l3_index; l3_entry->IsBlock()) { + const size_t num_to_clear = (l3_entry->IsContiguous() ? L3ContiguousBlockSize : L3BlockSize) / L3BlockSize; + + if (ShouldUnmap(l3_entry)) { + for (size_t i = 0; i < num_to_clear; ++i) { + static_cast(l3_entry)[i] = InvalidPageTableEntry; + } + } else { + remaining_l3_entries += num_to_clear; + } + + l3_index = l3_index + num_to_clear - 1; + } + } + + /* If we unmapped all L3 entries, clear the L2 entry. */ + if (remaining_l3_entries == 0) { + *static_cast(l2_entry) = InvalidPageTableEntry; + + /* Invalidate the entire tlb. */ + cpu::DataSynchronizationBarrierInnerShareable(); + cpu::InvalidateEntireTlb(); + } else { + remaining_l2_entries++; + } + } + } + + /* If we unmapped all L2 entries, clear the L1 entry. */ + if (remaining_l2_entries == 0) { + *static_cast(l1_entry) = InvalidPageTableEntry; + + /* Invalidate the entire tlb. */ + cpu::DataSynchronizationBarrierInnerShareable(); + cpu::InvalidateEntireTlb(); + } + } + } + + /* Invalidate the entire tlb. */ + cpu::DataSynchronizationBarrierInnerShareable(); + cpu::InvalidateEntireTlb(); + } + KPhysicalAddress GetPhysicalAddress(KVirtualAddress virt_addr) const { /* Get the L1 entry. */ const L1PageTableEntry *l1_entry = this->GetL1Entry(virt_addr); @@ -861,6 +946,4 @@ namespace ams::kern::arch::arm64::init { }; static_assert(IsInitialPageAllocator); - using KInitialPageTable = KInitialPageTableTemplate; - } diff --git a/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h b/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h index 3ca96a71..b585f44e 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h +++ b/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h @@ -246,7 +246,7 @@ #define THREAD_LOCAL_REGION_SIZE 0x200 /* ams::kern::init::KInitArguments, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp */ -#define INIT_ARGUMENTS_SIZE 0x60 +#define INIT_ARGUMENTS_SIZE 0x50 #define INIT_ARGUMENTS_TTBR0 0x00 #define INIT_ARGUMENTS_TTBR1 0x08 #define INIT_ARGUMENTS_TCR 0x10 @@ -257,8 +257,6 @@ #define INIT_ARGUMENTS_SP 0x38 #define INIT_ARGUMENTS_ENTRYPOINT 0x40 #define INIT_ARGUMENTS_ARGUMENT 0x48 -#define INIT_ARGUMENTS_SETUP_FUNCTION 0x50 -#define INIT_ARGUMENTS_EXCEPTION_STACK 0x58 /* ams::kern::KScheduler (::SchedulingState), https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_scheduler.hpp */ /* NOTE: Due to constraints on ldarb relative offsets, KSCHEDULER_NEEDS_SCHEDULING cannot trivially be changed, and will require assembly edits. */ diff --git a/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp b/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp index 164042d5..9f0494ec 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp +++ b/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp @@ -189,6 +189,14 @@ namespace ams::kern::arch::arm64::cpu { return (par >> (BITSIZEOF(par) - BITSIZEOF(u8))) == 0xFF; } + ALWAYS_INLINE void StoreDataCacheForInitArguments(const void *addr, size_t size) { + const uintptr_t start = util::AlignDown(reinterpret_cast(addr), DataCacheLineSize); + for (size_t stored = 0; stored < size; stored += cpu::DataCacheLineSize) { + __asm__ __volatile__("dc cvac, %[cur]" :: [cur]"r"(start + stored) : "memory"); + } + DataSynchronizationBarrier(); + } + /* Synchronization helpers. */ NOINLINE void SynchronizeAllCores(); void SynchronizeCores(u64 core_mask); diff --git a/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp b/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp index ce0cc921..93925e3f 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp +++ b/libmesosphere/include/mesosphere/arch/arm64/kern_k_page_table_entry.hpp @@ -161,9 +161,12 @@ namespace ams::kern::arch::arm64 { constexpr ALWAYS_INLINE bool IsPrivilegedExecuteNever() const { return this->GetBits(53, 1) != 0; } constexpr ALWAYS_INLINE bool IsContiguous() const { return this->GetBits(52, 1) != 0; } constexpr ALWAYS_INLINE bool IsGlobal() const { return this->GetBits(11, 1) == 0; } - constexpr ALWAYS_INLINE AccessFlag GetAccessFlag() const { return static_cast(this->GetBits(10, 1)); } - constexpr ALWAYS_INLINE Shareable GetShareable() const { return static_cast(this->GetBits(8, 2)); } - constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast(this->GetBits(2, 3)); } + constexpr ALWAYS_INLINE AccessFlag GetAccessFlag() const { return static_cast(this->SelectBits(10, 1)); } + constexpr ALWAYS_INLINE Shareable GetShareable() const { return static_cast(this->SelectBits(8, 2)); } + constexpr ALWAYS_INLINE PageAttribute GetPageAttribute() const { return static_cast(this->SelectBits(2, 3)); } + constexpr ALWAYS_INLINE int GetAccessFlagInteger() const { return static_cast(this->GetBits(10, 1)); } + constexpr ALWAYS_INLINE int GetShareableInteger() const { return static_cast(this->GetBits(8, 2)); } + constexpr ALWAYS_INLINE int GetPageAttributeInteger() const { return static_cast(this->GetBits(2, 3)); } constexpr ALWAYS_INLINE bool IsReadOnly() const { return this->GetBits(7, 1) != 0; } constexpr ALWAYS_INLINE bool IsUserAccessible() const { return this->GetBits(6, 1) != 0; } constexpr ALWAYS_INLINE bool IsNonSecure() const { return this->GetBits(5, 1) != 0; } diff --git a/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp b/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp index 02021759..a0cfce5a 100644 --- a/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp +++ b/libmesosphere/include/mesosphere/board/nintendo/nx/kern_k_system_control.hpp @@ -25,12 +25,15 @@ namespace ams::kern::board::nintendo::nx { static constexpr size_t SecureAppletMemorySize = 4_MB; public: class Init : public KSystemControlBase::Init { + private: + friend class KSystemControlBase::Init; + private: + static void CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg); public: /* Initialization. */ static size_t GetRealMemorySize(); static size_t GetIntendedMemorySize(); static bool ShouldIncreaseThreadResourceLimit(); - static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg); static size_t GetApplicationPoolSize(); static size_t GetAppletPoolSize(); static size_t GetMinimumNonSecureSystemPoolSize(); diff --git a/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp b/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp index 1e34dbcc..6892d330 100644 --- a/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp +++ b/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp @@ -26,6 +26,6 @@ namespace ams::kern::init { static_assert(util::IsPowerOfTwo(alignof(KInitArguments)) && util::IsPowerOfTwo(sizeof(KInitArguments))); - KPhysicalAddress GetInitArgumentsAddress(s32 core_id); + KInitArguments *GetInitArguments(s32 core_id); } diff --git a/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index e3099198..f8ed6ad0 100644 --- a/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -183,10 +183,10 @@ namespace ams::kern { return std::make_tuple(total_size, kernel_size); } - static void InitializeLinearMemoryAddresses(KPhysicalAddress aligned_linear_phys_start, KVirtualAddress linear_virtual_start) { + static void InitializeLinearMemoryAddresses(u64 phys_to_virt_diff) { /* Set static differences. */ - s_linear_phys_to_virt_diff = GetInteger(linear_virtual_start) - GetInteger(aligned_linear_phys_start); - s_linear_virt_to_phys_diff = GetInteger(aligned_linear_phys_start) - GetInteger(linear_virtual_start); + s_linear_phys_to_virt_diff = phys_to_virt_diff; + s_linear_virt_to_phys_diff = -phys_to_virt_diff; } static void InitializeLinearMemoryRegionTrees(); diff --git a/libmesosphere/include/mesosphere/kern_k_system_control_base.hpp b/libmesosphere/include/mesosphere/kern_k_system_control_base.hpp index d169079a..095bff49 100644 --- a/libmesosphere/include/mesosphere/kern_k_system_control_base.hpp +++ b/libmesosphere/include/mesosphere/kern_k_system_control_base.hpp @@ -21,6 +21,12 @@ namespace ams::kern { struct InitialProcessBinaryLayout; + namespace init { + + struct KInitArguments; + + } + } namespace ams::kern { @@ -40,6 +46,8 @@ namespace ams::kern { static constinit inline KSpinLock s_random_lock; public: class Init { + private: + static void CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg); public: /* Initialization. */ static size_t GetRealMemorySize(); @@ -47,7 +55,7 @@ namespace ams::kern { static KPhysicalAddress GetKernelPhysicalBaseAddress(KPhysicalAddress base_address); static void GetInitialProcessBinaryLayout(InitialProcessBinaryLayout *out); static bool ShouldIncreaseThreadResourceLimit(); - static void CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg); + static void TurnOnCpu(u64 core_id, const ams::kern::init::KInitArguments *args); static size_t GetApplicationPoolSize(); static size_t GetAppletPoolSize(); static size_t GetMinimumNonSecureSystemPoolSize(); diff --git a/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp b/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp index 901a708a..95c7d7a6 100644 --- a/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp +++ b/libmesosphere/source/arch/arm64/kern_k_page_table_impl.cpp @@ -353,12 +353,12 @@ namespace ams::kern::arch::arm64 { l1_entry->IsPrivilegedExecuteNever(), l1_entry->IsContiguous(), !l1_entry->IsGlobal(), - static_cast(l1_entry->GetAccessFlag()), - static_cast(l1_entry->GetShareable()), + static_cast(l1_entry->GetAccessFlagInteger()), + static_cast(l1_entry->GetShareableInteger()), l1_entry->IsReadOnly(), l1_entry->IsUserAccessible(), l1_entry->IsNonSecure(), - static_cast(l1_entry->GetPageAttribute()), + static_cast(l1_entry->GetPageAttributeInteger()), l1_entry->IsHeadMergeDisabled(), l1_entry->IsHeadAndBodyMergeDisabled(), l1_entry->IsTailMergeDisabled()); @@ -398,12 +398,12 @@ namespace ams::kern::arch::arm64 { l2_entry->IsPrivilegedExecuteNever(), l2_entry->IsContiguous(), !l2_entry->IsGlobal(), - static_cast(l2_entry->GetAccessFlag()), - static_cast(l2_entry->GetShareable()), + static_cast(l2_entry->GetAccessFlagInteger()), + static_cast(l2_entry->GetShareableInteger()), l2_entry->IsReadOnly(), l2_entry->IsUserAccessible(), l2_entry->IsNonSecure(), - static_cast(l2_entry->GetPageAttribute()), + static_cast(l2_entry->GetPageAttributeInteger()), l2_entry->IsHeadMergeDisabled(), l2_entry->IsHeadAndBodyMergeDisabled(), l2_entry->IsTailMergeDisabled()); @@ -443,12 +443,12 @@ namespace ams::kern::arch::arm64 { l3_entry->IsPrivilegedExecuteNever(), l3_entry->IsContiguous(), !l3_entry->IsGlobal(), - static_cast(l3_entry->GetAccessFlag()), - static_cast(l3_entry->GetShareable()), + static_cast(l3_entry->GetAccessFlagInteger()), + static_cast(l3_entry->GetShareableInteger()), l3_entry->IsReadOnly(), l3_entry->IsUserAccessible(), l3_entry->IsNonSecure(), - static_cast(l3_entry->GetPageAttribute()), + static_cast(l3_entry->GetPageAttributeInteger()), l3_entry->IsHeadMergeDisabled(), l3_entry->IsHeadAndBodyMergeDisabled(), l3_entry->IsTailMergeDisabled()); diff --git a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp index 85c01bc1..00b606ff 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp +++ b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.cpp @@ -18,6 +18,12 @@ #include "kern_secure_monitor.hpp" #include "kern_lps_driver.hpp" +namespace ams::kern::init { + + void StartOtherCore(const ams::kern::init::KInitArguments *init_args); + +} + namespace ams::kern::board::nintendo::nx { namespace { @@ -67,15 +73,10 @@ namespace ams::kern::board::nintendo::nx { constinit KLightLock g_request_lock; constinit KLightLock g_cv_lock; constinit KLightConditionVariable g_cv{util::ConstantInitialize}; - constinit KPhysicalAddress g_sleep_buffer_phys_addrs[cpu::NumCores]; alignas(1_KB) constinit u64 g_sleep_buffers[cpu::NumCores][1_KB / sizeof(u64)]; + constinit ams::kern::init::KInitArguments g_sleep_init_arguments[cpu::NumCores]; constinit SavedSystemRegisters g_sleep_system_registers[cpu::NumCores] = {}; - void PowerOnCpu(int core_id, KPhysicalAddress entry_phys_addr, u64 context_id) { - /* Request the secure monitor power on the core. */ - ::ams::kern::arch::arm64::smc::CpuOn(cpu::MultiprocessorAffinityRegisterAccessor().GetCpuOnArgument() | core_id, GetInteger(entry_phys_addr), context_id); - } - void WaitOtherCpuPowerOff() { constexpr u64 PmcPhysicalAddress = 0x7000E400; constexpr u32 PWRGATE_STATUS_CE123_MASK = ((1u << 3) - 1) << 9; @@ -473,18 +474,20 @@ namespace ams::kern::board::nintendo::nx { } } - void KSleepManager::ProcessRequests(uintptr_t buffer) { + void KSleepManager::ProcessRequests(uintptr_t sleep_buffer) { const auto target_fw = GetTargetFirmware(); const s32 core_id = GetCurrentCoreId(); - KPhysicalAddress resume_entry_phys_addr = Null; + + ams::kern::init::KInitArguments * const init_args = g_sleep_init_arguments + core_id; + KPhysicalAddress start_core_phys_addr = Null; + KPhysicalAddress init_args_phys_addr = Null; /* Get the physical addresses we'll need. */ { - MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(g_sleep_buffer_phys_addrs[core_id]), KProcessAddress(buffer))); - MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(resume_entry_phys_addr), KProcessAddress(&::ams::kern::board::nintendo::nx::KSleepManager::ResumeEntry))); - + MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(start_core_phys_addr), KProcessAddress(&::ams::kern::init::StartOtherCore))); + MESOSPHERE_ABORT_UNLESS(Kernel::GetKernelPageTable().GetPhysicalAddress(std::addressof(init_args_phys_addr), KProcessAddress(init_args))); } - const KPhysicalAddress sleep_buffer_phys_addr = g_sleep_buffer_phys_addrs[core_id]; + const u64 target_core_mask = (1ul << core_id); const bool use_legacy_lps_driver = target_fw < TargetFirmware_2_0_0; @@ -547,15 +550,29 @@ namespace ams::kern::board::nintendo::nx { /* Save the interrupt manager's state. */ Kernel::GetInterruptManager().Save(core_id); + /* Setup the initial arguments. */ + { + init_args->ttbr0 = cpu::GetTtbr0El1(); + init_args->ttbr1 = cpu::GetTtbr1El1(); + init_args->tcr = cpu::GetTcrEl1(); + init_args->mair = cpu::GetMairEl1(); + init_args->cpuactlr = cpu::GetCpuActlrEl1(); + init_args->cpuectlr = cpu::GetCpuEctlrEl1(); + init_args->sctlr = cpu::GetSctlrEl1(); + init_args->sp = 0; + init_args->entrypoint = reinterpret_cast(::ams::kern::board::nintendo::nx::KSleepManager::ResumeEntry); + init_args->argument = sleep_buffer; + } + /* Ensure that all cores get to this point before continuing. */ cpu::SynchronizeAllCores(); /* Log that the core is going to sleep. */ - MESOSPHERE_LOG("Core[%d]: Going to sleep, buffer = %010lx\n", core_id, GetInteger(sleep_buffer_phys_addr)); + MESOSPHERE_LOG("Core[%d]: Going to sleep, buffer = %010lx\n", core_id, sleep_buffer); /* If we're on a core other than zero, we can just invoke the sleep handler. */ if (core_id != 0) { - CpuSleepHandler(GetInteger(sleep_buffer_phys_addr), GetInteger(resume_entry_phys_addr)); + CpuSleepHandler(sleep_buffer, GetInteger(start_core_phys_addr), GetInteger(init_args_phys_addr)); } else { /* Wait for all other cores to be powered off. */ WaitOtherCpuPowerOff(); @@ -574,9 +591,9 @@ namespace ams::kern::board::nintendo::nx { /* Invoke the sleep handler. */ if (!use_legacy_lps_driver) { /* When not using the legacy driver, invoke directly. */ - CpuSleepHandler(GetInteger(sleep_buffer_phys_addr), GetInteger(resume_entry_phys_addr)); + CpuSleepHandler(sleep_buffer, GetInteger(start_core_phys_addr), GetInteger(init_args_phys_addr)); } else { - lps::InvokeCpuSleepHandler(GetInteger(sleep_buffer_phys_addr), GetInteger(resume_entry_phys_addr)); + lps::InvokeCpuSleepHandler(sleep_buffer, GetInteger(start_core_phys_addr), GetInteger(init_args_phys_addr)); } /* Restore the debug log state. */ @@ -586,8 +603,10 @@ namespace ams::kern::board::nintendo::nx { MESOSPHERE_LOG("Exiting SC7\n"); /* Wake up the other cores. */ + cpu::MultiprocessorAffinityRegisterAccessor mpidr; + const auto arg = mpidr.GetCpuOnArgument(); for (s32 i = 1; i < static_cast(cpu::NumCores); ++i) { - PowerOnCpu(i, resume_entry_phys_addr, GetInteger(g_sleep_buffer_phys_addrs[i])); + KSystemControl::Init::TurnOnCpu(arg | i, g_sleep_init_arguments + i); } } diff --git a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.hpp b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.hpp index 195db36d..e5d3add1 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.hpp +++ b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager.hpp @@ -22,14 +22,12 @@ namespace ams::kern::board::nintendo::nx { private: static void ResumeEntry(uintptr_t arg); - static void InvalidateDataCacheForResumeEntry(uintptr_t level); - static void ProcessRequests(uintptr_t buffer); public: static void Initialize(); static void SleepSystem(); public: - static void CpuSleepHandler(uintptr_t arg, uintptr_t entry); + static void CpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_args); }; diff --git a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s index fca5455c..1da1ac14 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s +++ b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s @@ -22,14 +22,14 @@ mov reg, #(((val) >> 0x00) & 0xFFFF); \ movk reg, #(((val) >> 0x10) & 0xFFFF), lsl#16 -/* ams::kern::board::nintendo::nx::KSleepManager::CpuSleepHandler(uintptr_t arg, uintptr_t entry) */ -.section .sleep._ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm, "ax", %progbits -.global _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm -.type _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm, %function -_ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm: +/* ams::kern::board::nintendo::nx::KSleepManager::CpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_arg) */ +.section .sleep._ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm, "ax", %progbits +.global _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm +.type _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm, %function +_ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmmm: /* Save arguments. */ - mov x16, x0 - mov x17, x1 + mov x16, x1 + mov x17, x2 /* Enable access to FPU registers. */ mrs x1, cpacr_el1 @@ -74,28 +74,8 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm: stp q28, q29, [x0], #0x20 stp q30, q31, [x0], #0x20 - /* Save cpuactlr/cpuectlr. */ - mrs x1, cpuectlr_el1 - mrs x2, cpuactlr_el1 - stp x1, x2, [x0], #0x10 - - /* Save ttbr0/ttbr1. */ - mrs x1, ttbr0_el1 - mrs x2, ttbr1_el1 - stp x1, x2, [x0], #0x10 - - /* Save tcr/mair. */ - mrs x1, tcr_el1 - mrs x2, mair_el1 - stp x1, x2, [x0], #0x10 - - /* Save sctlr/tpidr. */ - mrs x1, sctlr_el1 - mrs x2, tpidr_el1 - stp x1, x2, [x0], #0x10 - - /* Save the virtual resumption entrypoint and cntv_cval_el0. */ - adr x1, 77f + /* Save tpidr/cntv_cval_el0. */ + mrs x1, tpidr_el1 mrs x2, cntv_cval_el0 stp x1, x2, [x0], #0x10 @@ -114,8 +94,8 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm: 1: /* Suspend. */ LOAD_IMMEDIATE_32(x0, 0xC4000001) LOAD_IMMEDIATE_32(x1, 0x0201001B) - mov x2, x17 - mov x3, x16 + mov x2, x16 + mov x3, x17 smc #1 0: b 0b @@ -124,65 +104,6 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager15CpuSleepHandlerEmm: .global _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm .type _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm, %function _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm: - /* Mask interrupts. */ - msr daifset, #0xF - - /* Save the argument. */ - mov x21, x0 - - /* Check that we're at the correct exception level. */ - mrs x0, currentel - - /* Check if we're EL1. */ - cmp x0, #0x4 - b.eq 3f - - /* Check if we're EL2. */ - cmp x0, #0x8 - b.eq 2f - -1: /* We're running at EL3. */ - b 1b - -2: /* We're running at EL2. */ - b 2b - -3: /* We're running at EL1. */ - - /* Invalidate the L1 cache. */ - mov x0, #0 - bl _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm - - /* Get the current core id. */ - mrs x0, mpidr_el1 - and x0, x0, #0xFF - - /* If we're on core0, we want to invalidate the L2 cache. */ - cbnz x0, 4f - - mov x0, #1 - bl _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm - -4: /* Invalidate the L1 cache. */ - mov x0, #0 - bl _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm - - /* Invalidate the instruction cache. */ - ic ialluis - dsb sy - isb - - /* Invalidate the entire tlb. */ - tlbi vmalle1is - dsb sy - isb - - /* Switch to sp 1. */ - msr spsel, #1 - - /* Prepare to restore the saved context. */ - mov x0, x21 - /* Enable access to FPU registers. */ mrs x1, cpacr_el1 orr x1, x1, #0x100000 @@ -226,121 +147,12 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm: ldp q28, q29, [x0], #0x20 ldp q30, q31, [x0], #0x20 - /* Restore cpuactlr/cpuectlr. */ + /* Restore tpidr/cntv_cval_el0. */ ldp x1, x2, [x0], #0x10 - mrs x3, cpuectlr_el1 - cmp x1, x3 -5: b.ne 5b - mrs x3, cpuactlr_el1 - cmp x2, x3 -6: b.ne 6b - - /* Restore ttbr0/ttbr1. */ - ldp x1, x2, [x0], #0x10 - msr ttbr0_el1, x1 - msr ttbr1_el1, x2 - - /* Restore tcr/mair. */ - ldp x1, x2, [x0], #0x10 - msr tcr_el1, x1 - msr mair_el1, x2 - - /* Get sctlr, tpidr, the entrypoint, and cntv_cval_el0. */ - ldp x1, x2, [x0], #0x10 - ldp x3, x4, [x0], #0x10 - - /* Set the global context back into x18/tpidr. */ - msr tpidr_el1, x2 - msr cntv_cval_el0, x4 - dsb sy - isb - - /* Restore sctlr with the wxn bit cleared. */ - bic x2, x1, #0x80000 - msr sctlr_el1, x2 - dsb sy - isb - - /* Jump to the entrypoint. */ - br x3 - -77: /* Virtual resumption entrypoint. */ - - /* Restore sctlr. */ - msr sctlr_el1, x1 + msr tpidr_el1, x1 + msr cntv_cval_el0, x2 dsb sy isb - ret - -/* ams::kern::board::nintendo::nx::KSleepManager::InvalidateDataCacheForResumeEntry(uintptr_t level) */ -.section .sleep._ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm, "ax", %progbits -.global _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm -.type _ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm, %function -_ZN3ams4kern5board8nintendo2nx13KSleepManager33InvalidateDataCacheForResumeEntryEm: - /* cpu::DataSynchronizationBarrier(); */ - dsb sy - - /* const u64 level_sel_value = level << 1; */ - lsl x8, x0, #1 - - /* cpu::SetCsselrEl1(level_sel_value); */ - msr csselr_el1, x8 - - /* cpu::InstructionMemoryBarrier(); */ - isb - - /* CacheSizeIdAccessor ccsidr_el1; */ - mrs x13, ccsidr_el1 - - /* const int num_ways = ccsidr_el1.GetAssociativity(); */ - ubfx w10, w13, #3, #0xA - - /* const int line_size = ccsidr_el1.GetLineSize(); */ - and w11, w13, #7 - - /* const int num_sets = ccsidr_el1.GetNumberOfSets(); */ - ubfx w13, w13, #0xD, #0xF - - /* int way = 0; */ - mov w9, wzr - - /* const u64 set_shift = static_cast(line_size + 4); */ - add w11, w11, #4 - - /* const u64 way_shift = static_cast(__builtin_clz(num_ways)); */ - clz w12, w10 - - -0: /* do { */ - /* int set = 0; */ - mov w14, wzr - - /* const u64 way_value = (static_cast(way) << way_shift); */ - lsl w15, w9, w12 - -1: /* do { */ - - /* const u64 isw_value = (static_cast(set) << set_shift) | way_value | level_sel_value; */ - lsl w16, w14, w11 - orr w16, w16, w15 - sxtw x16, w16 - orr x16, x16, x8 - - /* __asm__ __volatile__("dc isw, %0" :: "r"(isw_value) : "memory"); */ - dc isw, x16 - - /* while (set <= num_sets); */ - cmp w13, w14 - add w14, w14, #1 - b.ne 1b - - /* while (way <= num_ways); */ - cmp w9, w10 - add w9, w9, #1 - b.ne 0b - - /* cpu::EnsureInstructionConsistency(); */ - dsb sy - isb + /* Return. */ ret diff --git a/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp b/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp index ea67d3b9..efe391c1 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp +++ b/libmesosphere/source/board/nintendo/nx/kern_k_system_control.cpp @@ -382,7 +382,7 @@ namespace ams::kern::board::nintendo::nx { return static_cast((value >> 32) & 0xFF); } - void KSystemControl::Init::CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) { + void KSystemControl::Init::CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg) { MESOSPHERE_INIT_ABORT_UNLESS((::ams::kern::arch::arm64::smc::CpuOn(core_id, entrypoint, arg)) == 0); } diff --git a/libmesosphere/source/board/nintendo/nx/kern_lps_driver.cpp b/libmesosphere/source/board/nintendo/nx/kern_lps_driver.cpp index bce9da12..a1f1cf2f 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_lps_driver.cpp +++ b/libmesosphere/source/board/nintendo/nx/kern_lps_driver.cpp @@ -395,7 +395,7 @@ namespace ams::kern::board::nintendo::nx::lps { R_SUCCEED(); } - void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry) { + void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_arg) { /* Verify that we're allowed to perform suspension. */ MESOSPHERE_ABORT_UNLESS(g_lps_init_done); MESOSPHERE_ABORT_UNLESS(GetCurrentCoreId() == 0); @@ -416,7 +416,7 @@ namespace ams::kern::board::nintendo::nx::lps { Read(g_pmc_address + APBDEV_PMC_SCRATCH0); /* Invoke the sleep hander. */ - KSleepManager::CpuSleepHandler(arg, entry); + KSleepManager::CpuSleepHandler(arg, entry, entry_arg); /* Disable deep power down. */ Write(g_pmc_address + APBDEV_PMC_DPD_ENABLE, 0); diff --git a/libmesosphere/source/board/nintendo/nx/kern_lps_driver.hpp b/libmesosphere/source/board/nintendo/nx/kern_lps_driver.hpp index 3d702a7a..0c69d5da 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_lps_driver.hpp +++ b/libmesosphere/source/board/nintendo/nx/kern_lps_driver.hpp @@ -22,7 +22,7 @@ namespace ams::kern::board::nintendo::nx { void Initialize(); Result EnableSuspend(bool enable); - void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry); + void InvokeCpuSleepHandler(uintptr_t arg, uintptr_t entry, uintptr_t entry_arg); void ResumeBpmpFirmware(); } diff --git a/libmesosphere/source/kern_k_system_control_base.cpp b/libmesosphere/source/kern_k_system_control_base.cpp index f04f896b..77582203 100644 --- a/libmesosphere/source/kern_k_system_control_base.cpp +++ b/libmesosphere/source/kern_k_system_control_base.cpp @@ -20,6 +20,13 @@ namespace ams::kern { + namespace init { + + /* TODO: Is this function name architecture specific? */ + void StartOtherCore(const ams::kern::init::KInitArguments *init_args); + + } + /* Initialization. */ size_t KSystemControlBase::Init::GetRealMemorySize() { return ams::kern::MainMemorySize; @@ -68,7 +75,7 @@ namespace ams::kern { return 0; } - void KSystemControlBase::Init::CpuOn(u64 core_id, uintptr_t entrypoint, uintptr_t arg) { + void KSystemControlBase::Init::CpuOnImpl(u64 core_id, uintptr_t entrypoint, uintptr_t arg) { #if defined(ATMOSPHERE_ARCH_ARM64) MESOSPHERE_INIT_ABORT_UNLESS((::ams::kern::arch::arm64::smc::CpuOn<0, false>(core_id, entrypoint, arg)) == 0); #else @@ -76,6 +83,22 @@ namespace ams::kern { #endif } + void KSystemControlBase::Init::TurnOnCpu(u64 core_id, const ams::kern::init::KInitArguments *args) { + /* Get entrypoint. */ + KPhysicalAddress entrypoint = Null; + while (!cpu::GetPhysicalAddressReadable(std::addressof(entrypoint), reinterpret_cast(::ams::kern::init::StartOtherCore), true)) { /* ... */ } + + /* Get arguments. */ + KPhysicalAddress args_addr = Null; + while (!cpu::GetPhysicalAddressReadable(std::addressof(args_addr), reinterpret_cast(args), true)) { /* ... */ } + + /* Ensure cache is correct for the initial arguments. */ + cpu::StoreDataCacheForInitArguments(args, sizeof(*args)); + + /* Turn on the cpu. */ + KSystemControl::Init::CpuOnImpl(core_id, GetInteger(entrypoint), GetInteger(args_addr)); + } + /* Randomness for Initialization. */ void KSystemControlBase::Init::GenerateRandom(u64 *dst, size_t count) { if (AMS_UNLIKELY(!s_initialized_random_generator)) {