diff --git a/libmesosphere/include/mesosphere.hpp b/libmesosphere/include/mesosphere.hpp index 66be8d6d..ee386030 100644 --- a/libmesosphere/include/mesosphere.hpp +++ b/libmesosphere/include/mesosphere.hpp @@ -48,7 +48,6 @@ #include #include #include -#include #include #include #include diff --git a/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp b/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp index b0334cb1..04abbd3d 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp +++ b/libmesosphere/include/mesosphere/arch/arm64/init/kern_k_init_arguments.hpp @@ -29,6 +29,7 @@ namespace ams::kern::init { u64 entrypoint; u64 argument; u64 setup_function; + u64 exception_stack; }; } \ No newline at end of file diff --git a/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp b/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp index 4182c519..c3c915a2 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp +++ b/libmesosphere/include/mesosphere/arch/arm64/kern_cpu.hpp @@ -220,16 +220,19 @@ namespace ams::kern::arch::arm64::cpu { DataSynchronizationBarrier(); } - ALWAYS_INLINE uintptr_t GetCoreLocalRegionAddress() { + ALWAYS_INLINE uintptr_t GetCurrentThreadPointerValue() { register uintptr_t x18 asm("x18"); __asm__ __volatile__("" : [x18]"=r"(x18)); return x18; } - ALWAYS_INLINE void SetCoreLocalRegionAddress(uintptr_t value) { + ALWAYS_INLINE void SetCurrentThreadPointerValue(uintptr_t value) { register uintptr_t x18 asm("x18") = value; __asm__ __volatile__("":: [x18]"r"(x18)); - SetTpidrEl1(value); + } + + ALWAYS_INLINE void SetExceptionThreadStackTop(uintptr_t top) { + SetTpidrEl1(top); } ALWAYS_INLINE void SwitchThreadLocalRegion(uintptr_t tlr) { diff --git a/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp b/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp index 46190b02..f6df949b 100644 --- a/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp +++ b/libmesosphere/include/mesosphere/init/kern_init_arguments_select.hpp @@ -25,6 +25,5 @@ namespace ams::kern::init { KPhysicalAddress GetInitArgumentsAddress(s32 core_id); - void SetInitArguments(s32 core_id, KPhysicalAddress address, uintptr_t arg); } diff --git a/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp b/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp deleted file mode 100644 index 9a686135..00000000 --- a/libmesosphere/include/mesosphere/kern_k_core_local_region.hpp +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright (c) 2018-2020 Atmosphère-NX - * - * This program is free software; you can redistribute it and/or modify it - * under the terms and conditions of the GNU General Public License, - * version 2, as published by the Free Software Foundation. - * - * This program is distributed in the hope it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for - * more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ -#pragma once -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ams::kern { - - struct KCoreLocalContext { - KCurrentContext current; - }; - static_assert(sizeof(KCoreLocalContext) < PageSize); - - struct KCoreLocalPage { - KCoreLocalContext context; - u8 padding[PageSize - sizeof(KCoreLocalContext)]; - }; - static_assert(sizeof(KCoreLocalPage) == PageSize); - - struct KCoreLocalRegion { - KCoreLocalPage current; - KCoreLocalPage absolute[cpu::NumCores]; - }; - static_assert(sizeof(KCoreLocalRegion) == PageSize * (1 + cpu::NumCores)); - -} diff --git a/libmesosphere/include/mesosphere/kern_k_current_context.hpp b/libmesosphere/include/mesosphere/kern_k_current_context.hpp index 9702fd46..645b7b22 100644 --- a/libmesosphere/include/mesosphere/kern_k_current_context.hpp +++ b/libmesosphere/include/mesosphere/kern_k_current_context.hpp @@ -21,80 +21,24 @@ namespace ams::kern { class KThread; class KProcess; class KScheduler; - class KInterruptTaskManager; - - struct KCurrentContext { - std::atomic current_thread; - std::atomic current_process; - KScheduler *scheduler; - KInterruptTaskManager *interrupt_task_manager; - s32 core_id; - void *exception_stack_top; - ams::svc::ThreadLocalRegion *tlr; - }; - static_assert(std::is_standard_layout::value && std::is_trivially_destructible::value); - static_assert(sizeof(KCurrentContext) <= cpu::DataCacheLineSize); - static_assert(sizeof(std::atomic) == sizeof(KThread *)); - static_assert(sizeof(std::atomic) == sizeof(KProcess *)); - - namespace impl { - - ALWAYS_INLINE KCurrentContext &GetCurrentContext() { - return *reinterpret_cast(cpu::GetCoreLocalRegionAddress()); - } - - } ALWAYS_INLINE KThread *GetCurrentThreadPointer() { - return impl::GetCurrentContext().current_thread.load(std::memory_order_relaxed); + return reinterpret_cast(cpu::GetCurrentThreadPointerValue()); } ALWAYS_INLINE KThread &GetCurrentThread() { return *GetCurrentThreadPointer(); } - ALWAYS_INLINE KProcess *GetCurrentProcessPointer() { - return impl::GetCurrentContext().current_process.load(std::memory_order_relaxed); - } - - ALWAYS_INLINE KProcess &GetCurrentProcess() { - return *GetCurrentProcessPointer(); - } - - ALWAYS_INLINE KScheduler *GetCurrentSchedulerPointer() { - return impl::GetCurrentContext().scheduler; - } - - ALWAYS_INLINE KScheduler &GetCurrentScheduler() { - return *GetCurrentSchedulerPointer(); - } - - ALWAYS_INLINE KInterruptTaskManager *GetCurrentInterruptTaskManagerPointer() { - return impl::GetCurrentContext().interrupt_task_manager; - } - - ALWAYS_INLINE KInterruptTaskManager &GetCurrentInterruptTaskManager() { - return *GetCurrentInterruptTaskManagerPointer(); - } - - ALWAYS_INLINE s32 GetCurrentCoreId() { - return impl::GetCurrentContext().core_id; - } - - ALWAYS_INLINE ams::svc::ThreadLocalRegion *GetCurrentThreadLocalRegion() { - return impl::GetCurrentContext().tlr; - } - ALWAYS_INLINE void SetCurrentThread(KThread *new_thread) { - impl::GetCurrentContext().current_thread = new_thread; + cpu::SetCurrentThreadPointerValue(reinterpret_cast(new_thread)); } - ALWAYS_INLINE void SetCurrentProcess(KProcess *new_process) { - impl::GetCurrentContext().current_process = new_process; - } + ALWAYS_INLINE KProcess *GetCurrentProcessPointer(); + ALWAYS_INLINE KProcess &GetCurrentProcess(); - ALWAYS_INLINE void SetCurrentThreadLocalRegion(void *address) { - impl::GetCurrentContext().tlr = static_cast(address); - } + ALWAYS_INLINE s32 GetCurrentCoreId(); + + ALWAYS_INLINE KScheduler &GetCurrentScheduler(); } diff --git a/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp b/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp index 9313c69b..88aa69f0 100644 --- a/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp +++ b/libmesosphere/include/mesosphere/kern_k_memory_layout.hpp @@ -134,7 +134,6 @@ namespace ams::kern { static NOINLINE KVirtualAddress GetExceptionStackTopAddress(s32 core_id) { return GetStackTopAddress(core_id, KMemoryRegionType_KernelMiscExceptionStack); } static NOINLINE KVirtualAddress GetSlabRegionAddress() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelSlab)).GetAddress(); } - static NOINLINE KVirtualAddress GetCoreLocalRegionAddress() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_CoreLocalRegion)).GetAddress(); } static NOINLINE const KMemoryRegion &GetDeviceRegion(KMemoryRegionType type) { return Dereference(GetPhysicalMemoryRegionTree().FindFirstDerived(type)); } static KPhysicalAddress GetDevicePhysicalAddress(KMemoryRegionType type) { return GetDeviceRegion(type).GetAddress(); } @@ -144,7 +143,6 @@ namespace ams::kern { static NOINLINE const KMemoryRegion &GetPageTableHeapRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelPtHeap)); } static NOINLINE const KMemoryRegion &GetKernelStackRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelStack)); } static NOINLINE const KMemoryRegion &GetTempRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_KernelTemp)); } - static NOINLINE const KMemoryRegion &GetCoreLocalRegion() { return Dereference(GetVirtualMemoryRegionTree().FindByType(KMemoryRegionType_CoreLocalRegion)); } static NOINLINE const KMemoryRegion &GetKernelTraceBufferRegion() { return Dereference(GetVirtualLinearMemoryRegionTree().FindByType(KMemoryRegionType_VirtualDramKernelTraceBuffer)); } @@ -216,7 +214,6 @@ namespace ams::kern { namespace init { /* These should be generic, regardless of board. */ - void SetupCoreLocalRegionMemoryRegions(KInitialPageTable &page_table, KInitialPageAllocator &page_allocator); void SetupPoolPartitionMemoryRegions(); /* These may be implemented in a board-specific manner. */ diff --git a/libmesosphere/include/mesosphere/kern_k_memory_region_type.hpp b/libmesosphere/include/mesosphere/kern_k_memory_region_type.hpp index b2794328..e0bfa725 100644 --- a/libmesosphere/include/mesosphere/kern_k_memory_region_type.hpp +++ b/libmesosphere/include/mesosphere/kern_k_memory_region_type.hpp @@ -156,12 +156,13 @@ namespace ams::kern { constexpr inline const auto KMemoryRegionType_None = impl::KMemoryRegionTypeValue(); - constexpr inline const auto KMemoryRegionType_Kernel = KMemoryRegionType_None.DeriveInitial(0, 2); - constexpr inline const auto KMemoryRegionType_Dram = KMemoryRegionType_None.DeriveInitial(1, 2); - constexpr inline const auto KMemoryRegionType_CoreLocalRegion = KMemoryRegionType_None.DeriveInitial(2).Finalize(); - static_assert(KMemoryRegionType_Kernel .GetValue() == 0x1); - static_assert(KMemoryRegionType_Dram .GetValue() == 0x2); - static_assert(KMemoryRegionType_CoreLocalRegion.GetValue() == 0x4); + constexpr inline const auto KMemoryRegionType_Kernel = KMemoryRegionType_None.DeriveInitial(0, 2); + constexpr inline const auto KMemoryRegionType_Dram = KMemoryRegionType_None.DeriveInitial(1, 2); + static_assert(KMemoryRegionType_Kernel.GetValue() == 0x1); + static_assert(KMemoryRegionType_Dram .GetValue() == 0x2); + + /* constexpr inline const auto KMemoryRegionType_CoreLocalRegion = KMemoryRegionType_None.DeriveInitial(2).Finalize(); */ + /* static_assert(KMemoryRegionType_CoreLocalRegion.GetValue() == 0x4); */ constexpr inline const auto KMemoryRegionType_DramKernelBase = KMemoryRegionType_Dram.DeriveSparse(0, 3, 0).SetAttribute(KMemoryRegionAttr_NoUserMap).SetAttribute(KMemoryRegionAttr_CarveoutProtected); constexpr inline const auto KMemoryRegionType_DramReservedBase = KMemoryRegionType_Dram.DeriveSparse(0, 3, 1); @@ -274,15 +275,15 @@ namespace ams::kern { /* UNUSED: .Derive(7, 0); */ constexpr inline const auto KMemoryRegionType_KernelMiscMainStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 1); constexpr inline const auto KMemoryRegionType_KernelMiscMappedDevice = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 2); - constexpr inline const auto KMemoryRegionType_KernelMiscIdleStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 3); + constexpr inline const auto KMemoryRegionType_KernelMiscExceptionStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 3); constexpr inline const auto KMemoryRegionType_KernelMiscUnknownDebug = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 4); /* UNUSED: .Derive(7, 5); */ - constexpr inline const auto KMemoryRegionType_KernelMiscExceptionStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 6); + constexpr inline const auto KMemoryRegionType_KernelMiscIdleStack = KMemoryRegionType_KernelMiscDerivedBase.Derive(7, 6); static_assert(KMemoryRegionType_KernelMiscMainStack .GetValue() == 0xB49); static_assert(KMemoryRegionType_KernelMiscMappedDevice .GetValue() == 0xD49); - static_assert(KMemoryRegionType_KernelMiscIdleStack .GetValue() == 0x1349); + static_assert(KMemoryRegionType_KernelMiscExceptionStack.GetValue() == 0x1349); static_assert(KMemoryRegionType_KernelMiscUnknownDebug .GetValue() == 0x1549); - static_assert(KMemoryRegionType_KernelMiscExceptionStack.GetValue() == 0x2349); + static_assert(KMemoryRegionType_KernelMiscIdleStack .GetValue() == 0x2349); constexpr inline const auto KMemoryRegionType_KernelTemp = KMemoryRegionType_Kernel.Advance(2).Derive(2, 0); static_assert(KMemoryRegionType_KernelTemp.GetValue() == 0x31); diff --git a/libmesosphere/include/mesosphere/kern_k_process.hpp b/libmesosphere/include/mesosphere/kern_k_process.hpp index 8eb427e1..d4887447 100644 --- a/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -361,9 +361,6 @@ namespace ams::kern { static void Switch(KProcess *cur_process, KProcess *next_process) { MESOSPHERE_UNUSED(cur_process); - /* Set the current process pointer. */ - SetCurrentProcess(next_process); - /* Update the current page table. */ if (next_process) { next_process->GetPageTable().Activate(next_process->GetProcessId()); diff --git a/libmesosphere/include/mesosphere/kern_k_scheduler.hpp b/libmesosphere/include/mesosphere/kern_k_scheduler.hpp index 885c1d07..e415afac 100644 --- a/libmesosphere/include/mesosphere/kern_k_scheduler.hpp +++ b/libmesosphere/include/mesosphere/kern_k_scheduler.hpp @@ -56,9 +56,10 @@ namespace ams::kern { KThread *prev_thread; s64 last_context_switch_time; KThread *idle_thread; + std::atomic current_thread; public: constexpr KScheduler() - : state(), is_active(false), core_id(0), prev_thread(nullptr), last_context_switch_time(0), idle_thread(nullptr) + : state(), is_active(false), core_id(0), prev_thread(nullptr), last_context_switch_time(0), idle_thread(nullptr), current_thread(nullptr) { this->state.needs_scheduling = true; this->state.interrupt_task_thread_runnable = false; @@ -96,6 +97,10 @@ namespace ams::kern { return this->prev_thread; } + ALWAYS_INLINE KThread *GetSchedulerCurrentThread() const { + return this->current_thread; + } + ALWAYS_INLINE s64 GetLastContextSwitchTime() const { return this->last_context_switch_time; } diff --git a/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libmesosphere/include/mesosphere/kern_k_thread.hpp index 357e8d08..b94f575d 100644 --- a/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -90,6 +90,7 @@ namespace ams::kern { bool is_pinned; s32 disable_count; KThreadContext *context; + KThread *cur_thread; }; static_assert(alignof(StackParameters) == 0x10); @@ -181,6 +182,7 @@ namespace ams::kern { Result wait_result; Result debug_exception_result; s32 priority{}; + s32 current_core_id{}; s32 core_id{}; s32 base_priority{}; s32 ideal_core_id{}; @@ -380,6 +382,9 @@ namespace ams::kern { constexpr s32 GetActiveCore() const { return this->core_id; } constexpr void SetActiveCore(s32 core) { this->core_id = core; } + constexpr ALWAYS_INLINE s32 GetCurrentCore() const { return this->current_core_id; } + constexpr void SetCurrentCore(s32 core) { this->current_core_id = core; } + constexpr s32 GetPriority() const { return this->priority; } constexpr void SetPriority(s32 prio) { this->priority = prio; } @@ -570,4 +575,16 @@ namespace ams::kern { return reinterpret_cast(reinterpret_cast(thread->GetKernelStackTop()) - sizeof(KThread::StackParameters) - sizeof(KExceptionContext)); } + ALWAYS_INLINE KProcess *GetCurrentProcessPointer() { + return GetCurrentThread().GetOwnerProcess(); + } + + ALWAYS_INLINE KProcess &GetCurrentProcess() { + return *GetCurrentProcessPointer(); + } + + ALWAYS_INLINE s32 GetCurrentCoreId() { + return GetCurrentThread().GetCurrentCore(); + } + } diff --git a/libmesosphere/include/mesosphere/kern_kernel.hpp b/libmesosphere/include/mesosphere/kern_kernel.hpp index 24adf4ea..6e057feb 100644 --- a/libmesosphere/include/mesosphere/kern_kernel.hpp +++ b/libmesosphere/include/mesosphere/kern_kernel.hpp @@ -19,7 +19,10 @@ #include #include #include -#include +#include +#include +#include +#include #include namespace ams::kern { @@ -77,13 +80,6 @@ namespace ams::kern { static KScheduler s_schedulers[cpu::NumCores]; static KInterruptTaskManager s_interrupt_task_managers[cpu::NumCores]; static KHardwareTimer s_hardware_timers[cpu::NumCores]; - private: - static ALWAYS_INLINE KCoreLocalContext &GetCoreLocalContext() { - return reinterpret_cast(cpu::GetCoreLocalRegionAddress())->current.context; - } - static ALWAYS_INLINE KCoreLocalContext &GetCoreLocalContext(s32 core_id) { - return reinterpret_cast(cpu::GetCoreLocalRegionAddress())->absolute[core_id].context; - } public: static NOINLINE void InitializeCoreLocalRegion(s32 core_id); static NOINLINE void InitializeMainAndIdleThreads(s32 core_id); @@ -96,10 +92,6 @@ namespace ams::kern { static KThread &GetMainThread(s32 core_id); static KThread &GetIdleThread(s32 core_id); - static ALWAYS_INLINE KCurrentContext &GetCurrentContext(s32 core_id) { - return GetCoreLocalContext(core_id).current; - } - static ALWAYS_INLINE KScheduler &GetScheduler() { return s_schedulers[GetCurrentCoreId()]; } @@ -166,4 +158,8 @@ namespace ams::kern { } }; + ALWAYS_INLINE KScheduler &GetCurrentScheduler() { + return Kernel::GetScheduler(); + } + } diff --git a/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s b/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s index db0692c6..435cafb4 100644 --- a/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s +++ b/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s @@ -32,7 +32,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: mrs x9, elr_el1 mrs x10, spsr_el1 mrs x11, tpidr_el0 - mrs x18, tpidr_el1 + ldr x18, [sp, #(0x120 + 0x28)] /* Save callee-saved registers. */ stp x19, x20, [sp, #(8 * 19)] @@ -66,7 +66,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: b.eq 3f /* Check if our disable count allows us to call SVCs. */ - ldr x10, [x18, #0x30] + mrs x10, tpidrro_el0 ldrh w10, [x10, #0x100] cbz w10, 1f @@ -138,7 +138,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: stp xzr, xzr, [sp, #(8 * 12)] stp xzr, xzr, [sp, #(8 * 14)] stp xzr, xzr, [sp, #(8 * 16)] - stp xzr, x19, [sp, #(8 * 18)] + str x19, [sp, #(8 * 19)] stp x20, x21, [sp, #(8 * 20)] stp x22, x23, [sp, #(8 * 22)] stp x24, x25, [sp, #(8 * 24)] @@ -146,7 +146,6 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: stp x28, x29, [sp, #(8 * 28)] /* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */ - mrs x18, tpidr_el1 mov x0, sp bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE @@ -246,7 +245,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: mrs x17, elr_el1 mrs x20, spsr_el1 mrs x19, tpidr_el0 - mrs x18, tpidr_el1 + ldr x18, [sp, #(0x120 + 0x28)] stp x17, x20, [sp, #(8 * 32)] str x19, [sp, #(8 * 34)] @@ -276,7 +275,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: b.eq 3f /* Check if our disable count allows us to call SVCs. */ - ldr x15, [x18, #0x30] + mrs x15, tpidrro_el0 ldrh w15, [x15, #0x100] cbz w15, 1f @@ -353,7 +352,6 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: stp xzr, xzr, [sp, #(8 * 30)] /* Call ams::kern::arch::arm64::HandleException(ams::kern::arch::arm64::KExceptionContext *) */ - mrs x18, tpidr_el1 mov x0, sp bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE diff --git a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s index 21015573..4478b241 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s +++ b/libmesosphere/source/board/nintendo/nx/kern_k_sleep_manager_asm.s @@ -249,7 +249,6 @@ _ZN3ams4kern5board8nintendo2nx13KSleepManager11ResumeEntryEm: /* Set the global context back into x18/tpidr. */ msr tpidr_el1, x2 - mov x18, x2 dsb sy isb diff --git a/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.cpp b/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.cpp index 76af4e48..628785da 100644 --- a/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.cpp +++ b/libmesosphere/source/board/nintendo/nx/kern_secure_monitor.cpp @@ -57,14 +57,18 @@ namespace ams::kern::board::nintendo::nx::smc { { /* Disable interrupts while making the call. */ KScopedInterruptDisable intr_disable; + + /* Backup the current thread pointer. */ + const uintptr_t current_thread_pointer_value = cpu::GetCurrentThreadPointerValue(); + __asm__ __volatile__("smc #1" : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7) : : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory" ); - /* Restore the CoreLocalRegion into X18. */ - cpu::SetCoreLocalRegionAddress(cpu::GetTpidrEl1()); + /* Restore the current thread pointer into X18. */ + cpu::SetCurrentThreadPointerValue(current_thread_pointer_value); } /* Store arguments to output. */ @@ -93,14 +97,18 @@ namespace ams::kern::board::nintendo::nx::smc { { /* Disable interrupts while making the call. */ KScopedInterruptDisable intr_disable; + + /* Backup the current thread pointer. */ + const uintptr_t current_thread_pointer_value = cpu::GetCurrentThreadPointerValue(); + __asm__ __volatile__("smc #0" : "+r"(x0), "+r"(x1), "+r"(x2), "+r"(x3), "+r"(x4), "+r"(x5), "+r"(x6), "+r"(x7) : : "x8", "x9", "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", "x18", "cc", "memory" ); - /* Restore the CoreLocalRegion into X18. */ - cpu::SetCoreLocalRegionAddress(cpu::GetTpidrEl1()); + /* Restore the current thread pointer into X18. */ + cpu::SetCurrentThreadPointerValue(current_thread_pointer_value); } /* Store arguments to output. */ diff --git a/libmesosphere/source/kern_k_debug_base.cpp b/libmesosphere/source/kern_k_debug_base.cpp index f7ab4328..601c1bb7 100644 --- a/libmesosphere/source/kern_k_debug_base.cpp +++ b/libmesosphere/source/kern_k_debug_base.cpp @@ -496,7 +496,7 @@ namespace ams::kern { if (thread->GetRawState() != KThread::ThreadState_Runnable) { bool current = false; for (auto i = 0; i < static_cast(cpu::NumCores); ++i) { - if (thread == Kernel::GetCurrentContext(i).current_thread) { + if (thread == Kernel::GetScheduler(i).GetSchedulerCurrentThread()) { current = true; } break; @@ -543,7 +543,7 @@ namespace ams::kern { if (thread->GetRawState() != KThread::ThreadState_Runnable) { bool current = false; for (auto i = 0; i < static_cast(cpu::NumCores); ++i) { - if (thread == Kernel::GetCurrentContext(i).current_thread) { + if (thread == Kernel::GetScheduler(i).GetSchedulerCurrentThread()) { current = true; } break; diff --git a/libmesosphere/source/kern_k_memory_layout.cpp b/libmesosphere/source/kern_k_memory_layout.cpp index 4eca1d72..468119c6 100644 --- a/libmesosphere/source/kern_k_memory_layout.cpp +++ b/libmesosphere/source/kern_k_memory_layout.cpp @@ -181,89 +181,4 @@ namespace ams::kern { return resource_region_size; } - namespace init { - - namespace { - - constexpr PageTableEntry KernelRwDataAttribute(PageTableEntry::Permission_KernelRW, PageTableEntry::PageAttribute_NormalMemory, PageTableEntry::Shareable_InnerShareable, PageTableEntry::MappingFlag_Mapped); - - constexpr size_t CoreLocalRegionAlign = PageSize; - constexpr size_t CoreLocalRegionSize = PageSize * (1 + cpu::NumCores); - constexpr size_t CoreLocalRegionSizeWithGuards = CoreLocalRegionSize + 2 * PageSize; - constexpr size_t CoreLocalRegionBoundsAlign = 1_GB; - static_assert(CoreLocalRegionSize == sizeof(KCoreLocalRegion)); - - KVirtualAddress GetCoreLocalRegionVirtualAddress() { - while (true) { - const uintptr_t candidate_start = GetInteger(KMemoryLayout::GetVirtualMemoryRegionTree().GetRandomAlignedRegion(CoreLocalRegionSizeWithGuards, CoreLocalRegionAlign, KMemoryRegionType_None)); - const uintptr_t candidate_end = candidate_start + CoreLocalRegionSizeWithGuards; - const uintptr_t candidate_last = candidate_end - 1; - - const auto &containing_region = *KMemoryLayout::GetVirtualMemoryRegionTree().Find(candidate_start); - - if (candidate_last > containing_region.GetLastAddress()) { - continue; - } - - if (containing_region.GetType() != KMemoryRegionType_None) { - continue; - } - - if (util::AlignDown(candidate_start, CoreLocalRegionBoundsAlign) != util::AlignDown(candidate_last, CoreLocalRegionBoundsAlign)) { - continue; - } - - if (containing_region.GetAddress() > util::AlignDown(candidate_start, CoreLocalRegionBoundsAlign)) { - continue; - } - - if (util::AlignUp(candidate_last, CoreLocalRegionBoundsAlign) - 1 > containing_region.GetLastAddress()) { - continue; - } - - return candidate_start + PageSize; - } - - } - - } - - void SetupCoreLocalRegionMemoryRegions(KInitialPageTable &page_table, KInitialPageAllocator &page_allocator) { - /* NOTE: Nintendo passes page table here to use num_l1_entries; we don't use this at present. */ - MESOSPHERE_UNUSED(page_table); - - /* Get the virtual address of the core local reigon. */ - const KVirtualAddress core_local_virt_start = GetCoreLocalRegionVirtualAddress(); - MESOSPHERE_INIT_ABORT_UNLESS(KMemoryLayout::GetVirtualMemoryRegionTree().Insert(GetInteger(core_local_virt_start), CoreLocalRegionSize, KMemoryRegionType_CoreLocalRegion)); - - /* Allocate a page for each core. */ - KPhysicalAddress core_local_region_start_phys[cpu::NumCores] = {}; - for (size_t i = 0; i < cpu::NumCores; i++) { - core_local_region_start_phys[i] = page_allocator.Allocate(); - } - - /* Allocate an l1 page table for each core. */ - KPhysicalAddress core_l1_ttbr1_phys[cpu::NumCores] = {}; - core_l1_ttbr1_phys[0] = util::AlignDown(cpu::GetTtbr1El1(), PageSize); - for (size_t i = 1; i < cpu::NumCores; i++) { - core_l1_ttbr1_phys[i] = page_allocator.Allocate(); - std::memcpy(reinterpret_cast(GetInteger(core_l1_ttbr1_phys[i])), reinterpret_cast(GetInteger(core_l1_ttbr1_phys[0])), PageSize); - } - - /* Use the l1 page table for each core to map the core local region for each core. */ - for (size_t i = 0; i < cpu::NumCores; i++) { - KInitialPageTable temp_pt(core_l1_ttbr1_phys[i], KInitialPageTable::NoClear{}); - temp_pt.Map(core_local_virt_start, PageSize, core_local_region_start_phys[i], KernelRwDataAttribute, page_allocator); - for (size_t j = 0; j < cpu::NumCores; j++) { - temp_pt.Map(core_local_virt_start + (j + 1) * PageSize, PageSize, core_local_region_start_phys[j], KernelRwDataAttribute, page_allocator); - } - - /* Setup the InitArguments. */ - SetInitArguments(static_cast(i), core_local_region_start_phys[i], GetInteger(core_l1_ttbr1_phys[i])); - } - } - - } - - } diff --git a/libmesosphere/source/kern_k_process.cpp b/libmesosphere/source/kern_k_process.cpp index ae767d13..7a7a7ff5 100644 --- a/libmesosphere/source/kern_k_process.cpp +++ b/libmesosphere/source/kern_k_process.cpp @@ -78,11 +78,6 @@ namespace ams::kern { } void KProcess::Finalize() { - /* Ensure we're not executing on any core. */ - for (size_t i = 0; i < cpu::NumCores; ++i) { - MESOSPHERE_ASSERT(Kernel::GetCurrentContext(static_cast(i)).current_process.load(std::memory_order_relaxed) != this); - } - /* Delete the process local region. */ this->DeleteThreadLocalRegion(this->plr_address); diff --git a/libmesosphere/source/kern_k_scheduler.cpp b/libmesosphere/source/kern_k_scheduler.cpp index 0a728712..227843bc 100644 --- a/libmesosphere/source/kern_k_scheduler.cpp +++ b/libmesosphere/source/kern_k_scheduler.cpp @@ -66,6 +66,9 @@ namespace ams::kern { /* Bind interrupt handler. */ Kernel::GetInterruptManager().BindHandler(GetSchedulerInterruptTask(), KInterruptName_Scheduler, this->core_id, KInterruptController::PriorityLevel_Scheduler, false, false); + + /* Set the current thread. */ + this->current_thread = GetCurrentThreadPointer(); } void KScheduler::Activate() { @@ -259,18 +262,21 @@ namespace ams::kern { MESOSPHERE_KTRACE_THREAD_SWITCH(next_thread); + if (next_thread->GetCurrentCore() != this->core_id) { + next_thread->SetCurrentCore(this->core_id); + } + /* Switch the current process, if we're switching processes. */ if (KProcess *next_process = next_thread->GetOwnerProcess(); next_process != cur_process) { - /* MESOSPHERE_LOG("!!! PROCESS SWITCH !!! %s -> %s\n", cur_process != nullptr ? cur_process->GetName() : nullptr, next_process != nullptr ? next_process->GetName() : nullptr); */ KProcess::Switch(cur_process, next_process); } /* Set the new thread. */ SetCurrentThread(next_thread); + this->current_thread = next_thread; /* Set the new Thread Local region. */ cpu::SwitchThreadLocalRegion(GetInteger(next_thread->GetThreadLocalRegionAddress())); - SetCurrentThreadLocalRegion(next_thread->GetThreadLocalRegionHeapAddress()); } void KScheduler::ClearPreviousThread(KThread *thread) { diff --git a/libmesosphere/source/kern_k_thread.cpp b/libmesosphere/source/kern_k_thread.cpp index eb9fc571..1564c93f 100644 --- a/libmesosphere/source/kern_k_thread.cpp +++ b/libmesosphere/source/kern_k_thread.cpp @@ -143,6 +143,9 @@ namespace ams::kern { this->num_kernel_waiters = 0; this->entrypoint = reinterpret_cast(func); + /* Set our current core id. */ + this->current_core_id = core; + /* We haven't released our resource limit hint, and we've spent no time on the cpu. */ this->resource_limit_release_hint = 0; this->cpu_time = 0; @@ -177,6 +180,7 @@ namespace ams::kern { this->parent->CopySvcPermissionsTo(sp); } sp.context = std::addressof(this->thread_context); + sp.cur_thread = this; sp.disable_count = 1; this->SetInExceptionHandler(); @@ -362,7 +366,7 @@ namespace ams::kern { for (size_t i = 0; i < cpu::NumCores; ++i) { KThread *core_thread; do { - core_thread = Kernel::GetCurrentContext(i).current_thread.load(std::memory_order_acquire); + core_thread = Kernel::GetScheduler(i).GetSchedulerCurrentThread(); } while (core_thread == this); } } @@ -619,7 +623,7 @@ namespace ams::kern { bool thread_is_current = false; s32 thread_core; for (thread_core = 0; thread_core < static_cast(cpu::NumCores); ++thread_core) { - if (Kernel::GetCurrentContext(thread_core).current_thread == this) { + if (Kernel::GetScheduler(thread_core).GetSchedulerCurrentThread() == this) { thread_is_current = true; break; } @@ -834,7 +838,7 @@ namespace ams::kern { thread_is_current = false; for (auto i = 0; i < static_cast(cpu::NumCores); ++i) { - if (Kernel::GetCurrentContext(i).current_thread == this) { + if (Kernel::GetScheduler(i).GetSchedulerCurrentThread() == this) { thread_is_current = true; break; } diff --git a/libmesosphere/source/kern_kernel.cpp b/libmesosphere/source/kern_kernel.cpp index 35ba1c3e..30f5c1d3 100644 --- a/libmesosphere/source/kern_kernel.cpp +++ b/libmesosphere/source/kern_kernel.cpp @@ -39,20 +39,9 @@ namespace ams::kern { } void Kernel::InitializeCoreLocalRegion(s32 core_id) { - /* Construct the core local region object in place. */ - KCoreLocalContext *clc = GetPointer(KMemoryLayout::GetCoreLocalRegionAddress()); - new (clc) KCoreLocalContext; - - /* Set the core local region address into the global register. */ - cpu::SetCoreLocalRegionAddress(reinterpret_cast(clc)); - - /* Initialize current context. */ - clc->current.current_thread = nullptr; - clc->current.current_process = nullptr; - clc->current.core_id = core_id; - clc->current.scheduler = std::addressof(Kernel::GetScheduler()); - clc->current.interrupt_task_manager = std::addressof(Kernel::GetInterruptTaskManager()); - clc->current.exception_stack_top = GetVoidPointer(KMemoryLayout::GetExceptionStackTopAddress(core_id) - sizeof(KThread::StackParameters)); + /* The core local region no longer exists, so just clear the current thread. */ + AMS_UNUSED(core_id); + SetCurrentThread(nullptr); } void Kernel::InitializeMainAndIdleThreads(s32 core_id) { @@ -68,7 +57,6 @@ namespace ams::kern { /* Set the current thread to be the main thread, and we have no processes running yet. */ SetCurrentThread(main_thread); - SetCurrentProcess(nullptr); /* Initialize the interrupt manager, hardware timer, and scheduler */ GetInterruptManager().Initialize(core_id); @@ -126,7 +114,6 @@ namespace ams::kern { PrintMemoryRegion(" Stack", KMemoryLayout::GetKernelStackRegionExtents()); PrintMemoryRegion(" Misc", KMemoryLayout::GetKernelMiscRegionExtents()); PrintMemoryRegion(" Slab", KMemoryLayout::GetKernelSlabRegionExtents()); - PrintMemoryRegion(" CoreLocalRegion", KMemoryLayout::GetCoreLocalRegion()); PrintMemoryRegion(" LinearRegion", KMemoryLayout::GetLinearRegionVirtualExtents()); MESOSPHERE_LOG("\n");