diff --git a/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_macros.h b/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_macros.h new file mode 100644 index 00000000..97b50afb --- /dev/null +++ b/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_macros.h @@ -0,0 +1,96 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once +#include + +#define ENABLE_FPU(tmp) \ + mrs tmp, cpacr_el1; \ + orr tmp, tmp, #0x300000; \ + msr cpacr_el1, tmp; \ + isb; + +#define GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \ + add ctx, sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_THREAD_CONTEXT); \ + ldp wtmp1, wtmp2, [ctx, #(THREAD_CONTEXT_FPCR_FPSR)]; \ + msr fpcr, xtmp1; \ + msr fpsr, xtmp2; + +#define RESTORE_FPU64_CALLEE_SAVE_REGISTERS(ctx) \ + ldp q8, q9, [ctx, #(THREAD_CONTEXT_FPU64_Q8_Q9)]; \ + ldp q10, q11, [ctx, #(THREAD_CONTEXT_FPU64_Q10_Q11)]; \ + ldp q12, q13, [ctx, #(THREAD_CONTEXT_FPU64_Q12_Q13)]; \ + ldp q14, q15, [ctx, #(THREAD_CONTEXT_FPU64_Q14_Q15)]; + +#define RESTORE_FPU64_CALLER_SAVE_REGISTERS(tmp) \ + ldr tmp, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS)]; \ + ldp q0, q1, [tmp, #(THREAD_FPU64_CONTEXT_Q0_Q1)]; \ + ldp q2, q3, [tmp, #(THREAD_FPU64_CONTEXT_Q2_Q3)]; \ + ldp q4, q5, [tmp, #(THREAD_FPU64_CONTEXT_Q4_Q5)]; \ + ldp q6, q7, [tmp, #(THREAD_FPU64_CONTEXT_Q6_Q7)]; \ + ldp q16, q17, [tmp, #(THREAD_FPU64_CONTEXT_Q16_Q17)]; \ + ldp q18, q19, [tmp, #(THREAD_FPU64_CONTEXT_Q18_Q19)]; \ + ldp q20, q21, [tmp, #(THREAD_FPU64_CONTEXT_Q20_Q21)]; \ + ldp q22, q23, [tmp, #(THREAD_FPU64_CONTEXT_Q22_Q23)]; \ + ldp q24, q25, [tmp, #(THREAD_FPU64_CONTEXT_Q24_Q25)]; \ + ldp q26, q27, [tmp, #(THREAD_FPU64_CONTEXT_Q26_Q27)]; \ + ldp q28, q29, [tmp, #(THREAD_FPU64_CONTEXT_Q28_Q29)]; \ + ldp q30, q31, [tmp, #(THREAD_FPU64_CONTEXT_Q30_Q31)]; + +#define RESTORE_FPU64_ALL_REGISTERS(ctx, tmp) \ + RESTORE_FPU64_CALLEE_SAVE_REGISTERS(ctx) \ + RESTORE_FPU64_CALLER_SAVE_REGISTERS(tmp) + +#define RESTORE_FPU32_CALLEE_SAVE_REGISTERS(ctx) \ + ldp q4, q5, [ctx, #(THREAD_CONTEXT_FPU32_Q4_Q5)]; \ + ldp q6, q7, [ctx, #(THREAD_CONTEXT_FPU32_Q6_Q7)]; + +#define RESTORE_FPU32_CALLER_SAVE_REGISTERS(tmp) \ + ldr tmp, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS)]; \ + ldp q0, q1, [tmp, #(THREAD_FPU32_CONTEXT_Q0_Q1)]; \ + ldp q2, q3, [tmp, #(THREAD_FPU32_CONTEXT_Q2_Q3)]; \ + ldp q8, q9, [tmp, #(THREAD_FPU32_CONTEXT_Q8_Q9)]; \ + ldp q10, q11, [tmp, #(THREAD_FPU32_CONTEXT_Q10_Q11)]; \ + ldp q12, q13, [tmp, #(THREAD_FPU32_CONTEXT_Q12_Q13)]; \ + ldp q14, q15, [tmp, #(THREAD_FPU32_CONTEXT_Q14_Q15)]; + +#define RESTORE_FPU32_ALL_REGISTERS(ctx, tmp) \ + RESTORE_FPU32_CALLEE_SAVE_REGISTERS(ctx) \ + RESTORE_FPU32_CALLER_SAVE_REGISTERS(tmp) + +#define ENABLE_AND_RESTORE_FPU(ctx, xtmp1, xtmp2, wtmp1, wtmp2, label_32, label_done) \ + ENABLE_FPU(xtmp1) \ + GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \ + \ + ldrb wtmp1, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)]; \ + tbz wtmp1, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_64_BIT), label_32##f; \ + \ + RESTORE_FPU64_ALL_REGISTERS(ctx, xtmp1) \ + \ + b label_done##f; \ + \ +label_32: \ + RESTORE_FPU32_ALL_REGISTERS(ctx, xtmp1) \ +label_done: + +#define ENABLE_AND_RESTORE_FPU64(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \ + ENABLE_FPU(xtmp1) \ + GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \ + RESTORE_FPU64_ALL_REGISTERS(ctx, xtmp1) + +#define ENABLE_AND_RESTORE_FPU32(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \ + ENABLE_FPU(xtmp1) \ + GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(ctx, xtmp1, xtmp2, wtmp1, wtmp2) \ + RESTORE_FPU32_ALL_REGISTERS(ctx, xtmp1) diff --git a/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h b/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h index 4ad886c2..d2988d92 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h +++ b/libmesosphere/include/mesosphere/arch/arm64/kern_assembly_offsets.h @@ -20,55 +20,157 @@ #define AMS_KERN_NUM_SUPERVISOR_CALLS 0xC0 /* ams::kern::KThread, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp */ -#define THREAD_THREAD_CONTEXT 0xD0 +#define THREAD_KERNEL_STACK_TOP 0x280 /* ams::kern::KThread::StackParameters, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/kern_k_thread.hpp */ -#define THREAD_STACK_PARAMETERS_SIZE 0x30 -#define THREAD_STACK_PARAMETERS_SVC_PERMISSION 0x00 -#define THREAD_STACK_PARAMETERS_CONTEXT 0x18 -#define THREAD_STACK_PARAMETERS_CUR_THREAD 0x20 -#define THREAD_STACK_PARAMETERS_DISABLE_COUNT 0x28 -#define THREAD_STACK_PARAMETERS_DPC_FLAGS 0x2A -#define THREAD_STACK_PARAMETERS_CURRENT_SVC_ID 0x2B -#define THREAD_STACK_PARAMETERS_IS_CALLING_SVC 0x2C -#define THREAD_STACK_PARAMETERS_IS_IN_EXCEPTION_HANDLER 0x2D -#define THREAD_STACK_PARAMETERS_IS_PINNED 0x2E +#define THREAD_STACK_PARAMETERS_SIZE 0x130 +#define THREAD_STACK_PARAMETERS_SVC_PERMISSION 0x00 +#define THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS 0x18 +#define THREAD_STACK_PARAMETERS_CUR_THREAD 0x20 +#define THREAD_STACK_PARAMETERS_DISABLE_COUNT 0x28 +#define THREAD_STACK_PARAMETERS_DPC_FLAGS 0x2A +#define THREAD_STACK_PARAMETERS_CURRENT_SVC_ID 0x2B +#define THREAD_STACK_PARAMETERS_RESERVED_2C 0x2C +#define THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS 0x2D +#define THREAD_STACK_PARAMETERS_IS_PINNED 0x2E +#define THREAD_STACK_PARAMETERS_RESERVED_2F 0x2F +#define THREAD_STACK_PARAMETERS_THREAD_CONTEXT 0x30 + +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_CALLING_SVC (0) +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_EXCEPTION_HANDLER (1) +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED (2) +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_64_BIT (3) +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER (4) +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_CACHE_MAINTENANCE_OPERATION (5) +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_TLB_MAINTENANCE_OPERATION (6) #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) -#define THREAD_STACK_PARAMETERS_IS_SINGLE_STEP 0x2F +#define THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_HARDWARE_SINGLE_STEP (7) +#endif + +#define THREAD_EXCEPTION_FLAG_IS_CALLING_SVC (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_CALLING_SVC) +#define THREAD_EXCEPTION_FLAG_IS_IN_EXCEPTION_HANDLER (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_EXCEPTION_HANDLER) +#define THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED) +#define THREAD_EXCEPTION_FLAG_IS_FPU_64_BIT (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_64_BIT) +#define THREAD_EXCEPTION_FLAG_IS_IN_USERMODE_EXCEPTION_HANDLER (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER) +#define THREAD_EXCEPTION_FLAG_IS_IN_CACHE_MAINTENANCE_OPERATION (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_CACHE_MAINTENANCE_OPERATION) +#define THREAD_EXCEPTION_FLAG_IS_IN_TLB_MAINTENANCE_OPERATION (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_TLB_MAINTENANCE_OPERATION) + +#if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) +#define THREAD_EXCEPTION_FLAG_IS_HARDWARE_SINGLE_STEP (1 << THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_HARDWARE_SINGLE_STEP) #endif /* ams::kern::arch::arm64::KThreadContext, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp */ -#define THREAD_CONTEXT_SIZE 0x290 -#define THREAD_CONTEXT_CPU_REGISTERS 0x000 -#define THREAD_CONTEXT_X19 0x000 -#define THREAD_CONTEXT_X20 0x008 -#define THREAD_CONTEXT_X21 0x010 -#define THREAD_CONTEXT_X22 0x018 -#define THREAD_CONTEXT_X23 0x020 -#define THREAD_CONTEXT_X24 0x028 -#define THREAD_CONTEXT_X25 0x030 -#define THREAD_CONTEXT_X26 0x038 -#define THREAD_CONTEXT_X27 0x040 -#define THREAD_CONTEXT_X28 0x048 -#define THREAD_CONTEXT_X29 0x050 -#define THREAD_CONTEXT_LR 0x058 -#define THREAD_CONTEXT_SP 0x060 -#define THREAD_CONTEXT_CPACR 0x068 -#define THREAD_CONTEXT_FPCR 0x070 -#define THREAD_CONTEXT_FPSR 0x078 -#define THREAD_CONTEXT_FPU_REGISTERS 0x080 -#define THREAD_CONTEXT_LOCKED 0x280 +#define THREAD_CONTEXT_SIZE 0x100 +#define THREAD_CONTEXT_CPU_REGISTERS 0x000 +#define THREAD_CONTEXT_X19 0x000 +#define THREAD_CONTEXT_X20 0x008 +#define THREAD_CONTEXT_X21 0x010 +#define THREAD_CONTEXT_X22 0x018 +#define THREAD_CONTEXT_X23 0x020 +#define THREAD_CONTEXT_X24 0x028 +#define THREAD_CONTEXT_X25 0x030 +#define THREAD_CONTEXT_X26 0x038 +#define THREAD_CONTEXT_X27 0x040 +#define THREAD_CONTEXT_X28 0x048 +#define THREAD_CONTEXT_X29 0x050 +#define THREAD_CONTEXT_LR 0x058 +#define THREAD_CONTEXT_SP 0x060 +#define THREAD_CONTEXT_FPCR 0x068 +#define THREAD_CONTEXT_FPSR 0x06C +#define THREAD_CONTEXT_FPU_REGISTERS 0x070 +#define THREAD_CONTEXT_LOCKED 0x0F0 -#define THREAD_CONTEXT_X19_X20 THREAD_CONTEXT_X19 -#define THREAD_CONTEXT_X21_X22 THREAD_CONTEXT_X21 -#define THREAD_CONTEXT_X23_X24 THREAD_CONTEXT_X23 -#define THREAD_CONTEXT_X25_X26 THREAD_CONTEXT_X25 -#define THREAD_CONTEXT_X27_X28 THREAD_CONTEXT_X27 -#define THREAD_CONTEXT_X29_X30 THREAD_CONTEXT_X29 -#define THREAD_CONTEXT_LR_SP THREAD_CONTEXT_LR -#define THREAD_CONTEXT_SP_CPACR THREAD_CONTEXT_SP -#define THREAD_CONTEXT_FPCR_FPSR THREAD_CONTEXT_FPCR +#define THREAD_CONTEXT_X19_X20 THREAD_CONTEXT_X19 +#define THREAD_CONTEXT_X21_X22 THREAD_CONTEXT_X21 +#define THREAD_CONTEXT_X23_X24 THREAD_CONTEXT_X23 +#define THREAD_CONTEXT_X25_X26 THREAD_CONTEXT_X25 +#define THREAD_CONTEXT_X27_X28 THREAD_CONTEXT_X27 +#define THREAD_CONTEXT_X29_X30 THREAD_CONTEXT_X29 +#define THREAD_CONTEXT_LR_SP THREAD_CONTEXT_LR +#define THREAD_CONTEXT_SP_FPCR_FPSR THREAD_CONTEXT_SP + +#define THREAD_CONTEXT_FPCR_FPSR THREAD_CONTEXT_FPCR + +#define THREAD_CONTEXT_FPU64_Q8 (THREAD_CONTEXT_FPU_REGISTERS + 0x00) +#define THREAD_CONTEXT_FPU64_Q9 (THREAD_CONTEXT_FPU_REGISTERS + 0x10) +#define THREAD_CONTEXT_FPU64_Q10 (THREAD_CONTEXT_FPU_REGISTERS + 0x20) +#define THREAD_CONTEXT_FPU64_Q11 (THREAD_CONTEXT_FPU_REGISTERS + 0x30) +#define THREAD_CONTEXT_FPU64_Q12 (THREAD_CONTEXT_FPU_REGISTERS + 0x40) +#define THREAD_CONTEXT_FPU64_Q13 (THREAD_CONTEXT_FPU_REGISTERS + 0x50) +#define THREAD_CONTEXT_FPU64_Q14 (THREAD_CONTEXT_FPU_REGISTERS + 0x60) +#define THREAD_CONTEXT_FPU64_Q15 (THREAD_CONTEXT_FPU_REGISTERS + 0x70) + +#define THREAD_CONTEXT_FPU64_Q8_Q9 THREAD_CONTEXT_FPU64_Q8 +#define THREAD_CONTEXT_FPU64_Q10_Q11 THREAD_CONTEXT_FPU64_Q10 +#define THREAD_CONTEXT_FPU64_Q12_Q13 THREAD_CONTEXT_FPU64_Q12 +#define THREAD_CONTEXT_FPU64_Q14_Q15 THREAD_CONTEXT_FPU64_Q14 + +#define THREAD_CONTEXT_FPU32_Q4 (THREAD_CONTEXT_FPU_REGISTERS + 0x00) +#define THREAD_CONTEXT_FPU32_Q5 (THREAD_CONTEXT_FPU_REGISTERS + 0x10) +#define THREAD_CONTEXT_FPU32_Q6 (THREAD_CONTEXT_FPU_REGISTERS + 0x20) +#define THREAD_CONTEXT_FPU32_Q7 (THREAD_CONTEXT_FPU_REGISTERS + 0x30) + +#define THREAD_CONTEXT_FPU32_Q4_Q5 THREAD_CONTEXT_FPU32_Q4 +#define THREAD_CONTEXT_FPU32_Q6_Q7 THREAD_CONTEXT_FPU32_Q6 + +#define THREAD_FPU64_CONTEXT_Q0 0x000 +#define THREAD_FPU64_CONTEXT_Q1 0x010 +#define THREAD_FPU64_CONTEXT_Q2 0x020 +#define THREAD_FPU64_CONTEXT_Q3 0x030 +#define THREAD_FPU64_CONTEXT_Q4 0x040 +#define THREAD_FPU64_CONTEXT_Q5 0x050 +#define THREAD_FPU64_CONTEXT_Q6 0x060 +#define THREAD_FPU64_CONTEXT_Q7 0x070 +#define THREAD_FPU64_CONTEXT_Q16 0x080 +#define THREAD_FPU64_CONTEXT_Q17 0x090 +#define THREAD_FPU64_CONTEXT_Q18 0x0A0 +#define THREAD_FPU64_CONTEXT_Q19 0x0B0 +#define THREAD_FPU64_CONTEXT_Q20 0x0C0 +#define THREAD_FPU64_CONTEXT_Q21 0x0D0 +#define THREAD_FPU64_CONTEXT_Q22 0x0E0 +#define THREAD_FPU64_CONTEXT_Q23 0x0F0 +#define THREAD_FPU64_CONTEXT_Q24 0x100 +#define THREAD_FPU64_CONTEXT_Q25 0x110 +#define THREAD_FPU64_CONTEXT_Q26 0x120 +#define THREAD_FPU64_CONTEXT_Q27 0x130 +#define THREAD_FPU64_CONTEXT_Q28 0x140 +#define THREAD_FPU64_CONTEXT_Q29 0x150 +#define THREAD_FPU64_CONTEXT_Q30 0x160 +#define THREAD_FPU64_CONTEXT_Q31 0x170 + +#define THREAD_FPU64_CONTEXT_Q0_Q1 THREAD_FPU64_CONTEXT_Q0 +#define THREAD_FPU64_CONTEXT_Q2_Q3 THREAD_FPU64_CONTEXT_Q2 +#define THREAD_FPU64_CONTEXT_Q4_Q5 THREAD_FPU64_CONTEXT_Q4 +#define THREAD_FPU64_CONTEXT_Q6_Q7 THREAD_FPU64_CONTEXT_Q6 +#define THREAD_FPU64_CONTEXT_Q16_Q17 THREAD_FPU64_CONTEXT_Q16 +#define THREAD_FPU64_CONTEXT_Q18_Q19 THREAD_FPU64_CONTEXT_Q18 +#define THREAD_FPU64_CONTEXT_Q20_Q21 THREAD_FPU64_CONTEXT_Q20 +#define THREAD_FPU64_CONTEXT_Q22_Q23 THREAD_FPU64_CONTEXT_Q22 +#define THREAD_FPU64_CONTEXT_Q24_Q25 THREAD_FPU64_CONTEXT_Q24 +#define THREAD_FPU64_CONTEXT_Q26_Q27 THREAD_FPU64_CONTEXT_Q26 +#define THREAD_FPU64_CONTEXT_Q28_Q29 THREAD_FPU64_CONTEXT_Q28 +#define THREAD_FPU64_CONTEXT_Q30_Q31 THREAD_FPU64_CONTEXT_Q30 + +#define THREAD_FPU32_CONTEXT_Q0 0x000 +#define THREAD_FPU32_CONTEXT_Q1 0x010 +#define THREAD_FPU32_CONTEXT_Q2 0x020 +#define THREAD_FPU32_CONTEXT_Q3 0x030 +#define THREAD_FPU32_CONTEXT_Q8 0x040 +#define THREAD_FPU32_CONTEXT_Q9 0x050 +#define THREAD_FPU32_CONTEXT_Q10 0x060 +#define THREAD_FPU32_CONTEXT_Q11 0x070 +#define THREAD_FPU32_CONTEXT_Q12 0x080 +#define THREAD_FPU32_CONTEXT_Q13 0x090 +#define THREAD_FPU32_CONTEXT_Q14 0x0A0 +#define THREAD_FPU32_CONTEXT_Q15 0x0B0 + +#define THREAD_FPU32_CONTEXT_Q0_Q1 THREAD_FPU32_CONTEXT_Q0 +#define THREAD_FPU32_CONTEXT_Q2_Q3 THREAD_FPU32_CONTEXT_Q2 +#define THREAD_FPU32_CONTEXT_Q8_Q9 THREAD_FPU32_CONTEXT_Q8 +#define THREAD_FPU32_CONTEXT_Q10_Q11 THREAD_FPU32_CONTEXT_Q10 +#define THREAD_FPU32_CONTEXT_Q12_Q13 THREAD_FPU32_CONTEXT_Q12 +#define THREAD_FPU32_CONTEXT_Q14_Q15 THREAD_FPU32_CONTEXT_Q14 /* ams::kern::arch::arm64::KExceptionContext, https://github.com/Atmosphere-NX/Atmosphere/blob/master/libraries/libmesosphere/include/mesosphere/arch/arm64/kern_k_exception_context.hpp */ #define EXCEPTION_CONTEXT_SIZE 0x120 diff --git a/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp b/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp index b4fb3a08..40cc1f86 100644 --- a/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp +++ b/libmesosphere/include/mesosphere/arch/arm64/kern_k_thread_context.hpp @@ -27,10 +27,12 @@ namespace ams::kern::arch::arm64 { class KThreadContext { public: - static constexpr size_t NumCalleeSavedRegisters = (29 - 19) + 1; - static constexpr size_t NumFpuRegisters = 32; - private: - union { + static constexpr size_t NumCalleeSavedRegisters = (29 - 19) + 1; + static constexpr size_t NumCalleeSavedFpuRegisters = 8; + static constexpr size_t NumCallerSavedFpuRegisters = 24; + static constexpr size_t NumFpuRegisters = NumCalleeSavedFpuRegisters + NumCallerSavedFpuRegisters; + public: + union CalleeSaveRegisters { u64 registers[NumCalleeSavedRegisters]; struct { u64 x19; @@ -45,23 +47,125 @@ namespace ams::kern::arch::arm64 { u64 x28; u64 x29; }; - } m_callee_saved; + }; + + union CalleeSaveFpu64Registers { + u128 v[NumCalleeSavedFpuRegisters]; + struct { + u128 q8; + u128 q9; + u128 q10; + u128 q11; + u128 q12; + u128 q13; + u128 q14; + u128 q15; + }; + }; + + union CalleeSaveFpu32Registers { + u128 v[NumCalleeSavedFpuRegisters / 2]; + struct { + u128 q4; + u128 q5; + u128 q6; + u128 q7; + }; + }; + + union CalleeSaveFpuRegisters { + CalleeSaveFpu64Registers fpu64; + CalleeSaveFpu32Registers fpu32; + }; + + union CallerSaveFpu64Registers { + u128 v[NumCallerSavedFpuRegisters]; + struct { + union { + u128 v0_7[NumCallerSavedFpuRegisters / 3]; + struct { + u128 q0; + u128 q1; + u128 q2; + u128 q3; + u128 q4; + u128 q5; + u128 q6; + u128 q7; + }; + }; + union { + u128 v16_31[2 * NumCallerSavedFpuRegisters / 3]; + struct { + u128 q16; + u128 q17; + u128 q18; + u128 q19; + u128 q20; + u128 q21; + u128 q22; + u128 q23; + u128 q24; + u128 q25; + u128 q26; + u128 q27; + u128 q28; + u128 q29; + u128 q30; + u128 q31; + }; + }; + }; + }; + + union CallerSaveFpu32Registers { + u128 v[NumCallerSavedFpuRegisters / 2]; + struct { + union { + u128 v0_3[(NumCallerSavedFpuRegisters / 3) / 2]; + struct { + u128 q0; + u128 q1; + u128 q2; + u128 q3; + }; + }; + union { + u128 v8_15[(2 * NumCallerSavedFpuRegisters / 3) / 2]; + struct { + u128 q8; + u128 q9; + u128 q10; + u128 q11; + u128 q12; + u128 q13; + u128 q14; + u128 q15; + }; + }; + }; + }; + + union CallerSaveFpuRegisters { + CallerSaveFpu64Registers fpu64; + CallerSaveFpu32Registers fpu32; + }; + private: + CalleeSaveRegisters m_callee_saved; u64 m_lr; u64 m_sp; - u64 m_cpacr; - u64 m_fpcr; - u64 m_fpsr; - alignas(0x10) u128 m_fpu_registers[NumFpuRegisters]; + u32 m_fpcr; + u32 m_fpsr; + alignas(0x10) CalleeSaveFpuRegisters m_callee_saved_fpu; bool m_locked; private: static void RestoreFpuRegisters64(const KThreadContext &); static void RestoreFpuRegisters32(const KThreadContext &); public: - constexpr explicit KThreadContext(util::ConstantInitializeTag) : m_callee_saved(), m_lr(), m_sp(), m_cpacr(), m_fpcr(), m_fpsr(), m_fpu_registers(), m_locked() { /* ... */ } + constexpr explicit KThreadContext(util::ConstantInitializeTag) : m_callee_saved(), m_lr(), m_sp(), m_fpcr(), m_fpsr(), m_callee_saved_fpu(), m_locked() { /* ... */ } explicit KThreadContext() { /* ... */ } Result Initialize(KVirtualAddress u_pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, bool is_user, bool is_64_bit, bool is_main); - Result Finalize(); void SetArguments(uintptr_t arg0, uintptr_t arg1); @@ -75,13 +179,58 @@ namespace ams::kern::arch::arm64 { void CloneFpuStatus(); - void SetFpuRegisters(const u128 *v, bool is_64_bit); - - const u128 *GetFpuRegisters() const { return m_fpu_registers; } + const auto &GetCalleeSaveFpuRegisters() const { return m_callee_saved_fpu; } + auto &GetCalleeSaveFpuRegisters() { return m_callee_saved_fpu; } public: static void OnThreadTerminating(const KThread *thread); public: static consteval bool ValidateOffsets(); + + template requires ((std::same_as && std::same_as) || (std::same_as && std::same_as)) + static void GetFpuRegisters(u128 *out, const CallerSave &caller_save, const CalleeSave &callee_save) { + /* Check that the register counts are correct. */ + constexpr size_t RegisterUnitCount = util::size(CalleeSave{}.v); + static_assert(util::size(CalleeSave{}.v) == 1 * RegisterUnitCount); + static_assert(util::size(CallerSave{}.v) == 3 * RegisterUnitCount); + + /* Copy the low caller-save registers. */ + for (size_t i = 0; i < RegisterUnitCount; ++i) { + *(out++) = caller_save.v[i]; + } + + /* Copy the callee-save registers. */ + for (size_t i = 0; i < RegisterUnitCount; ++i) { + *(out++) = callee_save.v[i]; + } + + /* Copy the remaining caller-save registers. */ + for (size_t i = 0; i < 2 * RegisterUnitCount; ++i) { + *(out++) = caller_save.v[RegisterUnitCount + i]; + } + } + + template requires ((std::same_as && std::same_as) || (std::same_as && std::same_as)) + static ALWAYS_INLINE void SetFpuRegisters(CallerSave &caller_save, CalleeSave &callee_save, const u128 *v) { + /* Check that the register counts are correct. */ + constexpr size_t RegisterUnitCount = util::size(CalleeSave{}.v); + static_assert(util::size(CalleeSave{}.v) == 1 * RegisterUnitCount); + static_assert(util::size(CallerSave{}.v) == 3 * RegisterUnitCount); + + /* Copy the low caller-save registers. */ + for (size_t i = 0; i < RegisterUnitCount; ++i) { + caller_save.v[i] = *(v++); + } + + /* Copy the callee-save registers. */ + for (size_t i = 0; i < RegisterUnitCount; ++i) { + callee_save.v[i] = *(v++); + } + + /* Copy the remaining caller-save registers. */ + for (size_t i = 0; i < 2 * RegisterUnitCount; ++i) { + caller_save.v[RegisterUnitCount + i] = *(v++); + } + } }; consteval bool KThreadContext::ValidateOffsets() { @@ -101,17 +250,67 @@ namespace ams::kern::arch::arm64 { static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved.x29) == THREAD_CONTEXT_X29); static_assert(AMS_OFFSETOF(KThreadContext, m_lr) == THREAD_CONTEXT_LR); static_assert(AMS_OFFSETOF(KThreadContext, m_sp) == THREAD_CONTEXT_SP); - static_assert(AMS_OFFSETOF(KThreadContext, m_cpacr) == THREAD_CONTEXT_CPACR); static_assert(AMS_OFFSETOF(KThreadContext, m_fpcr) == THREAD_CONTEXT_FPCR); static_assert(AMS_OFFSETOF(KThreadContext, m_fpsr) == THREAD_CONTEXT_FPSR); - static_assert(AMS_OFFSETOF(KThreadContext, m_fpu_registers) == THREAD_CONTEXT_FPU_REGISTERS); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu) == THREAD_CONTEXT_FPU_REGISTERS); static_assert(AMS_OFFSETOF(KThreadContext, m_locked) == THREAD_CONTEXT_LOCKED); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q8 ) == THREAD_CONTEXT_FPU64_Q8 ); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q9 ) == THREAD_CONTEXT_FPU64_Q9 ); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q10) == THREAD_CONTEXT_FPU64_Q10); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q11) == THREAD_CONTEXT_FPU64_Q11); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q12) == THREAD_CONTEXT_FPU64_Q12); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q13) == THREAD_CONTEXT_FPU64_Q13); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q14) == THREAD_CONTEXT_FPU64_Q14); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu64.q15) == THREAD_CONTEXT_FPU64_Q15); + + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q4 ) == THREAD_CONTEXT_FPU32_Q4 ); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q5 ) == THREAD_CONTEXT_FPU32_Q5 ); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q6 ) == THREAD_CONTEXT_FPU32_Q6 ); + static_assert(AMS_OFFSETOF(KThreadContext, m_callee_saved_fpu.fpu32.q7 ) == THREAD_CONTEXT_FPU32_Q7 ); + + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q0 ) == THREAD_FPU64_CONTEXT_Q0 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q1 ) == THREAD_FPU64_CONTEXT_Q1 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q2 ) == THREAD_FPU64_CONTEXT_Q2 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q3 ) == THREAD_FPU64_CONTEXT_Q3 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q4 ) == THREAD_FPU64_CONTEXT_Q4 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q5 ) == THREAD_FPU64_CONTEXT_Q5 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q6 ) == THREAD_FPU64_CONTEXT_Q6 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q7 ) == THREAD_FPU64_CONTEXT_Q7 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q16) == THREAD_FPU64_CONTEXT_Q16); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q17) == THREAD_FPU64_CONTEXT_Q17); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q18) == THREAD_FPU64_CONTEXT_Q18); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q19) == THREAD_FPU64_CONTEXT_Q19); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q20) == THREAD_FPU64_CONTEXT_Q20); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q21) == THREAD_FPU64_CONTEXT_Q21); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q22) == THREAD_FPU64_CONTEXT_Q22); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q23) == THREAD_FPU64_CONTEXT_Q23); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q24) == THREAD_FPU64_CONTEXT_Q24); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q25) == THREAD_FPU64_CONTEXT_Q25); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q26) == THREAD_FPU64_CONTEXT_Q26); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q27) == THREAD_FPU64_CONTEXT_Q27); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q28) == THREAD_FPU64_CONTEXT_Q28); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q29) == THREAD_FPU64_CONTEXT_Q29); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q30) == THREAD_FPU64_CONTEXT_Q30); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu64.q31) == THREAD_FPU64_CONTEXT_Q31); + + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q0 ) == THREAD_FPU32_CONTEXT_Q0 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q1 ) == THREAD_FPU32_CONTEXT_Q1 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q2 ) == THREAD_FPU32_CONTEXT_Q2 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q3 ) == THREAD_FPU32_CONTEXT_Q3 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q8 ) == THREAD_FPU32_CONTEXT_Q8 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q9 ) == THREAD_FPU32_CONTEXT_Q9 ); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q10) == THREAD_FPU32_CONTEXT_Q10); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q11) == THREAD_FPU32_CONTEXT_Q11); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q12) == THREAD_FPU32_CONTEXT_Q12); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q13) == THREAD_FPU32_CONTEXT_Q13); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q14) == THREAD_FPU32_CONTEXT_Q14); + static_assert(AMS_OFFSETOF(KThreadContext::CallerSaveFpuRegisters, fpu32.q15) == THREAD_FPU32_CONTEXT_Q15); + return true; } static_assert(KThreadContext::ValidateOffsets()); - void GetUserContext(ams::svc::ThreadContext *out, const KThread *thread); } \ No newline at end of file diff --git a/libmesosphere/include/mesosphere/kern_k_capabilities.hpp b/libmesosphere/include/mesosphere/kern_k_capabilities.hpp index e175f118..dbc973de 100644 --- a/libmesosphere/include/mesosphere/kern_k_capabilities.hpp +++ b/libmesosphere/include/mesosphere/kern_k_capabilities.hpp @@ -239,78 +239,7 @@ namespace ams::kern { constexpr u64 GetPriorityMask() const { return m_priority_mask; } constexpr s32 GetHandleTableSize() const { return m_handle_table_size; } - ALWAYS_INLINE void CopySvcPermissionsTo(KThread::StackParameters &sp) const { - /* Copy permissions. */ - sp.svc_access_flags = m_svc_access_flags; - - /* Clear specific SVCs based on our state. */ - sp.svc_access_flags[svc::SvcId_ReturnFromException] = false; - sp.svc_access_flags[svc::SvcId_SynchronizePreemptionState] = false; - if (sp.is_pinned) { - sp.svc_access_flags[svc::SvcId_GetInfo] = false; - } - } - - ALWAYS_INLINE void CopyPinnedSvcPermissionsTo(KThread::StackParameters &sp) const { - /* Get whether we have access to return from exception. */ - const bool return_from_exception = sp.svc_access_flags[svc::SvcId_ReturnFromException]; - - /* Clear all permissions. */ - sp.svc_access_flags.Reset(); - - /* Set SynchronizePreemptionState if allowed. */ - if (m_svc_access_flags[svc::SvcId_SynchronizePreemptionState]) { - sp.svc_access_flags[svc::SvcId_SynchronizePreemptionState] = true; - } - - /* If we previously had ReturnFromException, potentially grant it and GetInfo. */ - if (return_from_exception) { - /* Set ReturnFromException (guaranteed allowed, if we're here). */ - sp.svc_access_flags[svc::SvcId_ReturnFromException] = true; - - /* Set GetInfo if allowed. */ - if (m_svc_access_flags[svc::SvcId_GetInfo]) { - sp.svc_access_flags[svc::SvcId_GetInfo] = true; - } - } - } - - ALWAYS_INLINE void CopyUnpinnedSvcPermissionsTo(KThread::StackParameters &sp) const { - /* Get whether we have access to return from exception. */ - const bool return_from_exception = sp.svc_access_flags[svc::SvcId_ReturnFromException]; - - /* Copy permissions. */ - sp.svc_access_flags = m_svc_access_flags; - - /* Clear specific SVCs based on our state. */ - sp.svc_access_flags[svc::SvcId_SynchronizePreemptionState] = false; - - if (!return_from_exception) { - sp.svc_access_flags[svc::SvcId_ReturnFromException] = false; - } - } - - ALWAYS_INLINE void CopyEnterExceptionSvcPermissionsTo(KThread::StackParameters &sp) const { - /* Set ReturnFromException if allowed. */ - if (m_svc_access_flags[svc::SvcId_ReturnFromException]) { - sp.svc_access_flags[svc::SvcId_ReturnFromException] = true; - } - - /* Set GetInfo if allowed. */ - if (m_svc_access_flags[svc::SvcId_GetInfo]) { - sp.svc_access_flags[svc::SvcId_GetInfo] = true; - } - } - - ALWAYS_INLINE void CopyLeaveExceptionSvcPermissionsTo(KThread::StackParameters &sp) const { - /* Clear ReturnFromException. */ - sp.svc_access_flags[svc::SvcId_ReturnFromException] = false; - - /* If pinned, clear GetInfo. */ - if (sp.is_pinned) { - sp.svc_access_flags[svc::SvcId_GetInfo] = false; - } - } + constexpr const svc::SvcAccessFlagSet &GetSvcPermissions() const { return m_svc_access_flags; } constexpr bool IsPermittedSvc(svc::SvcId id) const { return (id < m_svc_access_flags.GetCount()) && m_svc_access_flags[id]; diff --git a/libmesosphere/include/mesosphere/kern_k_process.hpp b/libmesosphere/include/mesosphere/kern_k_process.hpp index 97429df0..8e3fb85a 100644 --- a/libmesosphere/include/mesosphere/kern_k_process.hpp +++ b/libmesosphere/include/mesosphere/kern_k_process.hpp @@ -233,25 +233,7 @@ namespace ams::kern { return m_pinned_threads[core_id]; } - void CopySvcPermissionsTo(KThread::StackParameters &sp) { - m_capabilities.CopySvcPermissionsTo(sp); - } - - void CopyPinnedSvcPermissionsTo(KThread::StackParameters &sp) { - m_capabilities.CopyPinnedSvcPermissionsTo(sp); - } - - void CopyUnpinnedSvcPermissionsTo(KThread::StackParameters &sp) { - m_capabilities.CopyUnpinnedSvcPermissionsTo(sp); - } - - void CopyEnterExceptionSvcPermissionsTo(KThread::StackParameters &sp) { - m_capabilities.CopyEnterExceptionSvcPermissionsTo(sp); - } - - void CopyLeaveExceptionSvcPermissionsTo(KThread::StackParameters &sp) { - m_capabilities.CopyLeaveExceptionSvcPermissionsTo(sp); - } + const svc::SvcAccessFlagSet &GetSvcPermissions() const { return m_capabilities.GetSvcPermissions(); } constexpr KResourceLimit *GetResourceLimit() const { return m_resource_limit; } diff --git a/libmesosphere/include/mesosphere/kern_k_thread.hpp b/libmesosphere/include/mesosphere/kern_k_thread.hpp index a9574e58..a3f19b0a 100644 --- a/libmesosphere/include/mesosphere/kern_k_thread.hpp +++ b/libmesosphere/include/mesosphere/kern_k_thread.hpp @@ -84,35 +84,56 @@ namespace ams::kern { DpcFlag_PerformDestruction = (1 << 2), }; + enum ExceptionFlag : u32 { + ExceptionFlag_IsCallingSvc = (1 << 0), + ExceptionFlag_IsInExceptionHandler = (1 << 1), + ExceptionFlag_IsFpuContextRestoreNeeded = (1 << 2), + ExceptionFlag_IsFpu64Bit = (1 << 3), + ExceptionFlag_IsInUsermodeExceptionHandler = (1 << 4), + ExceptionFlag_IsInCacheMaintenanceOperation = (1 << 5), + ExceptionFlag_IsInTlbMaintenanceOperation = (1 << 6), + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + ExceptionFlag_IsHardwareSingleStep = (1 << 7), + #endif + }; + struct StackParameters { - alignas(0x10) svc::SvcAccessFlagSet svc_access_flags; - KThreadContext *context; + svc::SvcAccessFlagSet svc_access_flags; + KThreadContext::CallerSaveFpuRegisters *caller_save_fpu_registers; KThread *cur_thread; s16 disable_count; util::Atomic dpc_flags; u8 current_svc_id; - bool is_calling_svc; - bool is_in_exception_handler; + u8 reserved_2c; + u8 exception_flags; bool is_pinned; - #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - bool is_single_step; - #endif + u8 reserved_2f; + KThreadContext context; }; - static_assert(alignof(StackParameters) == 0x10); - static_assert(sizeof(StackParameters) == THREAD_STACK_PARAMETERS_SIZE); - static_assert(AMS_OFFSETOF(StackParameters, svc_access_flags) == THREAD_STACK_PARAMETERS_SVC_PERMISSION); - static_assert(AMS_OFFSETOF(StackParameters, context) == THREAD_STACK_PARAMETERS_CONTEXT); - static_assert(AMS_OFFSETOF(StackParameters, cur_thread) == THREAD_STACK_PARAMETERS_CUR_THREAD); - static_assert(AMS_OFFSETOF(StackParameters, disable_count) == THREAD_STACK_PARAMETERS_DISABLE_COUNT); - static_assert(AMS_OFFSETOF(StackParameters, dpc_flags) == THREAD_STACK_PARAMETERS_DPC_FLAGS); - static_assert(AMS_OFFSETOF(StackParameters, current_svc_id) == THREAD_STACK_PARAMETERS_CURRENT_SVC_ID); - static_assert(AMS_OFFSETOF(StackParameters, is_calling_svc) == THREAD_STACK_PARAMETERS_IS_CALLING_SVC); - static_assert(AMS_OFFSETOF(StackParameters, is_in_exception_handler) == THREAD_STACK_PARAMETERS_IS_IN_EXCEPTION_HANDLER); - static_assert(AMS_OFFSETOF(StackParameters, is_pinned) == THREAD_STACK_PARAMETERS_IS_PINNED); + static_assert(util::IsAligned(AMS_OFFSETOF(StackParameters, context), 0x10)); + static_assert(AMS_OFFSETOF(StackParameters, svc_access_flags) == THREAD_STACK_PARAMETERS_SVC_PERMISSION); + static_assert(AMS_OFFSETOF(StackParameters, caller_save_fpu_registers) == THREAD_STACK_PARAMETERS_CALLER_SAVE_FPU_REGISTERS); + static_assert(AMS_OFFSETOF(StackParameters, cur_thread) == THREAD_STACK_PARAMETERS_CUR_THREAD); + static_assert(AMS_OFFSETOF(StackParameters, disable_count) == THREAD_STACK_PARAMETERS_DISABLE_COUNT); + static_assert(AMS_OFFSETOF(StackParameters, dpc_flags) == THREAD_STACK_PARAMETERS_DPC_FLAGS); + static_assert(AMS_OFFSETOF(StackParameters, current_svc_id) == THREAD_STACK_PARAMETERS_CURRENT_SVC_ID); + static_assert(AMS_OFFSETOF(StackParameters, reserved_2c) == THREAD_STACK_PARAMETERS_RESERVED_2C); + static_assert(AMS_OFFSETOF(StackParameters, exception_flags) == THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS); + static_assert(AMS_OFFSETOF(StackParameters, is_pinned) == THREAD_STACK_PARAMETERS_IS_PINNED); + static_assert(AMS_OFFSETOF(StackParameters, reserved_2f) == THREAD_STACK_PARAMETERS_RESERVED_2F); + static_assert(AMS_OFFSETOF(StackParameters, context) == THREAD_STACK_PARAMETERS_THREAD_CONTEXT); + + static_assert(ExceptionFlag_IsCallingSvc == THREAD_EXCEPTION_FLAG_IS_CALLING_SVC); + static_assert(ExceptionFlag_IsInExceptionHandler == THREAD_EXCEPTION_FLAG_IS_IN_EXCEPTION_HANDLER); + static_assert(ExceptionFlag_IsFpuContextRestoreNeeded == THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED); + static_assert(ExceptionFlag_IsFpu64Bit == THREAD_EXCEPTION_FLAG_IS_FPU_64_BIT); + static_assert(ExceptionFlag_IsInUsermodeExceptionHandler == THREAD_EXCEPTION_FLAG_IS_IN_USERMODE_EXCEPTION_HANDLER); + static_assert(ExceptionFlag_IsInCacheMaintenanceOperation == THREAD_EXCEPTION_FLAG_IS_IN_CACHE_MAINTENANCE_OPERATION); + static_assert(ExceptionFlag_IsInTlbMaintenanceOperation == THREAD_EXCEPTION_FLAG_IS_IN_TLB_MAINTENANCE_OPERATION); #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - static_assert(AMS_OFFSETOF(StackParameters, is_single_step) == THREAD_STACK_PARAMETERS_IS_SINGLE_STEP); + static_assert(ExceptionFlag_IsHardwareSingleStep == THREAD_EXCEPTION_FLAG_IS_HARDWARE_SINGLE_STEP); #endif struct QueueEntry { @@ -184,60 +205,60 @@ namespace ams::kern { using ConditionVariableThreadTreeTraits = util::IntrusiveRedBlackTreeMemberTraitsDeferredAssert<&KThread::m_condvar_arbiter_tree_node>; using ConditionVariableThreadTree = ConditionVariableThreadTreeTraits::TreeType; - ConditionVariableThreadTree *m_condvar_tree; - uintptr_t m_condvar_key; - alignas(16) KThreadContext m_thread_context; - u64 m_virtual_affinity_mask; - KAffinityMask m_physical_affinity_mask; - u64 m_thread_id; - util::Atomic m_cpu_time; - KProcessAddress m_address_key; - KProcess *m_parent; - void *m_kernel_stack_top; - u32 *m_light_ipc_data; - KProcessAddress m_tls_address; - void *m_tls_heap_address; - KLightLock m_activity_pause_lock; - SyncObjectBuffer m_sync_object_buffer; - s64 m_schedule_count; - s64 m_last_scheduled_tick; - QueueEntry m_per_core_priority_queue_entry[cpu::NumCores]; - KThreadQueue *m_wait_queue; - WaiterList m_waiter_list; - WaiterList m_pinned_waiter_list; - KThread *m_lock_owner; - uintptr_t m_debug_params[3]; - KAutoObject *m_closed_object; - u32 m_address_key_value; - u32 m_suspend_request_flags; - u32 m_suspend_allowed_flags; - s32 m_synced_index; - Result m_wait_result; - Result m_debug_exception_result; - s32 m_base_priority; - s32 m_base_priority_on_unpin; - s32 m_physical_ideal_core_id; - s32 m_virtual_ideal_core_id; - s32 m_num_kernel_waiters; - s32 m_current_core_id; - s32 m_core_id; - KAffinityMask m_original_physical_affinity_mask; - s32 m_original_physical_ideal_core_id; - s32 m_num_core_migration_disables; - ThreadState m_thread_state; - util::Atomic m_termination_requested; - bool m_wait_cancelled; - bool m_cancellable; - bool m_signaled; - bool m_initialized; - bool m_debug_attached; - s8 m_priority_inheritance_count; - bool m_resource_limit_release_hint; + ConditionVariableThreadTree *m_condvar_tree; + uintptr_t m_condvar_key; + alignas(16) KThreadContext::CallerSaveFpuRegisters m_caller_save_fpu_registers; + u64 m_virtual_affinity_mask; + KAffinityMask m_physical_affinity_mask; + u64 m_thread_id; + util::Atomic m_cpu_time; + KProcessAddress m_address_key; + KProcess *m_parent; + void *m_kernel_stack_top; + u32 *m_light_ipc_data; + KProcessAddress m_tls_address; + void *m_tls_heap_address; + KLightLock m_activity_pause_lock; + SyncObjectBuffer m_sync_object_buffer; + s64 m_schedule_count; + s64 m_last_scheduled_tick; + QueueEntry m_per_core_priority_queue_entry[cpu::NumCores]; + KThreadQueue *m_wait_queue; + WaiterList m_waiter_list; + WaiterList m_pinned_waiter_list; + KThread *m_lock_owner; + uintptr_t m_debug_params[3]; + KAutoObject *m_closed_object; + u32 m_address_key_value; + u32 m_suspend_request_flags; + u32 m_suspend_allowed_flags; + s32 m_synced_index; + Result m_wait_result; + Result m_debug_exception_result; + s32 m_base_priority; + s32 m_base_priority_on_unpin; + s32 m_physical_ideal_core_id; + s32 m_virtual_ideal_core_id; + s32 m_num_kernel_waiters; + s32 m_current_core_id; + s32 m_core_id; + KAffinityMask m_original_physical_affinity_mask; + s32 m_original_physical_ideal_core_id; + s32 m_num_core_migration_disables; + ThreadState m_thread_state; + util::Atomic m_termination_requested; + bool m_wait_cancelled; + bool m_cancellable; + bool m_signaled; + bool m_initialized; + bool m_debug_attached; + s8 m_priority_inheritance_count; + bool m_resource_limit_release_hint; public: constexpr explicit KThread(util::ConstantInitializeTag) : KAutoObjectWithSlabHeapAndContainer(util::ConstantInitialize), KTimerTask(util::ConstantInitialize), m_process_list_node{}, m_condvar_arbiter_tree_node{util::ConstantInitialize}, m_priority{-1}, m_condvar_tree{}, m_condvar_key{}, - m_thread_context{util::ConstantInitialize}, m_virtual_affinity_mask{}, m_physical_affinity_mask{}, m_thread_id{}, m_cpu_time{0}, m_address_key{Null}, m_parent{}, + m_caller_save_fpu_registers{}, m_virtual_affinity_mask{}, m_physical_affinity_mask{}, m_thread_id{}, m_cpu_time{0}, m_address_key{Null}, m_parent{}, m_kernel_stack_top{}, m_light_ipc_data{}, m_tls_address{Null}, m_tls_heap_address{}, m_activity_pause_lock{}, m_sync_object_buffer{util::ConstantInitialize}, m_schedule_count{}, m_last_scheduled_tick{}, m_per_core_priority_queue_entry{}, m_wait_queue{}, m_waiter_list{}, m_pinned_waiter_list{}, m_lock_owner{}, m_debug_params{}, m_closed_object{}, m_address_key_value{}, m_suspend_request_flags{}, m_suspend_allowed_flags{}, m_synced_index{}, @@ -269,17 +290,8 @@ namespace ams::kern { static void ResumeThreadsSuspendedForInit(); private: - StackParameters &GetStackParameters() { - return *(reinterpret_cast(m_kernel_stack_top) - 1); - } - - const StackParameters &GetStackParameters() const { - return *(reinterpret_cast(m_kernel_stack_top) - 1); - } - public: - StackParameters &GetStackParametersForExceptionSvcPermission() { - return *(reinterpret_cast(m_kernel_stack_top) - 1); - } + ALWAYS_INLINE StackParameters &GetStackParameters() { return *(reinterpret_cast< StackParameters *>(m_kernel_stack_top) - 1); } + ALWAYS_INLINE const StackParameters &GetStackParameters() const { return *(reinterpret_cast(m_kernel_stack_top) - 1); } public: ALWAYS_INLINE s16 GetDisableDispatchCount() const { MESOSPHERE_ASSERT_THIS(); @@ -315,51 +327,61 @@ namespace ams::kern { NOINLINE void DisableCoreMigration(); NOINLINE void EnableCoreMigration(); - - ALWAYS_INLINE void SetInExceptionHandler() { + private: + ALWAYS_INLINE void SetExceptionFlag(ExceptionFlag flag) { MESOSPHERE_ASSERT_THIS(); - this->GetStackParameters().is_in_exception_handler = true; + this->GetStackParameters().exception_flags |= flag; } - ALWAYS_INLINE void ClearInExceptionHandler() { + ALWAYS_INLINE void ClearExceptionFlag(ExceptionFlag flag) { MESOSPHERE_ASSERT_THIS(); - this->GetStackParameters().is_in_exception_handler = false; + this->GetStackParameters().exception_flags &= ~flag; } - ALWAYS_INLINE bool IsInExceptionHandler() const { + ALWAYS_INLINE bool IsExceptionFlagSet(ExceptionFlag flag) const { MESOSPHERE_ASSERT_THIS(); - return this->GetStackParameters().is_in_exception_handler; + return this->GetStackParameters().exception_flags & flag; } + public: + /* ALWAYS_INLINE void SetCallingSvc() { return this->SetExceptionFlag(ExceptionFlag_IsCallingSvc); } */ + /* ALWAYS_INLINE void ClearCallingSvc() { return this->ClearExceptionFlag(ExceptionFlag_IsCallingSvc); } */ + ALWAYS_INLINE bool IsCallingSvc() const { return this->IsExceptionFlagSet(ExceptionFlag_IsCallingSvc); } - ALWAYS_INLINE bool IsCallingSvc() const { - MESOSPHERE_ASSERT_THIS(); - return this->GetStackParameters().is_calling_svc; - } + ALWAYS_INLINE void SetInExceptionHandler() { return this->SetExceptionFlag(ExceptionFlag_IsInExceptionHandler); } + ALWAYS_INLINE void ClearInExceptionHandler() { return this->ClearExceptionFlag(ExceptionFlag_IsInExceptionHandler); } + ALWAYS_INLINE bool IsInExceptionHandler() const { return this->IsExceptionFlagSet(ExceptionFlag_IsInExceptionHandler); } + + /* ALWAYS_INLINE void SetFpuContextRestoreNeeded() { return this->SetExceptionFlag(ExceptionFlag_IsFpuContextRestoreNeeded); } */ + /* ALWAYS_INLINE void ClearFpuContextRestoreNeeded() { return this->ClearExceptionFlag(ExceptionFlag_IsFpuContextRestoreNeeded); } */ + /* ALWAYS_INLINE bool IsFpuContextRestoreNeeded() const { return this->IsExceptionFlagSet(ExceptionFlag_IsFpuContextRestoreNeeded); } */ + + ALWAYS_INLINE void SetFpu64Bit() { return this->SetExceptionFlag(ExceptionFlag_IsFpu64Bit); } + /* ALWAYS_INLINE void ClearFpu64Bit() { return this->ClearExceptionFlag(ExceptionFlag_IsFpu64Bit); } */ + /* ALWAYS_INLINE bool IsFpu64Bit() const { return this->IsExceptionFlagSet(ExceptionFlag_IsFpu64Bit); } */ + + ALWAYS_INLINE void SetInUsermodeExceptionHandler() { return this->SetExceptionFlag(ExceptionFlag_IsInUsermodeExceptionHandler); } + ALWAYS_INLINE void ClearInUsermodeExceptionHandler() { return this->ClearExceptionFlag(ExceptionFlag_IsInUsermodeExceptionHandler); } + ALWAYS_INLINE bool IsInUsermodeExceptionHandler() const { return this->IsExceptionFlagSet(ExceptionFlag_IsInUsermodeExceptionHandler); } + + ALWAYS_INLINE void SetInCacheMaintenanceOperation() { return this->SetExceptionFlag(ExceptionFlag_IsInCacheMaintenanceOperation); } + ALWAYS_INLINE void ClearInCacheMaintenanceOperation() { return this->ClearExceptionFlag(ExceptionFlag_IsInCacheMaintenanceOperation); } + ALWAYS_INLINE bool IsInCacheMaintenanceOperation() const { return this->IsExceptionFlagSet(ExceptionFlag_IsInCacheMaintenanceOperation); } + + ALWAYS_INLINE void SetInTlbMaintenanceOperation() { return this->SetExceptionFlag(ExceptionFlag_IsInTlbMaintenanceOperation); } + ALWAYS_INLINE void ClearInTlbMaintenanceOperation() { return this->ClearExceptionFlag(ExceptionFlag_IsInTlbMaintenanceOperation); } + ALWAYS_INLINE bool IsInTlbMaintenanceOperation() const { return this->IsExceptionFlagSet(ExceptionFlag_IsInTlbMaintenanceOperation); } + + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + ALWAYS_INLINE void SetHardwareSingleStep() { return this->SetExceptionFlag(ExceptionFlag_IsHardwareSingleStep); } + ALWAYS_INLINE void ClearHardwareSingleStep() { return this->ClearExceptionFlag(ExceptionFlag_IsHardwareSingleStep); } + ALWAYS_INLINE bool IsHardwareSingleStep() const { return this->IsExceptionFlagSet(ExceptionFlag_IsHardwareSingleStep); } + #endif ALWAYS_INLINE u8 GetSvcId() const { MESOSPHERE_ASSERT_THIS(); return this->GetStackParameters().current_svc_id; } - #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - - ALWAYS_INLINE void SetSingleStep() { - MESOSPHERE_ASSERT_THIS(); - this->GetStackParameters().is_single_step = true; - } - - ALWAYS_INLINE void ClearSingleStep() { - MESOSPHERE_ASSERT_THIS(); - this->GetStackParameters().is_single_step = false; - } - - ALWAYS_INLINE bool IsSingleStep() const { - MESOSPHERE_ASSERT_THIS(); - return this->GetStackParameters().is_single_step; - } - - #endif - ALWAYS_INLINE void RegisterDpc(DpcFlag flag) { this->GetStackParameters().dpc_flags |= flag; } @@ -376,6 +398,13 @@ namespace ams::kern { MESOSPHERE_ASSERT_THIS(); return this->GetDpc() != 0; } + + private: + void SetPinnedSvcPermissions(); + void SetUnpinnedSvcPermissions(); + + void SetUsermodeExceptionSvcPermissions(); + void ClearUsermodeExceptionSvcPermissions(); private: void UpdateState(); ALWAYS_INLINE void AddWaiterImpl(KThread *thread); @@ -391,8 +420,11 @@ namespace ams::kern { public: constexpr u64 GetThreadId() const { return m_thread_id; } - constexpr KThreadContext &GetContext() { return m_thread_context; } - constexpr const KThreadContext &GetContext() const { return m_thread_context; } + const KThreadContext &GetContext() const { return this->GetStackParameters().context; } + KThreadContext &GetContext() { return this->GetStackParameters().context; } + + const auto &GetCallerSaveFpuRegisters() const { return m_caller_save_fpu_registers; } + auto &GetCallerSaveFpuRegisters() { return m_caller_save_fpu_registers; } constexpr u64 GetVirtualAffinityMask() const { return m_virtual_affinity_mask; } constexpr const KAffinityMask &GetAffinityMask() const { return m_physical_affinity_mask; } @@ -518,6 +550,8 @@ namespace ams::kern { void SetInterruptFlag() const { static_cast(m_tls_heap_address)->interrupt_flag = 1; } void ClearInterruptFlag() const { static_cast(m_tls_heap_address)->interrupt_flag = 0; } + bool IsInUserCacheMaintenanceOperation() const { return static_cast(m_tls_heap_address)->cache_maintenance_flag != 0; } + ALWAYS_INLINE KAutoObject *GetClosedObject() { return m_closed_object; } ALWAYS_INLINE void SetClosedObject(KAutoObject *object) { @@ -606,6 +640,9 @@ namespace ams::kern { } size_t GetKernelStackUsage() const; + + void OnEnterUsermodeException(); + void OnLeaveUsermodeException(); public: /* Overridden parent functions. */ ALWAYS_INLINE u64 GetIdImpl() const { return this->GetThreadId(); } @@ -628,6 +665,7 @@ namespace ams::kern { static Result GetThreadList(s32 *out_num_threads, ams::kern::svc::KUserPointer out_thread_ids, s32 max_out_count); using ConditionVariableThreadTreeType = ConditionVariableThreadTree; + }; static_assert(alignof(KThread) == 0x10); @@ -636,7 +674,7 @@ namespace ams::kern { static_assert(ConditionVariableThreadTreeTraits::IsValid()); /* Check that the assembly offsets are valid. */ - static_assert(AMS_OFFSETOF(KThread, m_thread_context) == THREAD_THREAD_CONTEXT); + static_assert(AMS_OFFSETOF(KThread, m_kernel_stack_top) == THREAD_KERNEL_STACK_TOP); return true; } diff --git a/libmesosphere/include/mesosphere/kern_select_assembly_macros.h b/libmesosphere/include/mesosphere/kern_select_assembly_macros.h new file mode 100644 index 00000000..519863c8 --- /dev/null +++ b/libmesosphere/include/mesosphere/kern_select_assembly_macros.h @@ -0,0 +1,26 @@ +/* + * Copyright (c) Atmosphère-NX + * + * This program is free software; you can redistribute it and/or modify it + * under the terms and conditions of the GNU General Public License, + * version 2, as published by the Free Software Foundation. + * + * This program is distributed in the hope it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ +#pragma once + +#ifdef ATMOSPHERE_ARCH_ARM64 + + #include + +#else + + #error "Unknown architecture for CPU" + +#endif diff --git a/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp b/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp index 4d733406..52a2e5ce 100644 --- a/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp +++ b/libmesosphere/source/arch/arm64/kern_exception_handlers.cpp @@ -214,16 +214,16 @@ namespace ams::kern::arch::arm64 { context->psr = 0x10; } - /* Set exception SVC permissions. */ - cur_process.CopyEnterExceptionSvcPermissionsTo(GetCurrentThread().GetStackParametersForExceptionSvcPermission()); + /* Process that we're entering a usermode exception on the current thread. */ + GetCurrentThread().OnEnterUsermodeException(); return; } } /* If we should, clear the thread's state as single-step. */ #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - if (AMS_UNLIKELY(GetCurrentThread().IsSingleStep())) { - GetCurrentThread().ClearSingleStep(); + if (AMS_UNLIKELY(GetCurrentThread().IsHardwareSingleStep())) { + GetCurrentThread().ClearHardwareSingleStep(); cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(false).Store(); cpu::InstructionMemoryBarrier(); } @@ -388,8 +388,8 @@ namespace ams::kern::arch::arm64 { /* Try to leave the user exception. */ if (cur_process.LeaveUserException()) { - /* We left user exception. Alter our SVC permissions accordingly. */ - cur_process.CopyLeaveExceptionSvcPermissionsTo(cur_thread->GetStackParametersForExceptionSvcPermission()); + /* Process that we're leaving a usermode exception on the current thread. */ + GetCurrentThread().OnLeaveUsermodeException(); /* Copy the user context to the thread context. */ if (is_aarch64) { diff --git a/libmesosphere/source/arch/arm64/kern_k_debug.cpp b/libmesosphere/source/arch/arm64/kern_k_debug.cpp index 6f6b1334..690e3134 100644 --- a/libmesosphere/source/arch/arm64/kern_k_debug.cpp +++ b/libmesosphere/source/arch/arm64/kern_k_debug.cpp @@ -202,16 +202,13 @@ namespace ams::kern::arch::arm64 { /* Get the FPU registers, if required. */ if ((context_flags & ams::svc::ThreadContextFlag_Fpu) != 0) { static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters); - const u128 *f = t_ctx->GetFpuRegisters(); + const auto &caller_save = thread->GetCallerSaveFpuRegisters(); + const auto &callee_save = t_ctx->GetCalleeSaveFpuRegisters(); if (this->Is64Bit()) { - for (size_t i = 0; i < KThreadContext::NumFpuRegisters; ++i) { - out->v[i] = f[i]; - } + KThreadContext::GetFpuRegisters(out->v, caller_save.fpu64, callee_save.fpu64); } else { - for (size_t i = 0; i < KThreadContext::NumFpuRegisters / 2; ++i) { - out->v[i] = f[i]; - } + KThreadContext::GetFpuRegisters(out->v, caller_save.fpu32, callee_save.fpu32); for (size_t i = KThreadContext::NumFpuRegisters / 2; i < KThreadContext::NumFpuRegisters; ++i) { out->v[i] = 0; } @@ -240,7 +237,14 @@ namespace ams::kern::arch::arm64 { /* Set the FPU registers, if required. */ if ((context_flags & ams::svc::ThreadContextFlag_Fpu) != 0) { static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters); - t_ctx->SetFpuRegisters(ctx.v, this->Is64Bit()); + auto &caller_save = thread->GetCallerSaveFpuRegisters(); + auto &callee_save = t_ctx->GetCalleeSaveFpuRegisters(); + + if (this->Is64Bit()) { + KThreadContext::SetFpuRegisters(caller_save.fpu64, callee_save.fpu64, ctx.v); + } else { + KThreadContext::SetFpuRegisters(caller_save.fpu32, callee_save.fpu32, ctx.v); + } } R_SUCCEED(); diff --git a/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp b/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp index 95e91790..e4cf075d 100644 --- a/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp +++ b/libmesosphere/source/arch/arm64/kern_k_interrupt_manager.cpp @@ -178,19 +178,37 @@ namespace ams::kern::arch::arm64 { /* If we need scheduling, */ if (needs_scheduling) { - /* If the user disable count is set, we may need to pin the current thread. */ - if (user_mode && GetCurrentThread().GetUserDisableCount() != 0 && GetCurrentProcess().GetPinnedThread(GetCurrentCoreId()) == nullptr) { - KScopedSchedulerLock sl; + if (user_mode) { + /* If the interrupt occurred in the middle of a userland cache maintenance operation, ensure memory consistency before rescheduling. */ + if (GetCurrentThread().IsInUserCacheMaintenanceOperation()) { + cpu::DataSynchronizationBarrier(); + } - /* Pin the current thread. */ - GetCurrentProcess().PinCurrentThread(); + /* If the user disable count is set, we may need to pin the current thread. */ + if (GetCurrentThread().GetUserDisableCount() != 0 && GetCurrentProcess().GetPinnedThread(GetCurrentCoreId()) == nullptr) { + KScopedSchedulerLock sl; - /* Set the interrupt flag for the thread. */ - GetCurrentThread().SetInterruptFlag(); + /* Pin the current thread. */ + GetCurrentProcess().PinCurrentThread(); - /* Request interrupt scheduling. */ - Kernel::GetScheduler().RequestScheduleOnInterrupt(); + /* Set the interrupt flag for the thread. */ + GetCurrentThread().SetInterruptFlag(); + + /* Request interrupt scheduling. */ + Kernel::GetScheduler().RequestScheduleOnInterrupt(); + } else { + /* Request interrupt scheduling. */ + Kernel::GetScheduler().RequestScheduleOnInterrupt(); + } } else { + /* If the interrupt occurred in the middle of a cache maintenance operation, ensure memory consistency before rescheduling. */ + if (GetCurrentThread().IsInCacheMaintenanceOperation()) { + cpu::DataSynchronizationBarrier(); + } else if (GetCurrentThread().IsInTlbMaintenanceOperation()) { + /* Otherwise, if we're in the middle of a tlb maintenance operation, ensure inner shareable memory consistency before rescheduling. */ + cpu::DataSynchronizationBarrierInnerShareable(); + } + /* Request interrupt scheduling. */ Kernel::GetScheduler().RequestScheduleOnInterrupt(); } diff --git a/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp b/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp index 1fb245dd..321f4356 100644 --- a/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp +++ b/libmesosphere/source/arch/arm64/kern_k_thread_context.cpp @@ -52,11 +52,11 @@ namespace ams::kern::arch::arm64 { } uintptr_t SetupStackForUserModeThreadStarter(KVirtualAddress pc, KVirtualAddress k_sp, KVirtualAddress u_sp, uintptr_t arg, const bool is_64_bit) { - /* NOTE: Stack layout on entry looks like following: */ - /* SP */ - /* | */ - /* v */ - /* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x30) | */ + /* NOTE: Stack layout on entry looks like following: */ + /* SP */ + /* | */ + /* v */ + /* | KExceptionContext (size 0x120) | KThread::StackParameters (size 0x130) | */ KExceptionContext *ctx = GetPointer(k_sp) - 1; /* Clear context. */ @@ -92,12 +92,12 @@ namespace ams::kern::arch::arm64 { } uintptr_t SetupStackForSupervisorModeThreadStarter(KVirtualAddress pc, KVirtualAddress sp, uintptr_t arg) { - /* NOTE: Stack layout on entry looks like following: */ - /* SP */ - /* | */ - /* v */ - /* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x30) | */ - static_assert(sizeof(KThread::StackParameters) == 0x30); + /* NOTE: Stack layout on entry looks like following: */ + /* SP */ + /* | */ + /* v */ + /* | u64 argument | u64 entrypoint | KThread::StackParameters (size 0x130) | */ + static_assert(sizeof(KThread::StackParameters) == 0x130); u64 *stack = GetPointer(sp); *(--stack) = GetInteger(pc); @@ -142,9 +142,8 @@ namespace ams::kern::arch::arm64 { /* Clear FPU state. */ m_fpcr = 0; m_fpsr = 0; - m_cpacr = 0; - for (size_t i = 0; i < util::size(m_fpu_registers); i++) { - m_fpu_registers[i] = 0; + for (size_t i = 0; i < util::size(m_callee_saved_fpu.fpu64.v); ++i) { + m_callee_saved_fpu.fpu64.v[i] = 0; } /* Lock the context, if we're a main thread. */ @@ -153,37 +152,18 @@ namespace ams::kern::arch::arm64 { R_SUCCEED(); } - Result KThreadContext::Finalize() { - /* This doesn't actually do anything. */ - R_SUCCEED(); - } - void KThreadContext::SetArguments(uintptr_t arg0, uintptr_t arg1) { u64 *stack = reinterpret_cast(m_sp); stack[0] = arg0; stack[1] = arg1; } - void KThreadContext::FpuContextSwitchHandler(KThread *thread) { - MESOSPHERE_ASSERT(!KInterruptManager::AreInterruptsEnabled()); - MESOSPHERE_ASSERT(!IsFpuEnabled()); - - /* Enable the FPU. */ - EnableFpu(); - - /* Restore the FPU registers. */ - KProcess *process = thread->GetOwnerProcess(); - MESOSPHERE_ASSERT(process != nullptr); - if (process->Is64Bit()) { - RestoreFpuRegisters64(thread->GetContext()); - } else { - RestoreFpuRegisters32(thread->GetContext()); - } - } - void KThreadContext::CloneFpuStatus() { u64 pcr, psr; cpu::InstructionMemoryBarrier(); + + KScopedInterruptDisable di; + if (IsFpuEnabled()) { __asm__ __volatile__("mrs %[pcr], fpcr" : [pcr]"=r"(pcr) :: "memory"); __asm__ __volatile__("mrs %[psr], fpsr" : [psr]"=r"(psr) :: "memory"); @@ -196,18 +176,6 @@ namespace ams::kern::arch::arm64 { this->SetFpsr(psr); } - void KThreadContext::SetFpuRegisters(const u128 *v, bool is_64_bit) { - if (is_64_bit) { - for (size_t i = 0; i < KThreadContext::NumFpuRegisters; ++i) { - m_fpu_registers[i] = v[i]; - } - } else { - for (size_t i = 0; i < KThreadContext::NumFpuRegisters / 2; ++i) { - m_fpu_registers[i] = v[i]; - } - } - } - void GetUserContext(ams::svc::ThreadContext *out, const KThread *thread) { MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); MESOSPHERE_ASSERT(thread->IsSuspended()); @@ -244,9 +212,17 @@ namespace ams::kern::arch::arm64 { /* Copy fpu registers. */ static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters); - const u128 *f = t_ctx->GetFpuRegisters(); - for (size_t i = 0; i < KThreadContext::NumFpuRegisters; ++i) { - out->v[i] = f[i]; + static_assert(KThreadContext::NumCallerSavedFpuRegisters == KThreadContext::NumCalleeSavedFpuRegisters * 3); + static_assert(KThreadContext::NumFpuRegisters == KThreadContext::NumCallerSavedFpuRegisters + KThreadContext::NumCalleeSavedFpuRegisters); + const auto &caller_save_fpu = thread->GetCallerSaveFpuRegisters().fpu64; + const auto &callee_save_fpu = t_ctx->GetCalleeSaveFpuRegisters().fpu64; + + if (!thread->IsCallingSvc() || thread->IsInUsermodeExceptionHandler()) { + KThreadContext::GetFpuRegisters(out->v, caller_save_fpu, callee_save_fpu); + } else { + for (size_t i = 0; i < KThreadContext::NumCalleeSavedFpuRegisters; ++i) { + out->v[(KThreadContext::NumCallerSavedFpuRegisters / 3) + i] = caller_save_fpu.v[i]; + } } } else { /* Set special registers. */ @@ -271,12 +247,17 @@ namespace ams::kern::arch::arm64 { /* Copy fpu registers. */ static_assert(util::size(ams::svc::ThreadContext{}.v) == KThreadContext::NumFpuRegisters); - const u128 *f = t_ctx->GetFpuRegisters(); - for (size_t i = 0; i < KThreadContext::NumFpuRegisters / 2; ++i) { - out->v[i] = f[i]; - } - for (size_t i = KThreadContext::NumFpuRegisters / 2; i < KThreadContext::NumFpuRegisters; ++i) { - out->v[i] = 0; + static_assert(KThreadContext::NumCallerSavedFpuRegisters == KThreadContext::NumCalleeSavedFpuRegisters * 3); + static_assert(KThreadContext::NumFpuRegisters == KThreadContext::NumCallerSavedFpuRegisters + KThreadContext::NumCalleeSavedFpuRegisters); + const auto &caller_save_fpu = thread->GetCallerSaveFpuRegisters().fpu32; + const auto &callee_save_fpu = t_ctx->GetCalleeSaveFpuRegisters().fpu32; + + if (!thread->IsCallingSvc() || thread->IsInUsermodeExceptionHandler()) { + KThreadContext::GetFpuRegisters(out->v, caller_save_fpu, callee_save_fpu); + } else { + for (size_t i = 0; i < KThreadContext::NumCalleeSavedFpuRegisters / 2; ++i) { + out->v[((KThreadContext::NumCallerSavedFpuRegisters / 3) / 2) + i] = caller_save_fpu.v[i]; + } } } diff --git a/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s b/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s index 8a97715a..da0b65fb 100644 --- a/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s +++ b/libmesosphere/source/arch/arm64/svc/kern_svc_exception_asm.s @@ -13,7 +13,7 @@ * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ -#include +#include /* ams::kern::svc::CallReturnFromException64(Result result) */ .section .text._ZN3ams4kern3svc25CallReturnFromException64Ev, "ax", %progbits @@ -82,8 +82,20 @@ _ZN3ams4kern3svc14RestoreContextEm: b 0b 1: /* We're done with DPC, and should return from the svc. */ - /* Clear our in-SVC note. */ - strb wzr, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)] + + /* Get our exception flags. */ + ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* Clear in-svc and needs-fpu-restore flags. */ + and w10, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)) + and w10, w10, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC)) + strb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* If we don't need to restore the fpu, skip restoring it. */ + tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 3f + + /* Enable and restore the fpu. */ + ENABLE_AND_RESTORE_FPU(x10, x8, x9, w8, w9, 2, 3) /* Restore registers. */ ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)] diff --git a/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s b/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s index b7371c89..d795c6aa 100644 --- a/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s +++ b/libmesosphere/source/arch/arm64/svc/kern_svc_handlers_asm.s @@ -14,7 +14,7 @@ * along with this program. If not, see . */ #include -#include +#include /* ams::kern::arch::arm64::SvcHandler64() */ .section .text._ZN3ams4kern4arch5arm6412SvcHandler64Ev, "ax", %progbits @@ -81,9 +81,10 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: cbz x11, 3f /* Note that we're calling the SVC. */ - mov w10, #1 - strb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)] - strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)] + ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + orr w9, w9, #(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC) + strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)] /* If we should, trace the svc entry. */ #if defined(MESOSPHERE_BUILD_FOR_TRACING) @@ -111,7 +112,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: 2: /* We completed the SVC, and we should handle DPC. */ /* Check the dpc flags. */ ldrb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)] - cbz w8, 4f + cbz w8, 5f /* We have DPC to do! */ /* Save registers and call ams::kern::KDpcManager::HandleDpc(). */ @@ -150,7 +151,18 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: mov x0, sp bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE - /* Restore registers. */ + /* If we don't need to restore the fpu, skip restoring it. */ + ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 4f + + /* Clear the needs-fpu-restore flag. */ + and w9, w9, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED) + strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* Enable and restore the fpu. */ + ENABLE_AND_RESTORE_FPU64(x10, x8, x9, w8, w9) + +4: /* Restore registers. */ ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)] ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)] @@ -184,9 +196,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: add sp, sp, #(EXCEPTION_CONTEXT_SIZE) eret -4: /* Return from SVC. */ - /* Clear our in-SVC note. */ - strb wzr, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)] +5: /* Return from SVC. */ /* If we should, trace the svc exit. */ #if defined(MESOSPHERE_BUILD_FOR_TRACING) @@ -204,7 +214,60 @@ _ZN3ams4kern4arch5arm6412SvcHandler64Ev: add sp, sp, #0x40 #endif - /* Restore registers. */ + /* Get our exception flags. */ + ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* Clear in-svc and needs-fpu-restore flags. */ + and w8, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)) + and w8, w8, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC)) + strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* If we don't need to restore the fpu, skip restoring it. */ + tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 7f + + /* If we need to restore the fpu, check if we need to do a full restore. */ + tbnz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 6f + + /* Enable the fpu. */ + ENABLE_FPU(x8) + + /* Get the thread context and restore fpsr/fpcr. */ + GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(x10, x8, x9, w8, w9) + + /* Restore callee-saved registers to 64-bit fpu. */ + RESTORE_FPU64_CALLEE_SAVE_REGISTERS(x10) + + /* Clear caller-saved registers to 64-bit fpu. */ + movi v0.2d, #0 + movi v1.2d, #0 + movi v2.2d, #0 + movi v3.2d, #0 + movi v4.2d, #0 + movi v5.2d, #0 + movi v6.2d, #0 + movi v7.2d, #0 + movi v16.2d, #0 + movi v17.2d, #0 + movi v18.2d, #0 + movi v19.2d, #0 + movi v20.2d, #0 + movi v21.2d, #0 + movi v22.2d, #0 + movi v23.2d, #0 + movi v24.2d, #0 + movi v25.2d, #0 + movi v26.2d, #0 + movi v27.2d, #0 + movi v28.2d, #0 + movi v29.2d, #0 + movi v30.2d, #0 + movi v31.2d, #0 + b 7f + +6: /* We need to do a full fpu restore. */ + ENABLE_AND_RESTORE_FPU64(x10, x8, x9, w8, w9) + +7: /* Restore registers. */ ldp x30, x8, [sp, #(EXCEPTION_CONTEXT_X30_SP)] ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)] @@ -273,38 +336,40 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: stp x14, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)] /* Check if the SVC index is out of range. */ - mrs x16, esr_el1 - and x16, x16, #0xFF - cmp x16, #(AMS_KERN_NUM_SUPERVISOR_CALLS) + mrs x8, esr_el1 + and x8, x8, #0xFF + cmp x8, #(AMS_KERN_NUM_SUPERVISOR_CALLS) b.ge 3f /* Check the specific SVC permission bit for allowal. */ - mov x20, sp - add x20, x20, x16, lsr#3 - ldrb w20, [x20, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_SVC_PERMISSION)] - and x17, x16, #0x7 - lsr x17, x20, x17 - tst x17, #1 + mov x9, sp + add x9, x9, x8, lsr#3 + ldrb w9, [x9, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_SVC_PERMISSION)] + and x10, x8, #0x7 + lsr x10, x9, x10 + tst x10, #1 b.eq 3f /* Check if our disable count allows us to call SVCs. */ - mrs x15, tpidrro_el0 - ldrh w15, [x15, #(THREAD_LOCAL_REGION_DISABLE_COUNT)] - cbz w15, 1f + mrs x10, tpidrro_el0 + ldrh w10, [x10, #(THREAD_LOCAL_REGION_DISABLE_COUNT)] + cbz w10, 1f /* It might not, so check the stack params to see if we must not allow the SVC. */ - ldrb w15, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_PINNED)] - cbz w15, 3f + ldrb w10, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_PINNED)] + cbz w10, 3f 1: /* We can call the SVC. */ - adr x15, _ZN3ams4kern3svc16SvcTable64From32E - ldr x19, [x15, x16, lsl#3] - cbz x19, 3f + adr x10, _ZN3ams4kern3svc16SvcTable64From32E + ldr x11, [x10, x8, lsl#3] + cbz x11, 3f + /* Note that we're calling the SVC. */ - mov w15, #1 - strb w15, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)] - strb w16, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)] + ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + orr w9, w9, #(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC) + strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_CURRENT_SVC_ID)] /* If we should, trace the svc entry. */ #if defined(MESOSPHERE_BUILD_FOR_TRACING) @@ -313,26 +378,26 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: stp x2, x3, [sp, #(8 * 2)] stp x4, x5, [sp, #(8 * 4)] stp x6, x7, [sp, #(8 * 6)] - str x19, [sp, #(8 * 8)] + str x11, [sp, #(8 * 8)] mov x0, sp bl _ZN3ams4kern3svc13TraceSvcEntryEPKm ldp x0, x1, [sp, #(8 * 0)] ldp x2, x3, [sp, #(8 * 2)] ldp x4, x5, [sp, #(8 * 4)] ldp x6, x7, [sp, #(8 * 6)] - ldr x19, [sp, #(8 * 8)] + ldr x11, [sp, #(8 * 8)] add sp, sp, #0x50 #endif /* Invoke the SVC handler. */ msr daifclr, #2 - blr x19 + blr x11 msr daifset, #2 2: /* We completed the SVC, and we should handle DPC. */ /* Check the dpc flags. */ - ldrb w16, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)] - cbz w16, 4f + ldrb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_DPC_FLAGS)] + cbz w8, 5f /* We have DPC to do! */ /* Save registers and call ams::kern::KDpcManager::HandleDpc(). */ @@ -368,18 +433,29 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: mov x0, sp bl _ZN3ams4kern4arch5arm6415HandleExceptionEPNS2_17KExceptionContextE - /* Restore registers. */ - ldp x17, x20, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] - ldr x19, [sp, #(EXCEPTION_CONTEXT_TPIDR)] + /* If we don't need to restore the fpu, skip restoring it. */ + ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 4f + + /* Clear the needs-fpu-restore flag. */ + and w9, w9, #(~THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED) + strb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* Enable and restore the fpu. */ + ENABLE_AND_RESTORE_FPU32(x10, x8, x9, w8, w9) + +4: /* Restore registers. */ + ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] + ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)] #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) /* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */ - bic x20, x20, #(1 << 21) + bic x10, x10, #(1 << 21) #endif - msr elr_el1, x17 - msr spsr_el1, x20 - msr tpidr_el0, x19 + msr elr_el1, x9 + msr spsr_el1, x10 + msr tpidr_el0, x11 ldp x0, x1, [sp, #(EXCEPTION_CONTEXT_X0_X1)] ldp x2, x3, [sp, #(EXCEPTION_CONTEXT_X2_X3)] ldp x4, x5, [sp, #(EXCEPTION_CONTEXT_X4_X5)] @@ -393,9 +469,7 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: add sp, sp, #(EXCEPTION_CONTEXT_SIZE) eret -4: /* Return from SVC. */ - /* Clear our in-SVC note. */ - strb wzr, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_IS_CALLING_SVC)] +5: /* Return from SVC. */ /* If we should, trace the svc exit. */ #if defined(MESOSPHERE_BUILD_FOR_TRACING) @@ -413,22 +487,63 @@ _ZN3ams4kern4arch5arm6412SvcHandler32Ev: add sp, sp, #0x40 #endif - /* Restore registers. */ + /* Get our exception flags. */ + ldrb w9, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* Clear in-svc and needs-fpu-restore flags. */ + and w8, w9, #(~(THREAD_EXCEPTION_FLAG_IS_FPU_CONTEXT_RESTORE_NEEDED)) + and w8, w8, #(~(THREAD_EXCEPTION_FLAG_IS_CALLING_SVC)) + strb w8, [sp, #(EXCEPTION_CONTEXT_SIZE + THREAD_STACK_PARAMETERS_EXCEPTION_FLAGS)] + + /* If we don't need to restore the fpu, skip restoring it. */ + tbz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_FPU_CONTEXT_RESTORE_NEEDED), 7f + + /* If we need to restore the fpu, check if we need to do a full restore. */ + tbnz w9, #(THREAD_EXCEPTION_FLAG_BIT_INDEX_IS_IN_USERMODE_EXCEPTION_HANDLER), 6f + + /* Enable the fpu. */ + ENABLE_FPU(x8) + + /* Get the thread context and restore fpsr/fpcr. */ + GET_THREAD_CONTEXT_AND_RESTORE_FPCR_FPSR(x10, x8, x9, w8, w9) + + /* Restore callee-saved registers to 32-bit fpu. */ + RESTORE_FPU32_CALLEE_SAVE_REGISTERS(x10) + + /* Clear caller-saved registers to 32-bit fpu. */ + movi v0.2d, #0 + movi v1.2d, #0 + movi v2.2d, #0 + movi v3.2d, #0 + movi v8.2d, #0 + movi v9.2d, #0 + movi v10.2d, #0 + movi v11.2d, #0 + movi v12.2d, #0 + movi v13.2d, #0 + movi v14.2d, #0 + movi v15.2d, #0 + b 7f + +6: /* We need to do a full fpu restore. */ + ENABLE_AND_RESTORE_FPU32(x10, x8, x9, w8, w9) + +7: /* Restore registers. */ + ldp x9, x10, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] + ldr x11, [sp, #(EXCEPTION_CONTEXT_TPIDR)] + + #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) + /* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */ + bic x10, x10, #(1 << 21) + #endif + + msr elr_el1, x9 + msr spsr_el1, x10 + msr tpidr_el0, x11 ldp x8, x9, [sp, #(EXCEPTION_CONTEXT_X8_X9)] ldp x10, x11, [sp, #(EXCEPTION_CONTEXT_X10_X11)] ldp x12, x13, [sp, #(EXCEPTION_CONTEXT_X12_X13)] ldp x14, xzr, [sp, #(EXCEPTION_CONTEXT_X14_X15)] - ldp x17, x20, [sp, #(EXCEPTION_CONTEXT_PC_PSR)] - ldr x19, [sp, #(EXCEPTION_CONTEXT_TPIDR)] - - #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) - /* Since we're returning from an SVC, make sure SPSR.SS is cleared so that if we're single-stepping we break instantly on the instruction after the SVC. */ - bic x20, x20, #(1 << 21) - #endif - - msr elr_el1, x17 - msr spsr_el1, x20 - msr tpidr_el0, x19 /* Return. */ add sp, sp, #(EXCEPTION_CONTEXT_SIZE) diff --git a/libmesosphere/source/kern_k_debug_base.cpp b/libmesosphere/source/kern_k_debug_base.cpp index 5493e62f..c971a94b 100644 --- a/libmesosphere/source/kern_k_debug_base.cpp +++ b/libmesosphere/source/kern_k_debug_base.cpp @@ -447,7 +447,7 @@ namespace ams::kern { { auto end = target->GetThreadList().end(); for (auto it = target->GetThreadList().begin(); it != end; ++it) { - it->ClearSingleStep(); + it->ClearHardwareSingleStep(); } } #endif @@ -595,13 +595,13 @@ namespace ams::kern { { if ((context_flags & ams::svc::ThreadContextFlag_SetSingleStep) != 0) { /* Set single step. */ - thread->SetSingleStep(); + thread->SetHardwareSingleStep(); /* If no other thread flags are present, we're done. */ R_SUCCEED_IF((context_flags & ~ams::svc::ThreadContextFlag_SetSingleStep) == 0); } else if ((context_flags & ams::svc::ThreadContextFlag_ClearSingleStep) != 0) { /* Clear single step. */ - thread->ClearSingleStep(); + thread->ClearHardwareSingleStep(); /* If no other thread flags are present, we're done. */ R_SUCCEED_IF((context_flags & ~ams::svc::ThreadContextFlag_ClearSingleStep) == 0); @@ -1022,7 +1022,7 @@ namespace ams::kern { for (auto it = process->GetThreadList().begin(); it != end; ++it) { #if defined(MESOSPHERE_ENABLE_HARDWARE_SINGLE_STEP) /* Clear the thread's single-step state. */ - it->ClearSingleStep(); + it->ClearHardwareSingleStep(); #endif if (resume) { diff --git a/libmesosphere/source/kern_k_scheduler.cpp b/libmesosphere/source/kern_k_scheduler.cpp index df9ec681..810ed1c3 100644 --- a/libmesosphere/source/kern_k_scheduler.cpp +++ b/libmesosphere/source/kern_k_scheduler.cpp @@ -255,7 +255,7 @@ namespace ams::kern { /* in EL0...which implies a return-from-exception has occurred since we set the bit. Thus, forcing */ /* an ISB is unnecessary, and we can modify the register safely and be confident it will affect the next */ /* userland instruction executed. */ - cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(next_thread->IsSingleStep()).Store(); + cpu::MonitorDebugSystemControlRegisterAccessor().SetSoftwareStep(next_thread->IsHardwareSingleStep()).Store(); #endif /* Switch the current process, if we're switching processes. */ diff --git a/libmesosphere/source/kern_k_thread.cpp b/libmesosphere/source/kern_k_thread.cpp index a4f06e23..38f7f8a8 100644 --- a/libmesosphere/source/kern_k_thread.cpp +++ b/libmesosphere/source/kern_k_thread.cpp @@ -68,6 +68,91 @@ namespace ams::kern { } + ALWAYS_INLINE void KThread::SetPinnedSvcPermissions() { + /* Get our stack parameters. */ + auto &sp = this->GetStackParameters(); + + /* Get our parent's svc permissions. */ + MESOSPHERE_ASSERT(m_parent != nullptr); + const auto &svc_permissions = m_parent->GetSvcPermissions(); + + /* Get whether we have access to return from exception. */ + const bool return_from_exception = sp.svc_access_flags[svc::SvcId_ReturnFromException]; + + /* Clear all permissions. */ + sp.svc_access_flags.Reset(); + + /* Set SynchronizePreemptionState if allowed. */ + if (svc_permissions[svc::SvcId_SynchronizePreemptionState]) { + sp.svc_access_flags[svc::SvcId_SynchronizePreemptionState] = true; + } + + /* If we previously had ReturnFromException, potentially grant it and GetInfo. */ + if (return_from_exception) { + /* Set ReturnFromException (guaranteed allowed, if we're here). */ + sp.svc_access_flags[svc::SvcId_ReturnFromException] = true; + + /* Set GetInfo if allowed. */ + if (svc_permissions[svc::SvcId_GetInfo]) { + sp.svc_access_flags[svc::SvcId_GetInfo] = true; + } + } + } + + ALWAYS_INLINE void KThread::SetUnpinnedSvcPermissions() { + /* Get our stack parameters. */ + auto &sp = this->GetStackParameters(); + + /* Get our parent's svc permissions. */ + MESOSPHERE_ASSERT(m_parent != nullptr); + const auto &svc_permissions = m_parent->GetSvcPermissions(); + + /* Get whether we have access to return from exception. */ + const bool return_from_exception = sp.svc_access_flags[svc::SvcId_ReturnFromException]; + + /* Copy permissions. */ + sp.svc_access_flags = svc_permissions; + + /* Clear specific SVCs based on our state. */ + sp.svc_access_flags[svc::SvcId_SynchronizePreemptionState] = false; + + if (!return_from_exception) { + sp.svc_access_flags[svc::SvcId_ReturnFromException] = false; + } + } + + ALWAYS_INLINE void KThread::SetUsermodeExceptionSvcPermissions() { + /* Get our stack parameters. */ + auto &sp = this->GetStackParameters(); + + /* Get our parent's svc permissions. */ + MESOSPHERE_ASSERT(m_parent != nullptr); + const auto &svc_permissions = m_parent->GetSvcPermissions(); + + /* Set ReturnFromException if allowed. */ + if (svc_permissions[svc::SvcId_ReturnFromException]) { + sp.svc_access_flags[svc::SvcId_ReturnFromException] = true; + } + + /* Set GetInfo if allowed. */ + if (svc_permissions[svc::SvcId_GetInfo]) { + sp.svc_access_flags[svc::SvcId_GetInfo] = true; + } + } + + ALWAYS_INLINE void KThread::ClearUsermodeExceptionSvcPermissions() { + /* Get our stack parameters. */ + auto &sp = this->GetStackParameters(); + + /* Clear ReturnFromException. */ + sp.svc_access_flags[svc::SvcId_ReturnFromException] = false; + + /* If pinned, clear GetInfo. */ + if (sp.is_pinned) { + sp.svc_access_flags[svc::SvcId_GetInfo] = false; + } + } + Result KThread::Initialize(KThreadFunction func, uintptr_t arg, void *kern_stack_top, KProcessAddress user_stack_top, s32 prio, s32 virt_core, KProcess *owner, ThreadType type) { /* Assert parameters are valid. */ MESOSPHERE_ASSERT_THIS(); @@ -209,18 +294,23 @@ namespace ams::kern { const bool is_64_bit = m_parent ? m_parent->Is64Bit() : IsDefault64Bit; const bool is_user = (type == ThreadType_User); const bool is_main = (type == ThreadType_Main); - m_thread_context.Initialize(reinterpret_cast(func), reinterpret_cast(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main); + this->GetContext().Initialize(reinterpret_cast(func), reinterpret_cast(this->GetStackTop()), GetInteger(user_stack_top), arg, is_user, is_64_bit, is_main); /* Setup the stack parameters. */ StackParameters &sp = this->GetStackParameters(); if (m_parent != nullptr) { - m_parent->CopySvcPermissionsTo(sp); + this->SetUnpinnedSvcPermissions(); + this->ClearUsermodeExceptionSvcPermissions(); } - sp.context = std::addressof(m_thread_context); - sp.cur_thread = this; - sp.disable_count = 1; + sp.caller_save_fpu_registers = std::addressof(m_caller_save_fpu_registers); + sp.cur_thread = this; + sp.disable_count = 1; this->SetInExceptionHandler(); + if (m_parent != nullptr && is_64_bit) { + this->SetFpu64Bit(); + } + /* Set thread ID. */ m_thread_id = g_thread_id++; @@ -329,9 +419,6 @@ namespace ams::kern { } } - /* Finalize the thread context. */ - m_thread_context.Finalize(); - /* Cleanup the kernel stack. */ if (m_kernel_stack_top != nullptr) { CleanupKernelStack(reinterpret_cast(m_kernel_stack_top)); @@ -411,6 +498,16 @@ namespace ams::kern { this->FinishTermination(); } + void KThread::OnEnterUsermodeException() { + this->SetUsermodeExceptionSvcPermissions(); + this->SetInUsermodeExceptionHandler(); + } + + void KThread::OnLeaveUsermodeException() { + this->ClearUsermodeExceptionSvcPermissions(); + this->ClearInUsermodeExceptionHandler(); + } + void KThread::Pin() { MESOSPHERE_ASSERT_THIS(); MESOSPHERE_ASSERT(KScheduler::IsSchedulerLockedByCurrentThread()); @@ -458,8 +555,7 @@ namespace ams::kern { } /* Update our SVC access permissions. */ - MESOSPHERE_ASSERT(m_parent != nullptr); - m_parent->CopyPinnedSvcPermissionsTo(this->GetStackParameters()); + this->SetPinnedSvcPermissions(); } void KThread::Unpin() { @@ -507,7 +603,7 @@ namespace ams::kern { /* Update our SVC access permissions. */ MESOSPHERE_ASSERT(m_parent != nullptr); - m_parent->CopyUnpinnedSvcPermissionsTo(this->GetStackParameters()); + this->SetUnpinnedSvcPermissions(); } /* Resume any threads that began waiting on us while we were pinned. */ diff --git a/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp b/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp index e7b457a7..e32c9a57 100644 --- a/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp +++ b/libvapours/include/vapours/svc/arch/arm64/svc_thread_local_region.hpp @@ -25,9 +25,13 @@ namespace ams::svc::arch::arm64 { u32 message_buffer[MessageBufferSize / sizeof(u32)]; volatile u16 disable_count; volatile u16 interrupt_flag; + volatile u8 cache_maintenance_flag; /* TODO: How should we handle libnx vs Nintendo user thread local space? */ uintptr_t TODO[(0x200 - 0x108) / sizeof(uintptr_t)]; }; + static_assert(__builtin_offsetof(ThreadLocalRegion, disable_count) == 0x100); + static_assert(__builtin_offsetof(ThreadLocalRegion, interrupt_flag) == 0x102); + static_assert(__builtin_offsetof(ThreadLocalRegion, cache_maintenance_flag) == 0x104); ALWAYS_INLINE ThreadLocalRegion *GetThreadLocalRegion() { ThreadLocalRegion *tlr;