Added env AppletFlags handling. Don't enter the msg-loop for AppletType_Application when the output from _appletGetCurrentFocusState is already 1. This fixes the hang issue with AppletType_Application where applet init with AppletType_Application was previously done. Moved AppletType_*Application checks into a dedicated function which also checks for AppletType_SystemApplication, hence some of these funcs which didn't check for AppletType_SystemApplication now support it.

nx/include/switch/runtime/env.h

@@ -35,6 +35,10 @@ enum {
     EntryType_LastLoadResult=11       ///< Provides the last load result.
 };
 
+enum {
+    EnvAppletFlags_ApplicationOverride = BIT(0) ///< Use AppletType_Application instead of AppletType_SystemApplication.
+};
+
 /// Loader return function.
 typedef void NORETURN (*LoaderReturnFn)(int result_code);
 

nx/source/runtime/env.c

@@ -3,6 +3,7 @@
 #include "runtime/env.h"
 #include "services/sm.h"
 #include "services/fatal.h"
+#include "services/applet.h"
 
 void NORETURN __nx_exit(Result rc, LoaderReturnFn retaddr);
 
@@ -79,6 +80,7 @@ void envSetup(void* ctx, Handle main_thread, LoaderReturnFn saved_lr)
 
         case EntryType_AppletType:
             __nx_applet_type = ent->Value[0];
+            if ((ent->Value[1] & EnvAppletFlags_ApplicationOverride) && __nx_applet_type == AppletType_SystemApplication) __nx_applet_type = AppletType_Application;
             break;
 
         case EntryType_ProcessHandle:

nx/source/services/applet.c

@@ -186,30 +186,34 @@ Result appletInitialize(void)
 
     if (R_SUCCEEDED(rc) && (__nx_applet_type == AppletType_Application))
     {
-        do {
+        rc = _appletGetCurrentFocusState(&g_appletFocusState);
-            svcWaitSynchronizationSingle(g_appletMessageEventHandle, U64_MAX);
-            //When applet was previously initialized in the context of the current process for AppletType_Application, there's exactly 1 issue with initializing again: this loop hangs since there's no message available. If a timeout is added to the above waitsync where the loop is exited on timeout when _appletGetCurrentFocusState output is 1, initialization works fine.
 
-            u32 msg;
+        //Don't enter this msg-loop when g_appletFocusState is already 1, it will hang when applet was previously initialized in the context of the current process for AppletType_Application.
-            rc = _appletReceiveMessage(&msg);
+        if (R_SUCCEEDED(rc) && g_appletFocusState!=1) {
+            do {
+                svcWaitSynchronizationSingle(g_appletMessageEventHandle, U64_MAX);
 
-            if (R_FAILED(rc))
+                u32 msg;
-            {
+                rc = _appletReceiveMessage(&msg);
-                if ((rc & 0x3fffff) == 0x680)
+
+                if (R_FAILED(rc))
+                {
+                    if ((rc & 0x3fffff) == 0x680)
+                        continue;
+
+                    break;
+                }
+
+                if (msg != 0xF)
                     continue;
 
-                break;
+                rc = _appletGetCurrentFocusState(&g_appletFocusState);
-            }
 
-            if (msg != 0xF)
+                if (R_FAILED(rc))
-                continue;
+                    break;
 
-            rc = _appletGetCurrentFocusState(&g_appletFocusState);
+            } while(g_appletFocusState!=1);
-
+        }
-            if (R_FAILED(rc))
-                break;
-
-        } while(g_appletFocusState!=1);
 
         if (R_SUCCEEDED(rc))
             rc = _appletAcquireForegroundRights();
@@ -260,13 +264,16 @@ static void NORETURN _appletExitProcess(int result_code) {
     __builtin_unreachable();
 }
 
+static bool _appletIsApplication(void) {
+    return __nx_applet_type == AppletType_Application || __nx_applet_type == AppletType_SystemApplication;
+}
+
 void appletExit(void)
 {
     if (atomicDecrement64(&g_refCnt) == 0)
     {
         if ((envIsNso() && __nx_applet_exit_mode==0) || __nx_applet_exit_mode==1) {
-            if (__nx_applet_type == AppletType_Application ||
+            if (_appletIsApplication() ||
-                __nx_applet_type == AppletType_SystemApplication ||
                 __nx_applet_type == AppletType_LibraryApplet) {
                 if (!g_appletExitProcessFlag) {
                     g_appletExitProcessFlag = 1;
@@ -275,7 +282,7 @@ void appletExit(void)
                     return;
                 }
                 else {
-                    if (__nx_applet_type == AppletType_Application || __nx_applet_type == AppletType_SystemApplication) {
+                    if (_appletIsApplication()) {
                         //_appletSetTerminateResult(0);
                         _appletSelfExit();
                     }
@@ -564,7 +571,7 @@ Result appletGetDesiredLanguage(u64 *LanguageCode) {
     IpcCommand c;
     ipcInitialize(&c);
 
-    if (!serviceIsActive(&g_appletSrv) || __nx_applet_type != AppletType_Application)
+    if (!serviceIsActive(&g_appletSrv) || !_appletIsApplication())
         return MAKERESULT(Module_Libnx, LibnxError_NotInitialized);
 
     struct {
@@ -603,8 +610,7 @@ Result appletBeginBlockingHomeButton(s64 val) {
     IpcCommand c;
     ipcInitialize(&c);
 
-    if (!serviceIsActive(&g_appletSrv) || (__nx_applet_type!=AppletType_Application
+    if (!serviceIsActive(&g_appletSrv) || !_appletIsApplication())
-      && __nx_applet_type!=AppletType_SystemApplication))
         return MAKERESULT(Module_Libnx, LibnxError_NotInitialized);
 
     struct {
@@ -640,8 +646,7 @@ Result appletEndBlockingHomeButton(void) {
     IpcCommand c;
     ipcInitialize(&c);
 
-    if (!serviceIsActive(&g_appletSrv) || (__nx_applet_type!=AppletType_Application
+    if (!serviceIsActive(&g_appletSrv) || !_appletIsApplication())
-      && __nx_applet_type!=AppletType_SystemApplication))
         return MAKERESULT(Module_Libnx, LibnxError_NotInitialized);
 
     struct {