diff --git a/src/ui/summary-ui-handler.ts b/src/ui/summary-ui-handler.ts
index 452cbdb8418..17012182496 100644
--- a/src/ui/summary-ui-handler.ts
+++ b/src/ui/summary-ui-handler.ts
@@ -485,10 +485,10 @@ export default class SummaryUiHandler extends UiHandler {
         if (this.summaryUiMode === SummaryUiMode.LEARN_MOVE) {
           this.hideMoveSelect();
         } else {
-          if (!ui.getMessageHandler().onActionInput) {
-            ui.setMode(Mode.PARTY);
-          } else {
+          if (ui.getMessageHandler().onActionInput) {
             ui.setMode(Mode.MESSAGE);
+          } else {
+            ui.setMode(Mode.PARTY);
           }
         }
         success = true;
@@ -499,6 +499,8 @@ export default class SummaryUiHandler extends UiHandler {
         case Button.DOWN:
           if (this.summaryUiMode === SummaryUiMode.LEARN_MOVE) {
             break;
+          } else if (ui.getMessageHandler().onActionInput) {
+            break;
           }
           const isDown = button === Button.DOWN;
           const party = this.scene.getParty();