klippy: Rework shutdown handling

If an MCU signals a shutdown from the background thread, notify the
main thread and handle the shutdown there.  Dispatch shutdown handling
from the main Printer() class instead of from the Toolhead class.
This simplifies the shutdown logic.

Signed-off-by: Kevin O'Connor <kevin@koconnor.net>

4 files changed, 72 insertions, 63 deletions

diff --git a/klippy/gcode.py b/klippy/gcode.py
index 37e1c2af..12101331 100644
--- a/klippy/gcode.py
+++ b/klippy/gcode.py
@@ -23,8 +23,9 @@ class GCodeParser:
         self.bytes_read = 0
         self.input_log = collections.deque([], 50)
         # Command handling
-        self.gcode_handlers = self.build_handlers(False)
         self.is_printer_ready = False
+        self.gcode_handlers = {}
+        self.build_handlers()
         self.need_ack = False
         self.toolhead = self.fan = self.extruder = None
         self.heaters = []
@@ -34,28 +35,21 @@ class GCodeParser:
         self.last_position = [0.0, 0.0, 0.0, 0.0]
         self.homing_add = [0.0, 0.0, 0.0, 0.0]
         self.axis2pos = {'X': 0, 'Y': 1, 'Z': 2, 'E': 3}
-    def build_handlers(self, is_ready):
+    def build_handlers(self):
         handlers = self.all_handlers
-        if not is_ready:
+        if not self.is_printer_ready:
             handlers = [h for h in handlers
                         if getattr(self, 'cmd_'+h+'_when_not_ready', False)]
         gcode_handlers = { h: getattr(self, 'cmd_'+h) for h in handlers }
         for h, f in list(gcode_handlers.items()):
             aliases = getattr(self, 'cmd_'+h+'_aliases', [])
             gcode_handlers.update({ a: f for a in aliases })
-        return gcode_handlers
+        self.gcode_handlers = gcode_handlers
     def stats(self, eventtime):
         return "gcodein=%d" % (self.bytes_read,)
-    def set_printer_ready(self, is_ready):
-        if self.is_printer_ready == is_ready:
-            return
-        self.is_printer_ready = is_ready
-        self.gcode_handlers = self.build_handlers(is_ready)
-        if not is_ready:
-            # Printer is shutdown (could be running in a background thread)
-            if self.is_fileinput:
-                self.printer.request_exit()
-            return
+    def connect(self):
+        self.is_printer_ready = True
+        self.build_handlers()
         # Lookup printer components
         self.toolhead = self.printer.objects.get('toolhead')
         extruders = extruder.get_printer_extruders(self.printer)
@@ -67,6 +61,14 @@ class GCodeParser:
         self.fan = self.printer.objects.get('fan')
         if self.is_fileinput and self.fd_handle is None:
             self.fd_handle = self.reactor.register_fd(self.fd, self.process_data)
+    def do_shutdown(self):
+        if not self.is_printer_ready:
+            return
+        self.is_printer_ready = False
+        self.build_handlers()
+        self.dump_debug()
+        if self.is_fileinput:
+            self.printer.request_exit()
     def motor_heater_off(self):
         if self.toolhead is None:
             return
@@ -114,12 +116,10 @@ class GCodeParser:
             except error as e:
                 self.respond_error(str(e))
             except:
-                logging.exception("Exception in command handler")
-                self.toolhead.force_shutdown()
-                self.respond_error('Internal error on command:"%s"' % (cmd,))
-                if self.is_fileinput:
-                    self.printer.request_exit()
-                    break
+                msg = 'Internal error on command:"%s"' % (cmd,)
+                logging.exception(msg)
+                self.printer.invoke_shutdown(msg)
+                self.respond_error(msg)
             self.ack()
         self.need_ack = prev_need_ack
     def process_data(self, eventtime):
@@ -396,7 +396,7 @@ class GCodeParser:
         self.set_temp(params, wait=True)
     def cmd_M112(self, params):
         # Emergency Stop
-        self.toolhead.force_shutdown()
+        self.printer.invoke_shutdown("Shutdown due to M112 command")
     cmd_M114_when_not_ready = True
     def cmd_M114(self, params):
         # Get Current Position
diff --git a/klippy/klippy.py b/klippy/klippy.py
index ff3b2280..cdcdbedb 100644
--- a/klippy/klippy.py
+++ b/klippy/klippy.py
@@ -136,18 +136,15 @@ class Printer:
         self.connect_timer = self.reactor.register_timer(
             self._connect, self.reactor.NOW)
         self.all_config_options = {}
-        self.need_dump_debug = False
         self.state_message = message_startup
+        self.is_shutdown = False
+        self.async_shutdown_msg = ""
         self.run_result = None
         self.fileconfig = None
         self.mcus = []
     def get_start_args(self):
         return self.start_args
     def _stats(self, eventtime, force_output=False):
-        if self.need_dump_debug:
-            # Call dump_debug here so it is executed in the main thread
-            self.gcode.dump_debug()
-            self.need_dump_debug = False
         toolhead = self.objects.get('toolhead')
         if toolhead is None:
             return eventtime + 1.
@@ -196,7 +193,7 @@ class Printer:
             self._load_config()
             for m in self.mcus:
                 m.connect()
-            self.gcode.set_printer_ready(True)
+            self.gcode.connect()
             self.state_message = message_ready
             if self.start_args.get('debugoutput') is None:
                 self.reactor.update_timer(self.stats_timer, self.reactor.NOW)
@@ -219,34 +216,46 @@ class Printer:
         monotime = self.reactor.monotonic()
         logging.info("Start printer at %s (%.1f %.1f)",
                      time.asctime(time.localtime(systime)), systime, monotime)
-        # Enter main reactor loop
-        try:
-            self.reactor.run()
-        except:
-            logging.exception("Unhandled exception during run")
-            return "exit"
-        # Check restart flags
-        run_result = self.run_result
-        try:
-            self._stats(self.reactor.monotonic(), force_output=True)
-            for m in self.mcus:
-                if run_result == 'firmware_restart':
-                    m.microcontroller_restart()
-                m.disconnect()
-        except:
-            logging.exception("Unhandled exception during post run")
-        return run_result
+        while 1:
+            # Enter main reactor loop
+            try:
+                self.reactor.run()
+            except:
+                logging.exception("Unhandled exception during run")
+                return "exit"
+            # Check restart flags
+            run_result = self.run_result
+            try:
+                if run_result == 'shutdown':
+                    self.invoke_shutdown(self.async_shutdown_msg, True)
+                    continue
+                self._stats(self.reactor.monotonic(), force_output=True)
+                for m in self.mcus:
+                    if run_result == 'firmware_restart':
+                        m.microcontroller_restart()
+                    m.disconnect()
+            except:
+                logging.exception("Unhandled exception during post run")
+            return run_result
     def get_state_message(self):
         return self.state_message
-    def note_shutdown(self, msg):
-        if self.state_message == message_ready:
-            self.need_dump_debug = True
-        self.state_message = "%s%s" % (msg, message_shutdown)
-        self.gcode.set_printer_ready(False)
-    def note_mcu_error(self, msg):
-        self.state_message = "%s%s" % (msg, message_restart)
-        self.gcode.set_printer_ready(False)
-        self.gcode.motor_heater_off()
+    def invoke_shutdown(self, msg, is_mcu_shutdown=False):
+        if self.is_shutdown:
+            return
+        self.is_shutdown = True
+        if is_mcu_shutdown:
+            self.state_message = "%s%s" % (msg, message_shutdown)
+        else:
+            self.state_message = "%s%s" % (msg, message_restart)
+        for m in self.mcus:
+            m.do_shutdown()
+        self.gcode.do_shutdown()
+        toolhead = self.objects.get('toolhead')
+        if toolhead is not None:
+            toolhead.do_shutdown()
+    def invoke_async_shutdown(self, msg):
+        self.async_shutdown_msg = msg
+        self.request_exit("shutdown")
     def request_exit(self, result="exit"):
         self.run_result = result
         self.reactor.end()
diff --git a/klippy/mcu.py b/klippy/mcu.py
index 9d76e0fa..17e93e0b 100644
--- a/klippy/mcu.py
+++ b/klippy/mcu.py
@@ -472,7 +472,7 @@ class MCU:
         prefix = "MCU '%s' shutdown: " % (self._name,)
         if params['#name'] == 'is_shutdown':
             prefix = "Previous MCU '%s' shutdown: " % (self._name,)
-        self._printer.note_shutdown(prefix + msg + error_help(msg))
+        self._printer.invoke_async_shutdown(prefix + msg + error_help(msg))
     # Connection phase
     def _check_restart(self, reason):
         start_reason = self._printer.get_start_args().get("start_reason")
@@ -723,11 +723,11 @@ class MCU:
             return
         offset, freq = self._clocksync.calibrate_clock(print_time, eventtime)
         self._ffi_lib.steppersync_set_time(self._steppersync, offset, freq)
-        if self._clocksync.is_active(eventtime):
+        if self._clocksync.is_active(eventtime) or self.is_fileoutput():
             return
         logging.info("Timeout with MCU '%s' (eventtime=%f)",
                      self._name, eventtime)
-        self._printer.note_mcu_error("Lost communication with MCU '%s'" % (
+        self._printer.invoke_shutdown("Lost communication with MCU '%s'" % (
             self._name,))
     def stats(self, eventtime):
         msg = "%s: mcu_awake=%.03f mcu_task_avg=%.06f mcu_task_stddev=%.06f" % (
@@ -735,7 +735,9 @@ class MCU:
             self._mcu_tick_stddev)
         return ' '.join([msg, self._serial.stats(eventtime),
                          self._clocksync.stats(eventtime)])
-    def force_shutdown(self):
+    def do_shutdown(self):
+        if self._is_shutdown or self._emergency_stop_cmd is None:
+            return
         self.send(self._emergency_stop_cmd.encode())
     def disconnect(self):
         self._serial.disconnect()
diff --git a/klippy/toolhead.py b/klippy/toolhead.py
index 6df5e17f..50fea974 100644
--- a/klippy/toolhead.py
+++ b/klippy/toolhead.py
@@ -292,7 +292,7 @@ class ToolHead:
                 self.last_flush_from_idle = True
         except:
             logging.exception("Exception in flush_handler")
-            self.force_shutdown()
+            self.printer.invoke_shutdown("Exception in flush_handler")
         return self.reactor.NEVER
     # Motor off timer
     def _motor_off_handler(self, eventtime):
@@ -305,7 +305,7 @@ class ToolHead:
             self.motor_off()
         except:
             logging.exception("Exception in motor_off_handler")
-            self.force_shutdown()
+            self.printer.invoke_shutdown("Exception in motor_off_handler")
         return eventtime + self.motor_off_time
     # Movement commands
     def get_position(self):
@@ -371,14 +371,12 @@ class ToolHead:
         buffer_time = max(0., self.print_time - est_print_time)
         return "print_time=%.3f buffer_time=%.3f print_stall=%d" % (
             self.print_time, buffer_time, self.print_stall)
-    def force_shutdown(self):
+    def do_shutdown(self):
         try:
-            for m in self.all_mcus:
-                m.force_shutdown()
             self.move_queue.reset()
             self.reset_print_time()
         except:
-            logging.exception("Exception in force_shutdown")
+            logging.exception("Exception in do_shutdown")
     def get_max_velocity(self):
         return self.max_velocity, self.max_accel
     def get_max_axis_halt(self):