recreate process object when restarting a bureau

screenless/mgmt.py

@@ -27,6 +27,7 @@ class Management:
         self.procs = {}
         self.org_chart = []
         self._load_config()
+        self.wait_restart = False
 
     def _load_config(self): 
         config = configparser.ConfigParser()
@@ -46,16 +47,23 @@ class Management:
         Initialized and start all child bureaus
         """
         for buro in self.org_chart:
-            # TODO: this may need some sanity checking for crash/reload
-            lib = importlib.import_module("bureau." + buro)
-            proc = multiprocessing.Process(target=lib.main)
-            self.procs[buro] = proc
-            proc.start()
+            self._start_bureau(buro)
+        self.wait_restart = False
+
+    def _start_bureau(self, bureau_name):
+        """
+        starts an individual bureau
+        """
+        lib = importlib.import_module("bureau." + bureau_name)
+        proc = multiprocessing.Process(target=lib.main)
+        self.procs[bureau_name] = proc
+        proc.start()
 
     def _stop_bureaus(self):
         """
         Terminates all running bureau children
         """
+        self.wait_restart = True
         for buro in self.org_chart:
             proc = self.procs[buro]
             proc.terminate()
@@ -81,13 +89,16 @@ class Management:
 
         self._start_bureaus()
         while True:
+            if self.wait_restart:
+                time.sleep(1)
+                continue
             for buro in self.org_chart:
                 proc = self.procs[buro]
                 if not proc.is_alive():
                     print("bureau", buro, "has crashed!  Call the consultants!")
-                    print("restarting...")
-                    #TODO this should probably only retry a few times
-                    proc.start()
+                    time.sleep(3)
+                    print("restarting", buro)
+                    self._start_bureau(buro)
             time.sleep(1)