Script now writes to error.log anything output from state change scripts

to stdout or stderr pipes.
Return codes from state changes scripts are now checked. If non-zero, this
is treated as a failure.

1 files changed, 16 insertions, 4 deletions

diff --git a/healthcheck.py b/healthcheck.py
index 856a8b2..e1661ca 100644
--- a/healthcheck.py
+++ b/healthcheck.py
@@ -37,8 +37,16 @@ class HealthCheck(object):
         # If we had a state change process, poll it so that the process quits properly
         # when it has finished, rather than leaving a defunct thing lying around.
         if not (self.state_process is None):
-            if (self.state_process.poll() is None):
-                self.logmsg("State change script has finished")
+            # Get any output from the script
+            for line in iter(self.state_process.stdout.readline,''):
+                # Trim any whitespace off the end, including newlines
+                self.logmsg("Script output: %s" % line.rstrip())
+            if not (self.state_process.poll() is None):
+                self.logmsg("State change script has finished with code %s" % str(self.state_process.returncode))
+                if not (self.state_process.returncode is None):
+                    if (self.state_process.returncode != 0):
+                        new_state = States.Failed
+                        self.logmsg("***** STATE CHANGE SCRIPT FAILED *****")
                 self.state_process = None
 
         # We can't abort the daemon starting if we fail to get the right
@@ -55,7 +63,11 @@ class HealthCheck(object):
             if (service_ip != self.last_address):
                 self.logmsg("Service IP = %s, last service IP = %s, this IP = %s" % (service_ip, self.last_address, self.system_ip))
 
-            if (os.path.isfile(self.script_directory + "/state-change-happening")):
+            if (new_state != -1):
+                # Dummy command to satisfy IF. The state could have been set above if
+                # a state change script fails.
+                new_state = new_state
+            elif (os.path.isfile(self.script_directory + "/state-change-happening")):
                 self.logmsg("State change script is still executing ...")
                 new_state = self.last_state
             elif (os.path.isfile(self.script_directory + "/frozen")):
@@ -247,7 +259,7 @@ class HealthCheck(object):
                 if len(files) == 1:
                     # os.system(files[0])
                     self.logmsg("Firing state change script %s" % files[0])
-                    self.state_process = subprocess.Popen([files[0]])
+                    self.state_process = subprocess.Popen([files[0]], shell=False, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
                 elif (len(files) > 1):
                     self.logmsg("More than one matching script for stage change %s to %s" % (str(self.last_state), str(new_state)))
             else: