Changeset 2081

Timestamp:

08/14/12 13:21:47 (12 years ago)

Author:

sanmai

Message:

• Further development on the Nagios monitoring plugin for Discojuice (JSON).
  • It now checks each of the two Tomcat instances that serve the JSON content on different URL and returns the 'worst exit status' (highest exit code) out of those suggested by each check.
  • Error reporting to logs in /tmp was expanded and improved significantly.

Location:

monitoring/plugins/mpi

Files:

• check_clarin_discojuice_json.py (modified) (3 diffs)
• check_clarin_saml.py (modified) (1 diff)

monitoring/plugins/mpi/check_clarin_discojuice_json.py

@@ -24 +24 @@
                   "usage: %s -h host" % (sys.argv[0]))
 
-def nagios_return(code, response) :
-    """ prints the response message
-        and exits the script with one
-        of the defined exit codes
-        DOES NOT RETURN
-    """
-    print code + ": " + response
-    sys.exit(nagios_codes[code])
+def nagios_return_complex(results) :
+    def deal_with_result(result) :
+        print result['code'] + ": " + result['message']
+        return result['code']
 
-def check_response_data_well_formedness(data) :
+    # Scan all condition/status check results and create a list of appropriate exit codes.
+    exit_code_keys          = map(lambda result     : deal_with_result(result), results)
+    suggested_exit_codes    = list(map(lambda key   : nagios_codes[key], exit_code_keys))
+
+    # Exit with the highest suggested exit code, because the higher the exit code the more problematic the status is and problems have priority over harmony.
+    sys.exit(max(suggested_exit_codes))
+
+def check_response_data_well_formedness(data, descriptive_string) :
 
     timestamp = datetime.datetime.today().isoformat()
@@ -40 +43 @@
         json.loads(data)
     except :
+        traceback_string = traceback.format_exc()
         #pdb.set_trace()
 
-        traceback_string = traceback.format_exc()
+        err_log_file_path = os.path.normpath("/tmp/err_unparseable." + descriptive_string.replace('/' , '__') + "_" + timestamp + ".log")
 
-        with open(name = "/tmp/err_unparseable_discojuice_JSON_" + timestamp + ".log", mode = "wt") as debugging_output_file :
-                debugging_output_file.write(traceback_string)
+        with open(name = err_log_file_path, mode = "wt") as debugging_output_file :
+            debugging_output_file.write(traceback_string)
+        
         return False
     else :
         return True
 
-def check_condition(host) :
-    timestamp = datetime.datetime.today().isoformat()
-    UP_URL  = '/discojuice/metadata_clarin.json'
+def check_condition(host, UP_URL) :
 
-    conn    = httplib.HTTPConnection(host)
+    def handle_connection_failure(problem_description) :
+        err_log_file_path = os.path.normpath("/tmp/err_connection_failure." + UP_URL.replace('/' , '__') + "_" + timestamp + ".log")
+
+        with open(name = err_log_file_path, mode = "wt") as debugging_output_file :
+           debugging_output_file.write(problem_description)
+    
+    timestamp                = datetime.datetime.today().isoformat()
     
     try :
-        request = conn.request("GET", UP_URL)
+        conn                 = httplib.HTTPConnection(host)   
+
+        request              = conn.request("GET", UP_URL)
     except :
-        #pdb.set_trace()
+        traceback_string     = traceback.format_exc()
+
+        handle_connection_failure(traceback_string + "\nThis problem originates from location 1 in 'check_clarin_discojuice_json.py'.")
 
         return {
                 "code"      : "CRITICAL", 
-                "message"   : 'Host %s, service %s has a problem.' % (host, DESCRIPTION)
+                "message"   : '[%s] HTTP connection to host %s failed.' % (DESCRIPTION, host)
                }
+    else :
+        try :
+            response         = conn.getresponse()
 
-    else :    
-        r1      = conn.getresponse()
+            data             = response.read()
+            
+            conn.close()
+        except :
+            traceback_string = traceback.format_exc()
 
-        data    = r1.read()
-        
-        conn.close()
-        
-        if r1.status == 200 :
-            well_formed = check_response_data_well_formedness(data)
+            handle_connection_failure(traceback_string + "\nThis problem originates from location 2 in 'check_clarin_discojuice_json.py'.")
+        else :
+            if response.status == 200 :
+                well_formed  = check_response_data_well_formedness(data, UP_URL)
 
-            if well_formed :
-                with open(name = "/tmp/err_unparseable_discojuice_" + timestamp + ".json" , mode = "wt") as debugging_output_file :
-                    debugging_output_file.write(data)
+                if well_formed :
+                    return {
+                            "code"      : "OK", 
+                            "message"   : '[%s] Host %s is up and returns parseable JSON data at "%s".' % (DESCRIPTION, host, UP_URL)
+                           }
+                else :
+                    return {
+                            "code"      : "CRITICAL", 
+                            "message"   : '[%s] Host %s is up but returns unparseable JSON data at "%s".' % (DESCRIPTION, host, UP_URL)
+                            }
+            else :
 
-                return {
-                        "code"      : "OK", 
-                        "message"   : 'Host %s, service %s is up and returns parseable JSON data.' % (host, DESCRIPTION)
-                       }
-            else :
-                with open(name = "/tmp/err_unparseable_discojuice_" + timestamp + ".json" , mode = "wt") as debugging_output_file :
-                    debugging_output_file.write(data)
-
+                handle_connection_failure("Unreachable URL: HTTP response code" + str(response.status) + "\nThis problem originates from location 3 in 'check_clarin_discojuice_json.py'.")
                 return {
                         "code"      : "CRITICAL", 
-                        "message"   : 'Host %s, service %s is up but returns unparseable JSON data. ' % (host, DESCRIPTION)
-                        }
-        else :
-            return {
-                    "code"      : "CRITICAL", 
-                    "message"   : 'Host %s, service %s has a problem.' % (host, DESCRIPTION)
-                   }
+                        "message"   : '[%s] Host %s has a problem with the URL path component "%s".' % (DESCRIPTION, host, UP_URL)
+                       }
 
 def main() :
@@ -117 +130 @@
         else :
             usage()
+
+    UP_URLs          = ("/discojuice/metadata_clarin1.json", "/discojuice/metadata_clarin2.json",)
     
-    #result = test_case(host);
-    result          = check_condition(host)
-    nagios_return(result['code'], result['message'])
+    # Check status for all UP_URLs.
+    results          = map(lambda UP_URL : check_condition(host = host, UP_URL = UP_URL),
+                           UP_URLs)
+    
+    nagios_return_complex(results)
 
 if __name__ == "__main__" :
     main()
-

monitoring/plugins/mpi/check_clarin_saml.py

@@ -71 +71 @@
                                                            stdout = subprocess.PIPE, 
                                                            stderr = subprocess.PIPE,
-                                                           env    = environment_variables)
+                                                                                       env    = environment_variables)
 
     stdout, stderr                      = process.communicate()