Tighten up messaging and checks for VSPC, to eliminate situations where data has temporarily disappeared from parts of the VSPC API.

vspc_backup_checks/2.3/local/lib/python3/cmk_addons/plugins/vspc_backup_checks/libexec/agent_vspc_backup_checks

@@ -36,6 +36,23 @@ def get_url(conn, path, headers):
     return
 
 
+# Try to connect; if fail retry using exponential backoff. Returns boolean if
+# connect succeeded. This isn't the best way to handle things, but it is
+# minimally invasive.
+def connect(conn):
+    delay = 2
+
+    while delay <= 64:
+        try:
+            conn.connect()
+            return True
+        except:
+            time.sleep(delay)
+            delay *= 2
+
+    return False
+
+
 # GET HTTP with Bearer auth. Returns data structure parsed from JSON.
 #
 # Since results in VSPC are paginated, we go through all pages and return an
@@ -52,6 +69,10 @@ def get_paginated_json_url(host, port, path, token, insecure):
     conn = http.client.HTTPSConnection(host, port=port, timeout=10, context=ctx)
     headers = { "Authorization": f"Bearer {token}" }
 
+    connected = connect(conn)
+    if not connected:
+        raise ConnectionError("Failed to connect to VSPC")
+
     results = []
     offset = 0
 
@@ -68,6 +89,7 @@ def get_paginated_json_url(host, port, path, token, insecure):
 
         # If not meta, we're not paginated, so return the data immediately
         if not meta:
+            conn.close()
             return data
 
         results.extend(data)
@@ -76,7 +98,7 @@ def get_paginated_json_url(host, port, path, token, insecure):
         count  = meta["pagingInfo"]["count"]
         offset = offset + count
 
-        if offset >= total:
+        if offset >= total or count == 0:
             break
 
     conn.close()
@@ -173,6 +195,7 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy):
             bToR[bId][jId] = r
 
     results = defaultdict(list)
+
     for mAgent in mAgents:
         host = mAgent["tag"] or mAgent["hostName"]
         mAgentId = mAgent["instanceUid"]
@@ -212,12 +235,17 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy):
 
         rEntries = bToR.get(bAgentId, {})
 
-        jobs = bToJ.get(bAgentId)
+        bJobs = bToJ.get(bAgentId)
-        if not jobs:
+        if not bJobs:
+            results[host].append({
+                "status":  WARN,
+                "message": f"Backup agent {bAgentId} job query returned empty."
+            })
             continue
 
-        for job in jobs:
+        for job in bJobs:
             jobId = job["instanceUid"]
+            oriId = job["originalUid"]
             last  = job["lastEndTime"]
             sched = job["scheduleType"]
             polId = job["backupPolicyUid"]
@@ -316,8 +344,12 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy):
                         "message": f"Backup agent {bAgentId} job {jobId} is healthy; last backup ran {daysSinceLastRun:.1f} days ago."
                     })
 
-            rEntry = rEntries.get(jobId)
+            rEntry = rEntries.get(oriId)
             if not rEntry:
+                results[host].append({
+                    "status":  WARN,
+                    "message": f"Backup agent {bAgentId} job {jobId} has no restore points."
+                })
                 continue
 
             lastRestorePoint = rEntry["creationDate"]
@@ -327,17 +359,18 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy):
             if daysSinceLastSuccess > critDays:
                 results[host].append({
                     "status":  CRIT,
-                    "message": f"Job {jobId} last SUCCESSFULLY ran {daysSinceLastSuccess:.1f} days ago!"
+                    "message": f"Job {jobId}'s last successful restore point was {daysSinceLastSuccess:.1f} days ago!"
                 })
             elif daysSinceLastSuccess > warnDays:
                 results[host].append({
                     "status":  WARN,
-                    "message": f"Job {jobId} last SUCCESSFULLY ran {daysSinceLastSuccess:.1f} days ago!"
+                    "message": f"Job {jobId}'s last successful restore point was {daysSinceLastSuccess:.1f} days ago!"
                 })
-            else:
+            # To prevent noise in output, we only print this message if there were no others until now
+            elif len(results[host]) == 0:
                 results[host].append({
                     "status":  OK,
-                    "message": f"Job {jobId} last successfully ran {daysSinceLastSuccess:.1f} days ago."
+                    "message": f"Job {jobId}'s last successful restore point was {daysSinceLastSuccess:.1f} days ago."
                 })
 
     return results