diff --git a/vspc_backup_checks/2.3/local/lib/python3/cmk_addons/plugins/vspc_backup_checks/libexec/agent_vspc_backup_checks b/vspc_backup_checks/2.3/local/lib/python3/cmk_addons/plugins/vspc_backup_checks/libexec/agent_vspc_backup_checks index 319350b..ce263ba 100755 --- a/vspc_backup_checks/2.3/local/lib/python3/cmk_addons/plugins/vspc_backup_checks/libexec/agent_vspc_backup_checks +++ b/vspc_backup_checks/2.3/local/lib/python3/cmk_addons/plugins/vspc_backup_checks/libexec/agent_vspc_backup_checks @@ -36,6 +36,23 @@ def get_url(conn, path, headers): return +# Try to connect; if fail retry using exponential backoff. Returns boolean if +# connect succeeded. This isn't the best way to handle things, but it is +# minimally invasive. +def connect(conn): + delay = 2 + + while delay <= 64: + try: + conn.connect() + return True + except: + time.sleep(delay) + delay *= 2 + + return False + + # GET HTTP with Bearer auth. Returns data structure parsed from JSON. # # Since results in VSPC are paginated, we go through all pages and return an @@ -52,6 +69,10 @@ def get_paginated_json_url(host, port, path, token, insecure): conn = http.client.HTTPSConnection(host, port=port, timeout=10, context=ctx) headers = { "Authorization": f"Bearer {token}" } + connected = connect(conn) + if not connected: + raise ConnectionError("Failed to connect to VSPC") + results = [] offset = 0 @@ -68,6 +89,7 @@ def get_paginated_json_url(host, port, path, token, insecure): # If not meta, we're not paginated, so return the data immediately if not meta: + conn.close() return data results.extend(data) @@ -76,7 +98,7 @@ def get_paginated_json_url(host, port, path, token, insecure): count = meta["pagingInfo"]["count"] offset = offset + count - if offset >= total: + if offset >= total or count == 0: break conn.close() @@ -173,6 +195,7 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy): bToR[bId][jId] = r results = defaultdict(list) + for mAgent in mAgents: host = mAgent["tag"] or mAgent["hostName"] mAgentId = mAgent["instanceUid"] @@ -212,12 +235,17 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy): rEntries = bToR.get(bAgentId, {}) - jobs = bToJ.get(bAgentId) - if not jobs: + bJobs = bToJ.get(bAgentId) + if not bJobs: + results[host].append({ + "status": WARN, + "message": f"Backup agent {bAgentId} job query returned empty." + }) continue - for job in jobs: + for job in bJobs: jobId = job["instanceUid"] + oriId = job["originalUid"] last = job["lastEndTime"] sched = job["scheduleType"] polId = job["backupPolicyUid"] @@ -316,8 +344,12 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy): "message": f"Backup agent {bAgentId} job {jobId} is healthy; last backup ran {daysSinceLastRun:.1f} days ago." }) - rEntry = rEntries.get(jobId) + rEntry = rEntries.get(oriId) if not rEntry: + results[host].append({ + "status": WARN, + "message": f"Backup agent {bAgentId} job {jobId} has no restore points." + }) continue lastRestorePoint = rEntry["creationDate"] @@ -327,17 +359,18 @@ def process(mAgents, bAgents, jobs, restores, policies, get_customized_policy): if daysSinceLastSuccess > critDays: results[host].append({ "status": CRIT, - "message": f"Job {jobId} last SUCCESSFULLY ran {daysSinceLastSuccess:.1f} days ago!" + "message": f"Job {jobId}'s last successful restore point was {daysSinceLastSuccess:.1f} days ago!" }) elif daysSinceLastSuccess > warnDays: results[host].append({ "status": WARN, - "message": f"Job {jobId} last SUCCESSFULLY ran {daysSinceLastSuccess:.1f} days ago!" + "message": f"Job {jobId}'s last successful restore point was {daysSinceLastSuccess:.1f} days ago!" }) - else: + # To prevent noise in output, we only print this message if there were no others until now + elif len(results[host]) == 0: results[host].append({ "status": OK, - "message": f"Job {jobId} last successfully ran {daysSinceLastSuccess:.1f} days ago." + "message": f"Job {jobId}'s last successful restore point was {daysSinceLastSuccess:.1f} days ago." }) return results diff --git a/vspc_backup_checks/2.3/vspc_backup_checks-0.6.0.mkp b/vspc_backup_checks/2.3/vspc_backup_checks-0.6.0.mkp deleted file mode 100755 index 3b34f9f..0000000 Binary files a/vspc_backup_checks/2.3/vspc_backup_checks-0.6.0.mkp and /dev/null differ diff --git a/vspc_backup_checks/2.3/vspc_backup_checks-0.6.2.mkp b/vspc_backup_checks/2.3/vspc_backup_checks-0.6.2.mkp new file mode 100755 index 0000000..f0091c5 Binary files /dev/null and b/vspc_backup_checks/2.3/vspc_backup_checks-0.6.2.mkp differ