added vspc plugin for secondary repo
This commit is contained in:
George Pochiscan
parent
0632059835
commit
8cd662e2bc
@ -0,0 +1,325 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (C) 2026 Spearhead Systems SRL
|
||||
|
||||
import http.client, argparse, json, ssl
|
||||
from datetime import datetime, timezone
|
||||
from collections import defaultdict
|
||||
|
||||
|
||||
OK = 0
|
||||
WARN = 1
|
||||
CRIT = 2
|
||||
|
||||
SECONDS_PER_DAY = 24 * 60 * 60
|
||||
|
||||
|
||||
# GET HTTP with Bearer auth. Returns data structure parsed from JSON.
|
||||
#
|
||||
# Since results in VSPC are paginated, we go through all pages and return an
|
||||
# array of all results.
|
||||
#
|
||||
# We reuse the HTTP connection to reduce overhead.
|
||||
def get_paginated_json_url(host, port, path, token, insecure):
|
||||
ctx = None
|
||||
if insecure:
|
||||
ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS_CLIENT)
|
||||
ctx.check_hostname = False
|
||||
ctx.verify_mode = ssl.CERT_NONE
|
||||
|
||||
conn = http.client.HTTPSConnection(host, port=port, timeout=10, context=ctx)
|
||||
headers = { "Authorization": f"Bearer {token}" }
|
||||
|
||||
results = []
|
||||
offset = 0
|
||||
|
||||
while True:
|
||||
conn.request("GET", f"{path}?offset={offset}", headers=headers)
|
||||
response = conn.getresponse()
|
||||
|
||||
if response.status != 200:
|
||||
raise Exception(f"Status code for {path} was {response.status}")
|
||||
|
||||
page = json.loads(response.read())
|
||||
|
||||
meta = page["meta"]
|
||||
data = page["data"]
|
||||
|
||||
results.extend(data)
|
||||
|
||||
total = meta["pagingInfo"]["total"]
|
||||
count = meta["pagingInfo"]["count"]
|
||||
offset = offset + count
|
||||
|
||||
if offset >= total:
|
||||
break
|
||||
|
||||
conn.close()
|
||||
|
||||
return results
|
||||
|
||||
|
||||
# Parse the command-line arguments. We have several options, but hostname is
|
||||
# always required. Print out help to console if we get no args.
|
||||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument(
|
||||
"hostname", help="Hostname or IP of target VSPC"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-t", "--token", required=True, help="API token"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-p", "--port", default=1280, type=int, help="TCP port (default: 1280)"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-k", "--insecure", default=False, help="Skip certificate verification",
|
||||
action="store_true"
|
||||
)
|
||||
parser.add_argument(
|
||||
"-d", "--demo", default=False, help="Show an example output",
|
||||
action="store_true"
|
||||
)
|
||||
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
# Every server must have one management agent; if there's no management agent,
|
||||
# the server effectively doesn't exist for backups. Every server *probably*
|
||||
# has one backup agent, since it's what does the backups. If the backup agent
|
||||
# is present, then it must have a management agent.
|
||||
#
|
||||
# Every backup agent can perform multiple jobs.
|
||||
#
|
||||
# Therefore we looked up all management agents, all backup agents, and all
|
||||
# jobs. Then we walk from every management agent to the backup agent (if
|
||||
# present), then match all jobs against that backup agent. We ignore jobs
|
||||
# that aren't enabled.
|
||||
#
|
||||
# But it's more complicated than that: jobs tell us the last time they ran, not
|
||||
# the last time they *successfully* ran, which is important for when we WARN or
|
||||
# CRIT. To get this information we need info from
|
||||
# protectedWorkloads/computersManagedByConsole (here in the "managed" argument),
|
||||
# which contain the last restore point. That last restore point tells us when
|
||||
# the last successful backup was.
|
||||
#
|
||||
# Cumulatively, we check statuses, the scheduled job type, error messages, and
|
||||
# how long since the last run and successful run. We return a table of the
|
||||
# format:
|
||||
#
|
||||
# {
|
||||
# <hostname>: [
|
||||
# {
|
||||
# "status": 0/1/2,
|
||||
# "message": "..."
|
||||
# },
|
||||
# ...
|
||||
# ],
|
||||
# ...
|
||||
# }
|
||||
def process(mAgents, bAgents, jobs, managed):
|
||||
mToB = {}
|
||||
for agent in bAgents:
|
||||
mToB[agent["managementAgentUid"]] = agent
|
||||
|
||||
bToJ = defaultdict(list)
|
||||
for job in jobs:
|
||||
if job["isEnabled"]:
|
||||
bToJ[job["backupAgentUid"]].append(job)
|
||||
|
||||
bToM = {}
|
||||
for m in managed:
|
||||
bToM[m["backupAgentUid"]] = m
|
||||
|
||||
results = defaultdict(list)
|
||||
for mAgent in mAgents:
|
||||
host = mAgent["tag"] or mAgent["hostName"]
|
||||
mAgentId = mAgent["instanceUid"]
|
||||
|
||||
mStatus = mAgent["status"]
|
||||
if mStatus != "Healthy":
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Managment agent {mAgentId} is: {mStatus}."
|
||||
})
|
||||
continue
|
||||
|
||||
bAgent = mToB.get(mAgent["instanceUid"])
|
||||
if not bAgent:
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Host appears to have no backup agent."
|
||||
})
|
||||
continue
|
||||
|
||||
bAgentId = bAgent["instanceUid"]
|
||||
|
||||
bStatus = bAgent["status"]
|
||||
if bAgent["status"] != "Active":
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Backup agent {bAgentId} is {bStatus}."
|
||||
})
|
||||
continue
|
||||
|
||||
if bAgent["totalJobsCount"] == 0:
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Backup agent {bAgentId} has no jobs."
|
||||
})
|
||||
continue
|
||||
|
||||
mEntry = bToM.get(bAgentId)
|
||||
|
||||
jobs = bToJ.get(bAgentId)
|
||||
if not jobs:
|
||||
continue
|
||||
|
||||
for job in jobs:
|
||||
jobId = job["instanceUid"]
|
||||
last = job["lastEndTime"]
|
||||
sched = job["scheduleType"]
|
||||
|
||||
daysSinceLastRun = None
|
||||
if last:
|
||||
timeSinceLastRun = datetime.now(timezone.utc) - datetime.fromisoformat(last)
|
||||
daysSinceLastRun = timeSinceLastRun.seconds / SECONDS_PER_DAY
|
||||
|
||||
if sched == "NotScheduled":
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Backup agent {bAgentId} job {jobId} is not scheduled."
|
||||
})
|
||||
continue
|
||||
|
||||
failureMessage = job["failureMessage"]
|
||||
if failureMessage:
|
||||
# We use 2.2 here to give wiggle room for jobs to complete if
|
||||
# they take longer than expected.
|
||||
results[host].append({
|
||||
"status": CRIT if daysSinceLastRun > 2.2 else WARN,
|
||||
"message": f"Backup agent {bAgentId} job {jobId} failed {daysSinceLastRun:.1f} days ago: {failureMessage}"
|
||||
})
|
||||
continue
|
||||
|
||||
if sched != "Daily":
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Backup agent {bAgentId} job {jobId} has scheduleType unknown to CheckMK plugin: {sched}."
|
||||
})
|
||||
continue
|
||||
|
||||
if not daysSinceLastRun:
|
||||
results[host].append({
|
||||
"status": OK,
|
||||
"message": f"Backup agent {bAgentId} is healthy; no backups yet."
|
||||
})
|
||||
continue
|
||||
|
||||
# We use 1.2 & 2.2 here to give wiggle room for jobs to complete if
|
||||
# they take longer than expected.
|
||||
if daysSinceLastRun > 2.2:
|
||||
results[host].append({
|
||||
"status": CRIT,
|
||||
"message": f"Backup agent {bAgentId} job {jobId} is late! Last backup ran {daysSinceLastRun:.1f} days ago."
|
||||
})
|
||||
elif daysSinceLastRun > 1.2:
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Backup agent {bAgentId} job {jobId} is late! Last backup ran {daysSinceLastRun:.1f} days ago."
|
||||
})
|
||||
else:
|
||||
results[host].append({
|
||||
"status": OK,
|
||||
"message": f"Backup agent {bAgentId} job {jobId} is healthy; last backup ran {daysSinceLastRun:.1f} days ago."
|
||||
})
|
||||
|
||||
if not mEntry:
|
||||
continue
|
||||
|
||||
lastRestorePoint = mEntry["latestRestorePointDate"]
|
||||
if not lastRestorePoint:
|
||||
continue
|
||||
|
||||
timeSinceLastSuccess = datetime.now(timezone.utc) - datetime.fromisoformat(lastRestorePoint)
|
||||
daysSinceLastSuccess = timeSinceLastSuccess.seconds / SECONDS_PER_DAY
|
||||
|
||||
if daysSinceLastSuccess > 2.2:
|
||||
results[host].append({
|
||||
"status": CRIT,
|
||||
"message": f"Job {jobId} last SUCCESSFULLY ran {daysSinceLastSuccess:.1f} days ago!"
|
||||
})
|
||||
elif daysSinceLastSuccess > 1.2:
|
||||
results[host].append({
|
||||
"status": WARN,
|
||||
"message": f"Job {jobId} last SUCCESSFULLY ran {daysSinceLastSuccess:.1f} days ago!"
|
||||
})
|
||||
else:
|
||||
results[host].append({
|
||||
"status": OK,
|
||||
"message": f"Job {jobId} last successfully ran {daysSinceLastSuccess:.1f} days ago."
|
||||
})
|
||||
|
||||
return results
|
||||
|
||||
|
||||
def print_demo():
|
||||
print("""
|
||||
<<<<newveeam>>>>
|
||||
<<<local>>>
|
||||
1 "Veeam Backup" - Host appears to have no backup agent.
|
||||
<<<>>>
|
||||
<<<<retrip-nova>>>>
|
||||
<<<local>>>
|
||||
1 "Veeam Backup" - Managment agent e4ade74b-4c5d-4204-a35c-68ccf2c73d16 is: Inaccessible.
|
||||
<<<>>>
|
||||
<<<<guacamole.foo.bar>>>>
|
||||
<<<local>>>
|
||||
0 "Veeam Backup" - Backup agent c1e3d991-1fc7-409c-b9ee-8e25abeb1774 job d7fd4b02-a80c-6e9b-a046-75c8031768a8 is healthy; last backup ran 0.9 days ago. | Job d7fd4b02-a80c-6e9b-a046-75c8031768a8 last successfully ran 0.0 days ago. | Backup agent c1e3d991-1fc7-409c-b9ee-8e25abeb1774 job 42bba0eb-0d7c-66e7-aa6e-fb2fcfb63f67 is healthy; last backup ran 0.0 days ago. | Job 42bba0eb-0d7c-66e7-aa6e-fb2fcfb63f67 last successfully ran 0.0 days ago.
|
||||
<<<>>>
|
||||
<<<<nc.foo.bar>>>>
|
||||
<<<local>>>
|
||||
0 "Veeam Backup" - Backup agent 3279b5ec-e65e-cd44-b749-a8e2ee0b634d job beebdcd2-2624-60dc-ae88-bb105ac75d3d is healthy; last backup ran 0.8 days ago. | Job beebdcd2-2624-60dc-ae88-bb105ac75d3d last successfully ran 0.8 days ago.
|
||||
<<<>>>
|
||||
<<<<Catalina>>>>
|
||||
<<<local>>>
|
||||
2 "Veeam Backup" - Backup agent 795349ba-038a-5580-b4c2-5ab361d41f8f job 63270e37-9ff4-6491-b062-b95a09af82b7 failed 2.1 days ago: Failed to start a backup job. server=oatpp/1.2.5 / code=401 / description=Unauthorized / message=Unauthorized /
|
||||
<<<>>>
|
||||
""".strip())
|
||||
|
||||
|
||||
# Print out all our results in a format that CheckMK understands.
|
||||
def print_out(results):
|
||||
for host, rows in results.items():
|
||||
print(f"<<<<{host}>>>>")
|
||||
print("<<<local>>>")
|
||||
|
||||
max_status = OK
|
||||
messages = []
|
||||
for row in rows:
|
||||
max_status = max(max_status, row["status"])
|
||||
messages.append(row["message"].replace("\n", " / "))
|
||||
|
||||
print(f'{max_status} "Veeam Backup" - {" | ".join(messages)}')
|
||||
print("<<<>>>")
|
||||
|
||||
|
||||
# Check the status of all management agents, backup agents, and backup jobs.
|
||||
# Print results.
|
||||
def main(argv=None):
|
||||
args = parse_arguments()
|
||||
|
||||
if args.demo:
|
||||
return print_demo()
|
||||
|
||||
mAgents = get_paginated_json_url(args.hostname, args.port, '/api/v3/infrastructure/managementAgents', args.token, args.insecure)
|
||||
bAgents = get_paginated_json_url(args.hostname, args.port, '/api/v3/infrastructure/backupAgents', args.token, args.insecure)
|
||||
jobs = get_paginated_json_url(args.hostname, args.port, '/api/v3/infrastructure/backupAgents/jobs', args.token, args.insecure)
|
||||
managed = get_paginated_json_url(args.hostname, args.port, '/api/v3/protectedWorkloads/computersManagedByConsole', args.token, args.insecure)
|
||||
|
||||
results = process(mAgents, bAgents, jobs, managed)
|
||||
print_out(results)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@ -0,0 +1,64 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (C) 2026 Spearhead Systems SRL
|
||||
|
||||
from cmk.rulesets.v1.form_specs import Dictionary, DictElement, String, Integer, Password, BooleanChoice, DefaultValue
|
||||
from cmk.rulesets.v1.rule_specs import SpecialAgent, Topic, Title, Help
|
||||
from cmk.rulesets.v1.form_specs.validators import LengthInRange, NumberInRange
|
||||
|
||||
|
||||
def _formspec_vspc():
|
||||
return Dictionary(
|
||||
title = Title("VSPC Server Configuration"),
|
||||
elements = {
|
||||
"instance": DictElement(
|
||||
required = True,
|
||||
parameter_form = String(
|
||||
title = Title("Hostname / IP"),
|
||||
help_text = Help("Host or IP of VSPC host for queries"),
|
||||
custom_validate = (LengthInRange(min_value=1),),
|
||||
),
|
||||
),
|
||||
"port": DictElement(
|
||||
required = True,
|
||||
parameter_form = Integer(
|
||||
title = Title("Port"),
|
||||
help_text = Help("Port of VSPC host for query"),
|
||||
prefill = DefaultValue(1280),
|
||||
custom_validate = (NumberInRange(min_value=1, max_value=65535),),
|
||||
),
|
||||
),
|
||||
"token": DictElement(
|
||||
required = True,
|
||||
parameter_form = Password(
|
||||
title = Title("API Token"),
|
||||
help_text = Help("API token used for authentication by VSPC."),
|
||||
custom_validate = (LengthInRange(min_value=1),),
|
||||
),
|
||||
),
|
||||
"insecure": DictElement(
|
||||
required = True,
|
||||
parameter_form = BooleanChoice(
|
||||
title = Title("Insecure"),
|
||||
help_text = Help("Ignore unverified HTTPS certificate warnings when contacting VSPC"),
|
||||
prefill = DefaultValue(False),
|
||||
),
|
||||
),
|
||||
"demo": DictElement(
|
||||
required = True,
|
||||
parameter_form = BooleanChoice(
|
||||
title = Title("Demo"),
|
||||
help_text = Help("Puts agent into demo mode, returning fixed demo data regardless of VSPC results"),
|
||||
prefill = DefaultValue(False),
|
||||
),
|
||||
),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
rule_spec_agent_config_vspc_backup_checks = SpecialAgent(
|
||||
topic=Topic.NETWORKING,
|
||||
name="vspc_backup_checks",
|
||||
title=Title("VSPC Backup Checks"),
|
||||
parameter_form=_formspec_vspc,
|
||||
)
|
||||
|
||||
@ -0,0 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
# Copyright (C) 2026 Spearhead Systems SRL
|
||||
|
||||
from cmk.server_side_calls.v1 import noop_parser, SpecialAgentConfig, SpecialAgentCommand
|
||||
|
||||
|
||||
def _agent_arguments(params, host_config):
|
||||
token = params["token"]
|
||||
if type(token) != str:
|
||||
token = token.unsafe()
|
||||
|
||||
args = []
|
||||
|
||||
if params["insecure"]:
|
||||
args.append("--insecure")
|
||||
if params["demo"]:
|
||||
args.append("--demo")
|
||||
|
||||
args.extend(["--port", str(params["port"])])
|
||||
args.extend(["--token", token])
|
||||
args.append(params["instance"])
|
||||
|
||||
yield SpecialAgentCommand(command_arguments=args)
|
||||
|
||||
|
||||
special_agent_vspc_backup_checks = SpecialAgentConfig(
|
||||
name="vspc_backup_checks",
|
||||
parameter_parser=noop_parser,
|
||||
commands_function=_agent_arguments,
|
||||
)
|
||||
BIN
vspc_backup_checks/2.3/vspc_backup_checks-0.2.0.mkp
Executable file
BIN
vspc_backup_checks/2.3/vspc_backup_checks-0.2.0.mkp
Executable file
Binary file not shown.
Loading…
x
Reference in New Issue
Block a user