Add Graylog Input Metrics plugin for CheckMK 2.2.0.

check_mk-graylog_metrics/local/lib/python3/cmk/gui/plugins/wato/check_parameters/graylog_input_metrics.py

@@ -0,0 +1,92 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019 tribe29 GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+
+from cmk.gui.i18n import _
+from cmk.gui.plugins.wato.utils import (
+    CheckParameterRulespecWithItem,
+    rulespec_registry,
+    RulespecGroupCheckParametersDiscovery,
+)
+from cmk.gui.valuespec import Dictionary, Float, Tuple
+
+defaults={
+    "im_rate_upper": 999999,
+    "im_rate_lower": 0,
+    "rs_rate_upper": 999999,
+    "rs_rate_lower": 0,
+}
+mappings={
+    "im": "messages",
+    "rs": "bytes",
+    "upper": "above",
+    "lower": "below",
+}
+
+
+def _rate(metric, time, level):
+    unit=mappings[metric]
+    direction=mappings[level]
+    return (
+        f"{metric}_{time}_rate_{level}",
+        Tuple(
+            title=_(f"{level.capitalize()} level"),
+            elements=[
+                Float(
+                    title=_(f"Warning {direction}"),
+                    unit=f"{unit}/{time[1:]}min",
+                    default_value=defaults[f"{metric}_rate_{level}"],
+                ),
+                Float(
+                    title=_(f"Critical {direction}"),
+                    unit=f"{unit}/{time[1:]}min",
+                    default_value=defaults[f"{metric}_rate_{level}"],
+                ),
+            ],
+        ),
+    )
+
+
+def _element(metric, time):
+    return (
+        f"{metric}_{time}_rate",
+        Dictionary(
+            title=_(f"Incoming {mappings[metric]} for past {time[1:]} minute"),
+            elements=[
+                _rate(metric, time, "upper"),
+                _rate(metric, time, "lower")
+            ]
+        ),
+    )
+
+
+def _parameter_valuespec_graylog_input_metrics():
+    matrix=[
+        ("im", "m1"), ("im", "m5"), ("im", "m15"),
+        ("rs", "m1"), ("rs", "m5"), ("rs", "m15"),
+    ]
+
+    elements=[
+        _element(metric, time) for metric, time in matrix
+    ]
+
+    return Dictionary(
+        title=_("Message and data rates"),
+        help=_(
+            "These rates are queried directly from the Graylog instance. "
+            "Upper and lower levels can be specified for individual metric."
+        ),
+        elements=elements
+    )
+
+
+rulespec_registry.register(
+    CheckParameterRulespecWithItem(
+        check_group_name="graylog_input_metrics",
+        group=RulespecGroupCheckParametersDiscovery,
+        match_type="dict",
+        parameter_valuespec=_parameter_valuespec_graylog_input_metrics,
+        title=lambda: _("Graylog input metrics"),
+    )
+)

check_mk-graylog_metrics/local/lib/python3/cmk/special_agents/agent_graylog_input_metrics.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019 Checkmk GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+
+import argparse
+import json
+import sys
+from typing import NamedTuple
+
+import requests
+import urllib3
+
+import cmk.utils.password_store
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+cmk.utils.password_store.replace_passwords()
+
+
+class GraylogSection(NamedTuple):
+    name: str
+    uri: str
+
+
+def main(argv=None):
+    if argv is None:
+        argv = sys.argv[1:]
+
+    args = parse_arguments(argv)
+
+    try:
+        handle_request(args)
+    except Exception:
+        if args.debug:
+            return 1
+
+    return 0
+
+
+def handle_request(args):  # pylint: disable=too-many-branches
+    url_base = f"{args.proto}://{args.hostname}:{args.port}/api"
+    url = url_base + "/system/metrics"
+    value = handle_response(url, args).json()
+
+    # Handle the input_metrics section. We need to merge information from
+    # both inputstates and input_metrics, and we do that by extracting the
+    # ids returned by both calls. Once merged, we return a single dictionary
+    # with state, name, type, port, and the various rates (either raw or
+    # incomingMessages).
+    url_inputs_data = url_base + "/cluster/inputstates"
+    inputs_data = handle_response(url_inputs_data, args).json()
+    inputs_data = tuple(inputs_data.values())[0]
+    metrics_data = value.get("meters")
+
+    if inputs_data is None or metrics_data is None:
+        return
+
+    # Create a dictionary, containing all metrics with substrings
+    # "incomingMessages" and "rawSize".
+    # All rates should exist, created and with values as low as "0.0".
+    metrics_dict = {}
+    for metric, metric_rate in metrics_data.items():
+        metric_id = metric.split(".")[-2]
+        metric_type = None
+        if "incomingMessages" in metric:
+            metric_type = "im"
+        elif "rawSize" in metric:
+            metric_type = "rs"
+        if metric_type:
+            metric_key = metrics_dict.setdefault(metric_id, {})
+            metric_key[f"{metric_type}_m1_rate"] = metric_rate["m1_rate"]
+            metric_key[f"{metric_type}_m5_rate"] = metric_rate["m5_rate"]
+            metric_key[f"{metric_type}_m15_rate"] = metric_rate["m15_rate"]
+
+    # Create a dictionary with all inputs and add the rates from
+    # the previous dictionary, metrics_dict. This is passed as output.
+    # Some inputs don't have a "port", so we handle this with .get("port").
+    inputs_dict = {}
+    for inputs in inputs_data:
+        message_input = inputs["message_input"]
+        input_id = inputs["id"]
+        input_state = inputs["state"]
+        input_name = message_input["title"]
+        input_type = message_input["name"]
+        input_port = message_input["attributes"].get("port")
+        input_rate = metrics_dict[input_id]
+        inputs_dict[input_id] = {
+            "input_state": input_state,
+            "input_name": input_name,
+            "input_type": input_type,
+            "input_port": input_port,
+            "im_m1_rate": input_rate["im_m1_rate"],
+            "im_m5_rate": input_rate["im_m5_rate"],
+            "im_m15_rate": input_rate["im_m15_rate"],
+            "rs_m1_rate": input_rate["rs_m1_rate"],
+            "rs_m5_rate": input_rate["rs_m5_rate"],
+            "rs_m15_rate": input_rate["rs_m15_rate"]
+        }
+
+    if inputs_dict:
+        handle_output(inputs_dict)
+
+
+def handle_response(url, args):
+    try:
+        return requests.get(url, auth=(args.user, args.password), verify=not args.no_cert_check)
+    except requests.exceptions.RequestException as e:
+        sys.stderr.write("Error: %s\n" % e)
+        if args.debug:
+            raise
+
+
+def handle_output(value):
+    sys.stdout.write("<<<graylog_input_metrics:sep(0)>>>\n")
+    if isinstance(value, list):
+        for entry in value:
+            sys.stdout.write("%s\n" % json.dumps(entry))
+        return
+
+    sys.stdout.write("%s\n" % json.dumps(value))
+
+    return
+
+
+def parse_arguments(argv):
+    parser = argparse.ArgumentParser(description=__doc__)
+
+    parser.add_argument("-u", "--user", default=None, help="Username for graylog login")
+    parser.add_argument("-s", "--password", default=None, help="Password for graylog login")
+    parser.add_argument(
+        "-P",
+        "--proto",
+        default="https",
+        help="Use 'http' or 'https' for connection to graylog (default=https)",
+    )
+    parser.add_argument(
+        "-p", "--port", default=443, type=int, help="Use alternative port (default: 443)"
+    )
+    parser.add_argument(
+        "--debug", action="store_true", help="Debug mode: let Python exceptions come through"
+    )
+    parser.add_argument(
+        "--no-cert-check", action="store_true", help="Disable SSL certificate validation"
+    )
+
+    parser.add_argument(
+        "hostname", metavar="HOSTNAME", help="Name of the graylog instance to query."
+    )
+
+    return parser.parse_args(argv)
+
+
+if __name__ == "__main__":
+    sys.exit(main())

check_mk-graylog_metrics/local/share/check_mk/agents/special/agent_graylog_input_metrics

@@ -0,0 +1,11 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019 Checkmk GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+
+import sys
+
+from cmk.special_agents.agent_graylog_input_metrics import main
+
+if __name__ == "__main__":
+    sys.exit(main())

check_mk-graylog_metrics/local/share/check_mk/checks/agent_graylog_input_metrics

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019 Checkmk GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+#
+# {
+#     'proto': 'http',
+#     'port': 9000,
+#     'user': 'hell',
+#     'password': 'yeah',
+# }
+
+from typing import Any, Mapping, Optional, Sequence, Union
+
+
+def agent_graylog_input_metrics_arguments(
+    params: Mapping[str, Any], hostname: str, ipaddress: Optional[str]
+) -> Sequence[Union[str, tuple[str, str, str]]]:
+    args = [
+        "-P",
+        params["protocol"],
+        "-u",
+        params["user"],
+        "-s",
+        passwordstore_get_cmdline("%s", params["password"]),
+    ]
+
+    if "port" in params:
+        args += ["-p", params["port"]]
+
+    if "no-cert-check" in params:
+        args += ["--no-cert-check"]
+
+    args.append(params["instance"])
+
+    return args
+
+
+special_agent_info["graylog_input_metrics"] = agent_graylog_input_metrics_arguments

check_mk-graylog_metrics/local/share/check_mk/checks/graylog_input_metrics

@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+
+# NOTE: Careful when replacing the *-import below with a more specific import. This can cause
+# problems because it might remove variables from the check-context which are necessary for
+# resolving legacy discovery results such as [("SUMMARY", "diskstat_default_levels")]. Furthermore,
+# it might also remove variables needed for accessing discovery rulesets.
+import json
+
+#<<<graylog_input_metrics:sep(0)>>>
+# {"641e88d05d447a677efde199": {"input_state": "FAILED", "input_name": "kafka_cef_test",
+# "input_type": "CEF Kafka", "input_port": null, "im_m1_rate": 0.0, "im_m5_rate": 0.0,
+# "im_m15_rate": 0.0, "rs_m1_rate": 0.0, "rs_m5_rate": 0.0, "rs_m15_rate": 0.0},
+# "641e32885d447a677efd2dbf": {"input_state": "RUNNING", "input_name": "UDP-test",
+# "input_type": "Syslog UDP", "input_port": 1514, "im_m1_rate": 1.0846244336700077,
+# "im_m5_rate": 1.3700826278955827, "im_m15_rate": 1.254406787430692, "rs_m1_rate": 145.45579305762527,
+# "rs_m5_rate": 180.6486220431909, "rs_m15_rate": 165.26666376319292},
+# "641e32795d447a677efd2d9e": {"input_state": "RUNNING", "input_name": "testTCP", "input_type": "Syslog TCP",
+# "input_port": 1515, "im_m1_rate": 1.057872514816615, "im_m5_rate": 1.364957693749168,
+# "im_m15_rate": 1.2528742858546844, "rs_m1_rate": 140.4719944116262, "rs_m5_rate": 178.57816158901215,
+# "rs_m15_rate": 163.80530659055356}}
+
+def parse_graylog_input_metrics(section):
+    parsed = json.loads(section[0][0])
+    return parsed
+
+
+def inventory_graylog_input_metrics(parsed):
+    for input_id, input_info in parsed.items():
+        input_name = input_info["input_name"]
+        yield Service(f"{input_name} ({input_id})")
+
+
+def check_graylog_input_metrics(item, params, parsed):
+#    if parsed is None: return
+
+    item_id = item.split()[-1][1:-1]
+    input_info = parsed[item_id]
+    input_state = input_info["input_state"]
+
+    state = 1
+    if input_state == "RUNNING": state = 0
+    elif input_state == "FAILED": state = 2
+
+    yield state, "State: %s" % input_state
+    yield 0, "Type: %s" % input_info["input_type"]
+
+    if input_info["input_port"]:
+        yield 0, "Port: %s" % input_info["input_port"]
+
+    for key, dsname, unit, unit_func, infotext in [
+        ("im_m1_rate",  "im_m1_rate",  "/1min",  float, "Incoming messages"),
+        ("im_m5_rate",  "im_m5_rate",  "/5min",  float, ""),
+        ("im_m15_rate", "im_m15_rate", "/15min", float, ""),
+        ("rs_m1_rate",  "rs_m1_rate",  "/1min",  get_bytes_human_readable, "Incoming data"),
+        ("rs_m5_rate",  "rs_m5_rate",  "/5min",  get_bytes_human_readable, ""),
+        ("rs_m15_rate", "rs_m15_rate", "/15min", get_bytes_human_readable, ""),
+    ]:
+
+        value = parsed[item_id][key]
+        value = round(value, 2)
+        rate_upper = params.get(key, {}).get("%s_upper" % key, (None, None))
+        rate_lower = params.get(key, {}).get("%s_lower" % key, (None, None))
+        yield check_levels(
+            value,
+            dsname,
+            rate_upper + rate_lower,
+            human_readable_func=unit_func,
+            unit=unit,
+            infoname=infotext
+        )
+
+check_info["graylog_input_metrics"] = {
+    "parse_function": parse_graylog_input_metrics,
+    "check_function": check_graylog_input_metrics,
+    "inventory_function": inventory_graylog_input_metrics,
+    "service_description": "Graylog Input %s",
+    "has_perfdata": True,
+    "group": "graylog_input_metrics",
+}

check_mk-graylog_metrics/local/share/check_mk/web/plugins/wato/graylog_input_metrics.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# Copyright (C) 2019 Checkmk GmbH - License: GNU General Public License v2
+# This file is part of Checkmk (https://checkmk.com). It is subject to the terms and
+# conditions defined in the file COPYING, which is part of this source code package.
+
+
+from cmk.gui.i18n import _
+from cmk.gui.plugins.wato.special_agents.common import RulespecGroupDatasourceProgramsApps
+from cmk.gui.plugins.wato.utils import HostRulespec, IndividualOrStoredPassword, rulespec_registry
+from cmk.gui.valuespec import Age, Dictionary, DropdownChoice, Integer, ListChoice, TextInput
+from cmk.gui.watolib.rulespecs import Rulespec
+
+
+def _factory_default_special_agents_graylog_input_metrics():
+    # No default, do not use setting if no rule matches
+    return Rulespec.FACTORY_DEFAULT_UNUSED
+
+
+def _valuespec_special_agents_graylog_input_metrics():
+    return Dictionary(
+        title=_("Graylog Input Metrics"),
+        help=_("Requests input metrics data from a Graylog instance."),
+        optional_keys=["port", "no-cert-check"],
+        elements=[
+            (
+                "instance",
+                TextInput(
+                    title=_("Graylog instance to query"),
+                    help=_(
+                        "Use this option to set which instance should be "
+                        "checked by the special agent. Please add the "
+                        "hostname here, eg. my_graylog.com."
+                    ),
+                    size=32,
+                    allow_empty=False,
+                ),
+            ),
+            (
+                "user",
+                TextInput(
+                    title=_("Username"),
+                    help=_(
+                        "The username that should be used for accessing the "
+                        "Graylog API. Has to have read permissions at least."
+                    ),
+                    size=32,
+                    allow_empty=False,
+                ),
+            ),
+            (
+                "password",
+                IndividualOrStoredPassword(
+                    title=_("Password of the user"),
+                    allow_empty=False,
+                ),
+            ),
+            (
+                "protocol",
+                DropdownChoice(
+                    title=_("Protocol"),
+                    choices=[
+                        ("http", "HTTP"),
+                        ("https", "HTTPS"),
+                    ],
+                    default_value="https",
+                ),
+            ),
+            (
+                "port",
+                Integer(
+                    title=_("Port"),
+                    help=_(
+                        "Use this option to query a port which is different from standard port 443."
+                    ),
+                    default_value=443,
+                ),
+            ),
+            (
+                "no-cert-check",
+                FixedValue(
+                    True,
+                    title=_("Disable SSL certificate validation"),
+                    totext=_("SSL certificate validation is disabled"),
+                ),
+            ),
+        ],
+    )
+
+
+rulespec_registry.register(
+    HostRulespec(
+        factory_default=_factory_default_special_agents_graylog_input_metrics(),
+        group=RulespecGroupDatasourceProgramsApps,
+        name="special_agents:graylog_input_metrics",
+        valuespec=_valuespec_special_agents_graylog_input_metrics,
+    )
+)