#!/usr/bin/env python3
#
# Given a CUCM host, query the CUCM to get a complete list of phones, then
# query all those phones concurrently for additional details, then print
# the results out in a format CheckMK understands.
#
# Run the command on the console for a complete list of options.
#
# This script is designed to work with up to 90K phones, and ideally below 10K.
# If more than 10K phones are queried, this script should be modified to
# perform connection reuse to improve performance. Beyond 90K support for
# paginating the AXL API must be added.


phone_query_timeout = 10     # max time to query a single phone
phone_queries_timeout = 45  # max time to query all phones
cucm_page_size = 1000       # CUCM will not return pages larger than 2000


import urllib.request, base64, sys, argparse, asyncio, re, json, ssl, html
from xml.etree import ElementTree
from textwrap import wrap


# Create a TLS context for client connections.
def create_ssl_ctx():
    ctx = ssl.SSLContext(protocol=ssl.PROTOCOL_TLS_CLIENT)
    ctx.check_hostname = False
    ctx.verify_mode = ssl.CERT_NONE
    return ctx


# Typical GET or POST HTTP with Basic auth (using user and password
# credientials). Returns data structure parsed from XML.
def get_url(url, user, password, insecure, headers, data):
    request = urllib.request.Request(url, data=bytes(data, 'ascii'))

    for header, value in headers:
        request.add_header(header, value)

    if user and password:
        auth_str = base64.b64encode(bytes('%s:%s' % (user, password), 'ascii'))
        request.add_header('Authorization', 'Basic %s' % auth_str.decode('utf-8'))

    ctx = None
    if insecure:
        ctx = create_ssl_ctx()

    with urllib.request.urlopen(request, context=ctx) as conn:
        xml_data = conn.read()

    return ElementTree.fromstring(xml_data)


# Call the CUCM AXL API synchronously, using a SOAP query to fetch the names of
# all phones. It returns XML, which we parse. We call the AXL API because the
# RisPort70 API (see query_cucm_risport() below) does not support pagination,
# so we need to get a full list of phone names from AXL first, then do multiple
# queries on RisPort70 using subsets of the phone name list found from AXL.
#
# The AXL API has a return limit of 8MB, which is around 90K phones, so we
# don't bother paginating the AXL API itself; if more than that is needed, add
# pagination here.
#
# References:
# https://developer.cisco.com/docs/axl/
# https://github.com/reillychase/How-to-return-Cisco-RIS-with-more-than-1000-results/blob/master/main.py
def query_cucm_axl(addr, port, user, password, insecure):
    url = 'https://%s:%s/axl/' % (addr, port)
    headers = [
        ('Content-Type', 'text/xml'),
        ('Accept', 'text/xml'),
        ('SOAPAction', 'CUCM:DB ver=12.5'),
    ]

    try:
        return get_url(url, user, password, insecure, headers, """
            <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
                              xmlns:ns="http://www.cisco.com/AXL/API/12.5">
               <soapenv:Header/>
               <soapenv:Body>
                  <ns:listPhone>
                     <searchCriteria>
                       <name>%</name>
                     </searchCriteria>
                     <returnedTags>
                        <name/>
                     </returnedTags>
                  </ns:listPhone>
               </soapenv:Body>
            </soapenv:Envelope>
        """)
    except urllib.error.HTTPError as e:
        sys.stderr.write("AXL error: %s\n" % e)


# Call the CUCM RisPort70 API synchronously, using a SOAP query to fetch
# information about the phones with ids listed in the phone_ids arg. It returns
# XML, which we parse.
#
# Be aware that the API will return information about a maximum of 2000 devices,
# and provides no means of pagination. In order to do pagination, we first need
# to query the AXL API for a list of phone names, then all this function
# repeatedly with a different subset of 2000 phones from that complete list.
#
# References:
# https://developer.cisco.com/docs/sxml/#!risport70-api-reference
# https://paultursan.com/2018/12/getting-cucm-real-time-data-via-risport70-with-python-and-zeep-cisco-serviceability-api/
def query_cucm_risport(addr, port, user, password, insecure, phone_ids):
    assert len(phone_ids) <= 1000

    url = 'https://%s:%s/realtimeservice2/services/RISService70/' % (addr, port)
    headers = [('Content-Type', 'text/plain')]

    id_query = ''.join([f'<soap:item><soap:Item>{id}</soap:Item></soap:item>' for id in phone_ids])

    try:
        return get_url(url, user, password, insecure, headers, f"""
            <soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/"
                              xmlns:soap="http://schemas.cisco.com/ast/soap">
               <soapenv:Header/>
               <soapenv:Body>
                  <soap:selectCmDevice>
                     <soap:StateInfo></soap:StateInfo>
                     <soap:CmSelectionCriteria>
                        <soap:MaxReturnedDevices>2000</soap:MaxReturnedDevices>
                        <soap:DeviceClass>Any</soap:DeviceClass>
                        <soap:Model>255</soap:Model>
                        <soap:Status>Registered</soap:Status>
                        <soap:NodeName></soap:NodeName>
                        <soap:SelectBy>Name</soap:SelectBy>
                        <soap:SelectItems>
                           {id_query}
                        </soap:SelectItems>
                        <soap:Protocol>Any</soap:Protocol>
                        <soap:DownloadStatus>Any</soap:DownloadStatus>
                     </soap:CmSelectionCriteria>
                  </soap:selectCmDevice>
               </soapenv:Body>
            </soapenv:Envelope>
        """)
    except urllib.error.HTTPError as e:
        sys.stderr.write("CUCM error: %s\n" % e)


# Given AXL XML, use XPath to extract names for all phones.
def get_phone_ids(xml):
    # should this be ns2?
    namespace = {'ns': 'http://www.cisco.com/AXL/API/12.5'}
    items = xml.findall(".//phone/name", namespace)

    names = []
    for item in items:
        names.append(item.text)
    return names


# Given CUCM XML, use XPath to extract a bunch of details for each phone.
def get_phone_details(xml):
    namespace = {'ns1': 'http://schemas.cisco.com/ast/soap'}
    items = xml.findall(".//ns1:DeviceClass[.='Phone']/..", namespace)

    names_seen = {}
    phone_details = []
    for item in items:
        ip          = item.find('.//ns1:IP',       namespace).text
        name        = item.find('ns1:Name',        namespace).text
        dir_num     = item.find('ns1:DirNumber',   namespace).text
        description = item.find('ns1:Description', namespace).text
        user        = item.find('ns1:LoginUserId', namespace).text
        # These come with a -Registered on the end of the numbers.
        # Since all numbers we get from CUCM are registered, there's no
        # need for the -Registered, and we cut it off here.
        if dir_num:
            dir_num = dir_num.split('-')[0]

        if not names_seen.get(name):
            phone_details.append((name, ip, dir_num, user, description))
            names_seen[name] = True

    return phone_details


# If a phone (possibly) returns XML, attempt to extract the MAC, serial and
# model. We're using regex here, instead of full-blown XML parsing, to minimize
# the time and GC garbage generated.
def get_phone_details_from_xml(data):
    # if the HTTP server didn't return a 200...
    if data.find('200 OK') == -1:
        return None, None, None

    # attempt to extract info from XML
    mac    = re.search('<MACAddress>(.+)</MACAddress>',     data)
    serial = re.search('<serialNumber>(.+)</serialNumber>', data)
    model  = re.search('<modelNumber>(.+)</modelNumber>',   data)

    mac_str = mac    and normalize_mac(html.unescape(mac[1]))
    ser_str = serial and html.unescape(serial[1])
    mod_str = model  and html.unescape(model[1])

    return mac_str, ser_str, mod_str


# If a phone (possibly) returns HTML, attempt to extract the MAC, serial and
# model. We use regex here for the same reason we use it in
# get_phone_details_from_xml().
def get_phone_details_from_html(data):
    if data.find('200 OK') == -1:
        return None, None, None

    mac = None
    serial = None
    model = None

    # attempt to extract info from HTML
    matches = re.findall('<b>\s*(.*?)\s*</b>', data, re.M | re.I)
    for i, txt in enumerate(matches):
        txt = txt.lower()
        if   not mac    and txt == 'mac address':
            mac    = normalize_mac(html.unescape(matches[i + 1]))
        elif not serial and txt == 'serial number':
            serial = html.unescape(matches[i + 1])
        elif not model  and txt == 'model number':
            model  = html.unescape(matches[i + 1])
        elif mac and serial and model:
            break

    return mac, serial, model


# Different phones return MACs in different formats. We convert them to a single
# canonical format here.
def normalize_mac(mac):
    if mac.find(":") != -1:
        return mac.lower()
    else:
        return ":".join(wrap(mac, 2)).lower()


# Create a new HTTP/HTTPS connection, send a request, and extract any results.
async def get_async_url(ip, url, insecure=False):
    ctx = None
    port = 80
    if not insecure:
        ctx = create_ssl_ctx()
        port = 443

#   XXX switch from 127.0.0.1:8081 to ip:port
#    future = asyncio.open_connection('127.0.0.1', 8081, ssl=ctx)
    future = asyncio.open_connection(ip, port, ssl=ctx)
    reader, writer = await asyncio.wait_for(future, timeout=phone_query_timeout)
    query = f'GET {url} HTTP/1.1\r\nHost: {ip}\r\nConnection: close\r\n\r\n'
    writer.write(query.encode())
    await writer.drain()

    data = ''
    while not reader.at_eof():
        raw = await reader.read(-1)
        data += raw.decode()

    writer.close()
    await writer.wait_closed()

    return data

# Asynchronously contact the HTTP server in a phone. There are several
# different URLs that might return information, depending on the model of phone.
# To fetch the MAC and serial details we want requires us to potentially call
# all endpoints until we get some results. Be aware that some phone HTTP servers
# return 200 (and empty results) if we call the wrong URL for that model.
#
# We attempt to contact the phone using HTTPS first, falling back to HTTP if
# attempts with HTTPS failed.
#
# Originally we tried to take advantage of HTTP connection reuse, but this was
# causing some problems, and it wasn't worth the effort to handle the edge
# cases. Now we always make a new connection per request, even when it's
# multiple URLs on the same IP. If additional performance is ever needed, HTTP
# connection reuse is worth adding; depending on the latency it can easily
# 2x+ request rate.
async def query_phone_info_now(details):
    name, ip, dir_num, user, description = details
    mac = None
    serial = None
    model = None
    try:
        data = await get_async_url(ip, '/DeviceInformationX')
        mac, serial, model = get_phone_details_from_xml(data)

        if not mac:
            data = await get_async_url(ip, '/Device_Information.html')
            mac, serial, model = get_phone_details_from_html(data)

        if not mac:
            data = await get_async_url(ip, '/CGI/Java/Serviceability?adapter=device.statistics.device')
            mac, serial, model = get_phone_details_from_html(data)

        if not mac:
            data = await get_async_url(ip, '/')
            mac, serial, model = get_phone_details_from_html(data)
    except (ConnectionRefusedError, asyncio.TimeoutError, ssl.SSLError):
        try:
            if not mac:
                data = await get_async_url(ip, '/DeviceInformationX', True)
                mac, serial, model = get_phone_details_from_xml(data)

            if not mac:
                data = await get_async_url(ip, '/Device_Information.html', True)
                mac, serial, model = get_phone_details_from_html(data)

            if not mac:
                data = await get_async_url(ip, '/CGI/Java/Serviceability?adapter=device.statistics.device', True)
                mac, serial, model = get_phone_details_from_html(data)

            if not mac:
                data = await get_async_url(ip, '/', True)
                mac, serial, model = get_phone_details_from_html(data)
        except (ConnectionRefusedError, asyncio.TimeoutError):
            pass

    return {
        "name": name,
        "ip": ip,
        "mac": mac,
        "serial": serial,
        "dir_num": dir_num,
        "model": model,
        "user": user,
        "description": description
    }


# This functions job is solely to keep a limit on the concurrent number of
# connections made to the phones. Without this limit we'd quickly run out of
# spare sockets when dealing with large numbers of phones.
async def query_phone_info(details, semaphore):
    async with semaphore:
        return await query_phone_info_now(details)


# Given information about a list of phones (specifically, their IP addresses),
# we call the HTTP server on each phone to extract the MAC and serial. We
# return with a list of dicts containing information about all phones.
# Contacting thousands of phones serially would take too long, so we keep 200
# concurrent calls in-flight to the phones to shorten all querying to a few
# seconds.
async def query_phones(details):
    sem = asyncio.Semaphore(200)
    tasks = map(lambda d: asyncio.create_task(query_phone_info(d, sem)), details)
    done, pending = await asyncio.wait(tasks, timeout=phone_queries_timeout)
    # we silently ignore pending for now
    return map(lambda f: f.result(), done)


# Given an array of phone names, do paginated queries to the CUCM for phone
# information, then asynchronously query all the phones. While the CUCM has
# most of the information we want about a phone, it critically lacks the serial
# and MAC of the phone, which is why we need to fetch the details from the
# phone itself over an HTTP server each phone has.
def get_phones(addr, port, user, password, insecure):
    axl_xml = query_cucm_axl(addr, port, user, password, insecure)
    phone_ids = get_phone_ids(axl_xml)

    phone_details = []
    page_size = cucm_page_size
    for i in range(0, len(phone_ids), page_size):
        ids = phone_ids[i:i + page_size]
        cucm_xml = query_cucm_risport(addr, port, user, password, insecure, ids)
        details = get_phone_details(cucm_xml)
        phone_details.extend(details)
    return asyncio.run(query_phones(phone_details))


# Print out all our results in a format that CheckMK understands. Most of our
# output are in JSON rows.
def print_out(device_info, agent_name):
    sys.stdout.write(f"<<<{agent_name}:sep(0)>>>\n")
    device_info = list(device_info)
    device_info.sort(key=lambda d: d["ip"])
    for entry in device_info:
        sys.stdout.write("%s\n" % json.dumps(entry))


# Parse the command-line arguments. We have several options, but hostname is
# always required. Print out help to console if we get no args.
def parse_arguments(argv):
    parser = argparse.ArgumentParser()

    parser.add_argument(
        "-u", "--user", default=None, help="Username for CUCM login"
    )
    parser.add_argument(
        "-s", "--password", default=None, help="Password for CUCM login"
    )
    parser.add_argument(
        "-p", "--port", default=443, type=int, help="Use alternative port (default: 443)"
    )
    parser.add_argument(
        "hostname", metavar="HOSTNAME", help="Hostname of the CUCM to query."
    )
    parser.add_argument(
        "-k", "--insecure", default=False, help="Skip certificate verification",
        action="store_true"
    )

    return parser.parse_args(argv)


# Parse args, contact CUCM, query phones, and then print results
def main(argv=None):
    if argv is None:
        argv = sys.argv[1:]

    args = parse_arguments(argv)
    phones = get_phones(args.hostname, args.port, args.user, args.password,
                        args.insecure)
    print_out(phones, 'cucm_inv')


if __name__ == "__main__":
    sys.exit(main())

