monitoring-scripts/check_docker

#!/usr/bin/env python3
# logging.basicConfig(level=logging.DEBUG)
import argparse
import json
import logging
import math
import os
import re
import socket
import stat
import traceback
from collections import deque, namedtuple, UserDict, defaultdict
from concurrent import futures
from datetime import datetime, timezone
from functools import lru_cache
from http.client import HTTPConnection
from sys import argv
from urllib import request
from urllib.error import HTTPError, URLError
from urllib.request import AbstractHTTPHandler, HTTPHandler, HTTPSHandler, OpenerDirector, HTTPRedirectHandler, \
    Request, HTTPBasicAuthHandler

logger = logging.getLogger()
__author__ = 'Tim Laurence'
__copyright__ = "Copyright 2019"
__credits__ = ['Tim Laurence']
__license__ = "GPL"
__version__ = "2.2.2"

'''
nrpe compatible check for docker containers.

Requires Python 3

Note: I really would have preferred to have used requests for all the network connections but that would have added a
dependency.
'''

DEFAULT_SOCKET = '/var/run/docker.sock'
DEFAULT_TIMEOUT = 10.0
DEFAULT_PORT = 2375
DEFAULT_MEMORY_UNITS = 'B'
DEFAULT_HEADERS = [('Accept', 'application/vnd.docker.distribution.manifest.v2+json')]
DEFAULT_PUBLIC_REGISTRY = 'registry-1.docker.io'

# The second value is the power to raise the base to.
UNIT_ADJUSTMENTS_TEMPLATE = {
    '%': 0,
    'B': 0,
    'KB': 1,
    'MB': 2,
    'GB': 3,
    'TB': 4
}
unit_adjustments = None

# Reduce message to a single OK unless a checks fail.
no_ok = False

# Suppress performance data reporting
no_performance = False

OK_RC = 0
WARNING_RC = 1
CRITICAL_RC = 2
UNKNOWN_RC = 3

# These hold the final results
rc = -1
messages = []
performance_data = []

ImageName = namedtuple('ImageName', "registry name tag full_name")


class ThresholdSpec(UserDict):
    def __init__(self, warn, crit, units=''):
        super().__init__(warn=warn, crit=crit, units=units)

    def __getattr__(self, item):
        if item in ('warn', 'crit', 'units'):
            return self.data[item]
        return super().__getattr__(item)


# How much threading can we do? We are generally not CPU bound so I am using this a worse case cap
DEFAULT_PARALLELISM = 10

# Holds list of all threads
threads = []

# This is used during testing
DISABLE_THREADING = False


# Hacked up urllib to handle sockets
#############################################################################################
# Docker runs a http connection over a socket. http.client is knows how to deal with these
# but lacks some niceties. Urllib wraps that and makes up for some of the deficiencies but
# cannot fix the fact http.client can't read from socket files. In order to take advantage of
# urllib and http.client's  capabilities the class below tweaks HttpConnection and passes it
# to urllib registering for socket:// connections

# This is all side effect so excluding coverage
class SocketFileHandler(AbstractHTTPHandler):
    class SocketFileToHttpConnectionAdaptor(HTTPConnection): # pragma: no cover
        def __init__(self, socket_file, timeout=DEFAULT_TIMEOUT):
            super().__init__(host='', port=0, timeout=timeout)
            self.socket_file = socket_file

        def connect(self):
            self.sock = socket.socket(family=socket.AF_UNIX, type=socket.SOCK_STREAM, proto=0, fileno=None)
            self.sock.settimeout(self.timeout)
            self.sock.connect(self.socket_file)

    def socket_open(self, req):
        socket_file, path = req.selector.split(':', 1)
        req.host = socket_file
        req.selector = path
        return self.do_open(self.SocketFileToHttpConnectionAdaptor, req)


# Tokens are not cached because I expect the callers to cache the responses
class Oauth2TokenAuthHandler(HTTPBasicAuthHandler):
    auth_failure_tracker = defaultdict(int)

    def http_response(self, request, response):
        code, hdrs = response.code, response.headers

        www_authenticate_header = response.headers.get('www-authenticate', None)
        if code == 401 and www_authenticate_header:
            scheme = www_authenticate_header.split()[0]
            if scheme.lower() == 'bearer':
                return self.process_oauth2(request, response, www_authenticate_header)

        return response

    https_response = http_response

    @staticmethod
    def _get_outh2_token(www_authenticate_header):
        auth_fields = dict(re.findall(r"""(?:(?P<key>[^ ,=]+)="([^"]+)")""", www_authenticate_header))

        auth_url = "{realm}?scope={scope}&service={service}".format(
            realm=auth_fields['realm'],
            scope=auth_fields['scope'],
            service=auth_fields['service'],
        )
        token_request = Request(auth_url)
        token_request.add_header("Content-Type", "application/x-www-form-urlencoded; charset=utf-8")
        token_response = request.urlopen(token_request)
        return process_urllib_response(token_response)['token']

    def process_oauth2(self, request, response, www_authenticate_header):

        # This keeps infinite auth loops from happening
        full_url = request.full_url
        self.auth_failure_tracker[full_url] += 1
        if self.auth_failure_tracker[full_url] > 1:
            raise HTTPError(full_url, 401, "Stopping Oauth2 failure loop for {}".format(full_url),
                            response.headers, response)

        auth_token = self._get_outh2_token(www_authenticate_header)

        request.add_unredirected_header('Authorization', 'Bearer ' + auth_token)
        return self.parent.open(request, timeout=request.timeout)


better_urllib_get = OpenerDirector()
better_urllib_get.addheaders = DEFAULT_HEADERS.copy()
better_urllib_get.add_handler(HTTPHandler())
better_urllib_get.add_handler(HTTPSHandler())
better_urllib_get.add_handler(HTTPRedirectHandler())
better_urllib_get.add_handler(SocketFileHandler())
better_urllib_get.add_handler(Oauth2TokenAuthHandler())


class RegistryError(Exception):
    def __init__(self, response):
        self.response_obj = response


# Util functions
#############################################################################################
def parse_thresholds(spec, include_units=True, units_required=True):
    """
    Given a spec string break it up into ':' separated chunks. Convert strings to ints as it makes sense

    :param spec: The threshold specification being parsed
    :param include_units: Specifies that units should be processed and returned if present
    :param units_required: Mark spec as invalid if the units are missing.
    :return: A list containing the thresholds in order of warn, crit, and units(if included and present)
    """
    parts = deque(spec.split(':'))
    if not all(parts):
        raise ValueError("Blanks are not allowed in a threshold specification: {}".format(spec))

    # Warn
    warn = int(parts.popleft())
    # Crit
    crit = int(parts.popleft())

    units = ''
    if include_units:
        if len(parts):
            # units
            units = parts.popleft()
        elif units_required:
            raise ValueError("Missing units in {}".format(spec))

    if len(parts) != 0:
        raise ValueError("Too many threshold specifiers in {}".format(spec))

    return ThresholdSpec(warn=warn, crit=crit, units=units)


def pretty_time(seconds):
    remainder = seconds
    result = []
    if remainder > 24 * 60 * 60:
        days, remainder = divmod(remainder, 24 * 60 * 60)
        result.append("{}d".format(int(days)))
    if remainder > 60 * 60:
        hours, remainder = divmod(remainder, 60 * 60)
        result.append("{}h".format(int(hours)))
    if remainder > 60:
        minutes, remainder = divmod(remainder, 60)
        result.append("{}min".format(int(minutes)))
    result.append("{}s".format(int(remainder)))
    return result


def evaluate_numeric_thresholds(container, value, thresholds, name, short_name,
                                min=None, max=None, greater_than=True):
    rounder = lambda x: round(x, 2)

    INTEGER_UNITS = ['B', '%', '']

    # Some units don't have decimal places
    rounded_value = int(value) if thresholds.units in INTEGER_UNITS else rounder(value)

    perf_string = "{container}_{short_name}={value}{units};{warn};{crit}".format(
        container=container,
        short_name=short_name,
        value=rounded_value,
        **thresholds)
    if min is not None:
        rounded_min = math.floor(min) if thresholds.units in INTEGER_UNITS else rounder(min)
        perf_string += ';{}'.format(rounded_min)
        if max is not None:
            rounded_max = math.ceil(max) if thresholds.units in INTEGER_UNITS else rounder(max)
            perf_string += ';{}'.format(rounded_max)

    global performance_data
    performance_data.append(perf_string)

    if thresholds.units == 's':
        nice_time = ' '.join(pretty_time(rounded_value)[:2])
        results_str = "{} {} is {}".format(container, name, nice_time)
    else:
        results_str = "{} {} is {}{}".format(container, name, rounded_value, thresholds.units)

    if greater_than:
        comparator = lambda value, threshold: value >= threshold
    else:
        comparator = lambda value, threshold: value <= threshold

    if comparator(value, thresholds.crit):
        critical(results_str)
    elif comparator(value, thresholds.warn):
        warning(results_str)
    else:
        ok(results_str)


@lru_cache(maxsize=None)
def get_url(url):
    logger.debug("get_url: {}".format(url))
    response = better_urllib_get.open(url, timeout=timeout)
    logger.debug("get_url: {} {}".format(url, response.status))
    return process_urllib_response(response), response.status


def process_urllib_response(response):
    response_bytes = response.read()
    body = response_bytes.decode('utf-8')
    # logger.debug("BODY: {}".format(body))
    return json.loads(body)


def get_container_info(name):
    content, _ = get_url(daemon + '/containers/{container}/json'.format(container=name))
    return content


def get_image_info(name):
    content, _ = get_url(daemon + '/images/{image}/json'.format(image=name))
    return content


def get_state(container):
    return get_container_info(container)['State']


def get_stats(container):
    content, _ = get_url(daemon + '/containers/{container}/stats?stream=0'.format(container=container))
    return content


def get_ps_name(name_list):
    # Pick the name that starts with a '/' but doesn't contain a '/' and return that value
    for name in name_list:
        if '/' not in name[1:] and name[0] == '/':
            return name[1:]
    else:
        raise NameError("Error when trying to identify 'ps' name in {}".format(name_list))


def get_containers(names, require_present):
    containers_list, _ = get_url(daemon + '/containers/json?all=1')

    all_container_names = set(get_ps_name(x['Names']) for x in containers_list)

    if 'all' in names:
        return all_container_names

    filtered = set()
    for matcher in names:
        found = False
        for candidate in all_container_names:
            if re.match("^{}$".format(matcher), candidate):
                filtered.add(candidate)
                found = True
        # If we don't find a container that matches out regex
        if require_present and not found:
            critical("No containers match {}".format(matcher))

    return filtered


def get_container_image_id(container):
    # find registry and tag
    inspection = get_container_info(container)
    return inspection['Image']


def get_container_image_urls(container):
    inspection = get_container_info(container)
    image_id = inspection['Image']
    image_info = get_image_info(image_id)
    return image_info['RepoTags']


def normalize_image_name_to_manifest_url(image_name, insecure_registries):
    parsed_url = parse_image_name(image_name)

    lower_insecure = [reg.lower() for reg in insecure_registries]

    # Registry query url
    scheme = 'http' if parsed_url.registry.lower() in lower_insecure else 'https'
    url = '{scheme}://{registry}/v2/{image_name}/manifests/{image_tag}'.format(scheme=scheme,
                                                                               registry=parsed_url.registry,
                                                                               image_name=parsed_url.name,
                                                                               image_tag=parsed_url.tag)
    return url, parsed_url.registry


# Auth servers seem picky about being hit too hard. Can't figure out why. ;)
# As result it is best to single thread this check
# This is based on https://docs.docker.com/registry/spec/auth/token/#requesting-a-token
def get_digest_from_registry(url):
    logger.debug("get_digest_from_registry")
    # query registry
    # TODO: Handle logging in if needed
    registry_info, status_code = get_url(url=url)

    if status_code != 200:
        raise RegistryError(response=registry_info)
    return registry_info['config'].get('digest', None)


def set_rc(new_rc):
    global rc
    rc = new_rc if new_rc > rc else rc


def ok(message):
    set_rc(OK_RC)
    messages.append('OK: ' + message)


def warning(message):
    set_rc(WARNING_RC)
    messages.append('WARNING: ' + message)


def critical(message):
    set_rc(CRITICAL_RC)
    messages.append('CRITICAL: ' + message)


def unknown(message):
    set_rc(UNKNOWN_RC)
    messages.append('UNKNOWN: ' + message)


def require_running(name):
    def inner_decorator(func):
        def wrapper(container, *args, **kwargs):
            container_state = get_state(container)
            state = normalize_state(container_state)
            if state.lower() == "running":
                func(container, *args, **kwargs)
            else:
                # container is not running, can't perform check
                critical('{container} is not "running", cannot check {check}"'.format(container=container,
                                                                                      check=name))

        return wrapper

    return inner_decorator


def multithread_execution(disable_threading=DISABLE_THREADING):
    def inner_decorator(func):
        def wrapper(container, *args, **kwargs):
            if DISABLE_THREADING:
                func(container, *args, **kwargs)
            else:
                threads.append(parallel_executor.submit(func, container, *args, **kwargs))

        return wrapper

    return inner_decorator


def singlethread_execution(disable_threading=DISABLE_THREADING):
    def inner_decorator(func):
        def wrapper(container, *args, **kwargs):
            if DISABLE_THREADING:
                func(container, *args, **kwargs)
            else:
                threads.append(serial_executor.submit(func, container, *args, **kwargs))

        return wrapper

    return inner_decorator


def parse_image_name(image_name):
    """
    Parses image names into their constituent parts.
    :param image_name:
    :return: ImageName
    """

    # These are based on information found here
    #   https://docs.docker.com/engine/reference/commandline/tag/#extended-description
    #   https://github.com/docker/distribution/blob/master/reference/regexp.go
    host_segment_re = '[a-zA-Z0-9]([a-zA-Z0-9-]*[a-zA-Z0-9])?'
    hostname_re = r'({host_segment}\.)+{host_segment}'.format(host_segment=host_segment_re)
    registry_re = r'((?P<registry>({hostname_re}(:\d+)?|{host_segment_re}:\d+))/)'.format(
        host_segment_re=host_segment_re, hostname_re=hostname_re)
    name_component_ends_re = '[a-z0-9]'
    name_component_middle_re = '[a-z0-9._-]'  # Ignoring spec limit of two _
    name_component_re = '({end}{middle}*{end}|{end})'.format(end=name_component_ends_re,
                                                             middle=name_component_middle_re)
    image_name_re = "(?P<image_name>({name_component}/)*{name_component})".format(name_component=name_component_re)
    image_tag_re = '(?P<image_tag>[a-zA-Z0-9_][a-zA-Z0-9_.-]*)'
    full_re = '^{registry}?{image_name}(:{image_tag})?$'.format(registry=registry_re, image_name=image_name_re,
                                                                image_tag=image_tag_re)
    parsed = re.match(full_re, image_name)

    registry = parsed.group('registry') if parsed.group('registry') else DEFAULT_PUBLIC_REGISTRY

    image_name = parsed.group('image_name')
    image_name = image_name if '/' in image_name or registry != DEFAULT_PUBLIC_REGISTRY else 'library/' + image_name

    image_tag = parsed.group('image_tag')
    image_tag = image_tag if image_tag else 'latest'

    full_image_name = "{registry}/{image_name}:{image_tag}".format(
        registry=registry,
        image_name=image_name,
        image_tag=image_tag)

    return ImageName(registry=registry, name=image_name, tag=image_tag, full_name=full_image_name)


def normalize_state(status_info):
    # Ugh, docker used to report state in as silly way then they figured out how to do it better.
    # This tries the simpler new way and if that doesn't work fails back to the old way

    # On new docker engines the status holds whatever the current state is, running, stopped, paused, etc.
    if "Status" in status_info:
        return status_info['Status']

    status = 'Exited'
    if status_info["Restarting"]:
        status = 'Restarting'
    elif status_info["Paused"]:
        status = 'Paused'
    elif status_info["Dead"]:
        status = 'Dead'
    elif status_info["Running"]:
        return "Running"
    return status


# Checks
#############################################################################################

@multithread_execution()
@require_running(name='memory')
def check_memory(container, thresholds):
    if not thresholds.units in unit_adjustments:
        unknown("Memory units must be one of  {}".format(list(unit_adjustments.keys())))
        return

    inspection = get_stats(container)

    # Subtracting cache to match what `docker stats` does.
    adjusted_usage = inspection['memory_stats']['usage'] - inspection['memory_stats']['stats']['total_cache']
    if thresholds.units == '%':
        max = 100
        usage = int(100 * adjusted_usage / inspection['memory_stats']['limit'])
    else:
        max = inspection['memory_stats']['limit'] / unit_adjustments[thresholds.units]
        usage = adjusted_usage / unit_adjustments[thresholds.units]

    evaluate_numeric_thresholds(container=container, value=usage, thresholds=thresholds, name='memory',
                                short_name='mem', min=0, max=max)


@multithread_execution()
def check_status(container, desired_state):
    normized_desired_state = desired_state.lower()
    normalized_state = normalize_state(get_state(container)).lower()
    if normized_desired_state != normalized_state:
        critical("{} state is not {}".format(container, desired_state))
        return
    ok("{} status is {}".format(container, desired_state))


@multithread_execution()
@require_running('health')
def check_health(container):
    state = get_state(container)
    if "Health" in state and "Status" in state["Health"]:
        health = state["Health"]["Status"]
        message = "{} is {}".format(container, health)
        if health == 'healthy':
            ok(message)
        elif health == 'unhealthy':
            critical(message)
        else:
            unknown(message)
    else:
        unknown('{} has no health check data'.format(container))


@multithread_execution()
@require_running('uptime')
def check_uptime(container, thresholds):
    inspection = get_container_info(container)['State']['StartedAt']
    only_secs = inspection[0:19]
    start = datetime.strptime(only_secs, "%Y-%m-%dT%H:%M:%S")
    start = start.replace(tzinfo=timezone.utc)
    now = datetime.now(timezone.utc)
    uptime = (now - start).total_seconds()

    graph_padding = 2
    thresholds.units = 's'
    evaluate_numeric_thresholds(container=container, value=uptime, thresholds=thresholds, name='uptime',
                                short_name='up', min=0, max=graph_padding, greater_than=False)


@multithread_execution()
def check_image_age(container, thresholds):
    container_image = get_container_info(container)['Image']
    image_created = get_image_info(container_image)['Created']
    only_secs = image_created[0:19]
    start = datetime.strptime(only_secs, "%Y-%m-%dT%H:%M:%S")
    start = start.replace(tzinfo=timezone.utc)
    now = datetime.now(timezone.utc)
    image_age = (now - start).days

    graph_padding = 2
    thresholds.units = 'd'
    evaluate_numeric_thresholds(container=container, value=image_age, thresholds=thresholds, name='image_age',
                                short_name='age', min=0, max=graph_padding, greater_than=True)


@multithread_execution()
@require_running('restarts')
def check_restarts(container, thresholds):
    inspection = get_container_info(container)

    restarts = int(inspection['RestartCount'])
    graph_padding = 2
    evaluate_numeric_thresholds(container=container, value=restarts, thresholds=thresholds, name='restarts',
                                short_name='re', min=0, max=graph_padding)


@singlethread_execution()
def check_version(container, insecure_registries):
    image_id = get_container_image_id(container)
    logger.debug("Local container image ID: {}".format(image_id))
    if image_id is None:
        unknown('Checksum missing for "{}", try doing a pull'.format(container))
        return

    image_urls = get_container_image_urls(container=container)
    if len(image_urls) > 1:
        unknown('"{}" has multiple tags/names. Unsure which one to use to check the version.'.format(container))
        return
    elif len(image_urls) == 0:
        unknown('"{}" has last no repository tag. Is this anywhere else?'.format(container))
        return

    url, registry = normalize_image_name_to_manifest_url(image_urls[0], insecure_registries)
    logger.debug("Looking up image digest here {}".format(url))
    try:
        registry_hash = get_digest_from_registry(url)
    except URLError as e:
        if hasattr(e.reason, 'reason') and e.reason.reason == 'UNKNOWN_PROTOCOL':
            unknown(
                "TLS error connecting to registry {} for {}, should you use the '--insecure-registry' flag?" \
                    .format(registry, container))
            return
        elif hasattr(e.reason, 'strerror') and e.reason.strerror == 'nodename nor servname provided, or not known':
            unknown(
                "Cannot reach registry for {} at {}".format(container, url))
            return
        else:
            raise e
    except RegistryError as e:
        unknown("Cannot check version, couldn't retrieve digest for {} while checking {}.".format(container, url))
        return
    logger.debug("Image digests, local={} remote={}".format(image_id, registry_hash))
    if registry_hash == image_id:
        ok("{}'s version matches registry".format(container))
        return
    critical("{}'s version does not match registry".format(container))


def calculate_cpu_capacity_precentage(info, stats):
    host_config = info['HostConfig']

    if 'online_cpus' in stats['cpu_stats']:
        num_cpus = stats['cpu_stats']['online_cpus']
    else:
        num_cpus = len(stats['cpu_stats']['cpu_usage']['percpu_usage'])

    # Identify limit system being used
    # --cpus
    if 'NanoCpus' in host_config and host_config['NanoCpus'] != 0:
        period = 1000000000
        quota = host_config['NanoCpus']
    # --cpu-quota
    elif 'CpuQuota' in host_config and host_config['CpuQuota'] != 0:
        period = 100000 if host_config['CpuPeriod'] == 0 else host_config['CpuPeriod']
        quota = host_config['CpuQuota']
    # unlimited
    else:
        period = 1
        quota = num_cpus

    if period * num_cpus < quota:
        # This handles the case where the quota is actually bigger than amount available by all the cpus.
        available_limit_ratio = 1
    else:
        available_limit_ratio = (period * num_cpus) / quota

    cpu_delta = stats['cpu_stats']['cpu_usage']['total_usage'] - stats['precpu_stats']['cpu_usage']['total_usage']
    system_delta = stats['cpu_stats']['system_cpu_usage'] - stats['precpu_stats']['system_cpu_usage']
    usage = (cpu_delta / system_delta) * available_limit_ratio
    usage = round(usage * 100, 0)
    return usage


@multithread_execution()
@require_running('cpu')
def check_cpu(container, thresholds):
    info = get_container_info(container)

    stats = get_stats(container=container)

    usage = calculate_cpu_capacity_precentage(info=info, stats=stats)

    max = 100
    thresholds.units = '%'
    evaluate_numeric_thresholds(container=container, value=usage, thresholds=thresholds, name='cpu', short_name='cpu',
                                min=0, max=max)


def process_args(args):
    parser = argparse.ArgumentParser(description='Check docker containers.')

    # Connect to local socket or ip address
    connection_group = parser.add_mutually_exclusive_group()
    connection_group.add_argument('--connection',
                                  dest='connection',
                                  action='store',
                                  default=DEFAULT_SOCKET,
                                  type=str,
                                  metavar='[/<path to>/docker.socket|<ip/host address>:<port>]',
                                  help='Where to find docker daemon socket. (default: %(default)s)')

    connection_group.add_argument('--secure-connection',
                                  dest='secure_connection',
                                  action='store',
                                  type=str,
                                  metavar='[<ip/host address>:<port>]',
                                  help='Where to find TLS protected docker daemon socket.')

    base_group = parser.add_mutually_exclusive_group()
    base_group.add_argument('--binary_units',
                            dest='units_base',
                            action='store_const',
                            const=1024,
                            help='Use a base of 1024 when doing calculations of KB, MB, GB, & TB (This is default)')

    base_group.add_argument('--decimal_units',
                            dest='units_base',
                            action='store_const',
                            const=1000,
                            help='Use a base of 1000 when doing calculations of KB, MB, GB, & TB')
    parser.set_defaults(units_base=1024)

    # Connection timeout
    parser.add_argument('--timeout',
                        dest='timeout',
                        action='store',
                        type=float,
                        default=DEFAULT_TIMEOUT,
                        help='Connection timeout in seconds. (default: %(default)s)')

    # Container name
    parser.add_argument('--containers',
                        dest='containers',
                        action='store',
                        nargs='+',
                        type=str,
                        default=['all'],
                        help='One or more RegEx that match the names of the container(s) to check. If omitted all containers are checked. (default: %(default)s)')

    # Container name
    parser.add_argument('--present',
                        dest='present',
                        default=False,
                        action='store_true',
                        help='Modifies --containers so that each RegEx must match at least one container.')

    # Threads
    parser.add_argument('--threads',
                        dest='threads',
                        default=DEFAULT_PARALLELISM,
                        action='store',
                        type=int,
                        help='This + 1 is the maximum number of concurent threads/network connections. (default: %(default)s)')

    # CPU
    parser.add_argument('--cpu',
                        dest='cpu',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Check cpu usage percentage taking into account any limits.')

    # Memory
    parser.add_argument('--memory',
                        dest='memory',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT:UNITS',
                        help='Check memory usage taking into account any limits. Valid values for units are %%,B,KB,MB,GB.')

    # State
    parser.add_argument('--status',
                        dest='status',
                        action='store',
                        type=str,
                        help='Desired container status (running, exited, etc).')

    # Health
    parser.add_argument('--health',
                        dest='health',
                        default=None,
                        action='store_true',
                        help="Check container's health check status")

    # Age
    parser.add_argument('--uptime',
                        dest='uptime',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Minimum container uptime in seconds. Use when infrequent crashes are tolerated.')

    # Image Age
    parser.add_argument('--image-age',
                        dest='image_age',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Maximum image age in days.')

    # Version
    parser.add_argument('--version',
                        dest='version',
                        default=None,
                        action='store_true',
                        help='Check if the running images are the same version as those in the registry. Useful for finding stale images. Does not support login.')

    # Version
    parser.add_argument('--insecure-registries',
                        dest='insecure_registries',
                        action='store',
                        nargs='+',
                        type=str,
                        default=[],
                        help='List of registries to connect to with http(no TLS). Useful when using "--version" with images from insecure registries.')

    # Restart
    parser.add_argument('--restarts',
                        dest='restarts',
                        action='store',
                        type=str,
                        metavar='WARN:CRIT',
                        help='Container restart thresholds.')

    # no-ok
    parser.add_argument('--no-ok',
                        dest='no_ok',
                        action='store_true',
                        help='Make output terse suppressing OK messages. If all checks are OK return a single OK.')

    # no-performance
    parser.add_argument('--no-performance',
                        dest='no_performance',
                        action='store_true',
                        help='Suppress performance data. Reduces output when performance data is not being used.')

    parser.add_argument('-V', action='version', version='%(prog)s {}'.format(__version__))

    if len(args) == 0:
        parser.print_help()

    parsed_args = parser.parse_args(args=args)

    global timeout
    timeout = parsed_args.timeout

    global daemon
    global connection_type
    if parsed_args.secure_connection:
        daemon = 'https://' + parsed_args.secure_connection
        connection_type = 'https'
    elif parsed_args.connection:
        if parsed_args.connection[0] == '/':
            daemon = 'socket://' + parsed_args.connection + ':'
            connection_type = 'socket'
        else:
            daemon = 'http://' + parsed_args.connection
            connection_type = 'http'

    return parsed_args


def no_checks_present(parsed_args):
    # Look for all functions whose name starts with 'check_'
    checks = [key[6:] for key in globals().keys() if key.startswith('check_')]
    # Act like --present is a check though it is not implemented like one
    return all(getattr(parsed_args, check) is None for check in checks) and not parsed_args.present


def socketfile_permissions_failure(parsed_args):
    if connection_type == 'socket':
        return not (os.path.exists(parsed_args.connection)
                    and stat.S_ISSOCK(os.stat(parsed_args.connection).st_mode)
                    and os.access(parsed_args.connection, os.R_OK)
                    and os.access(parsed_args.connection, os.W_OK))
    else:
        return False


def print_results():
    if no_ok:
        # Remove all the "OK"s
        filtered_messages = [message for message in messages if not message.startswith('OK: ')]
        if len(filtered_messages) == 0:
            messages_concat = 'OK'
        else:
            messages_concat = '; '.join(filtered_messages)

    else:
        messages_concat = '; '.join(messages)

    if no_performance or len(performance_data) == 0:
        print(messages_concat)
    else:
        perfdata_concat = ' '.join(performance_data)
        print(messages_concat + '|' + perfdata_concat)


def perform_checks(raw_args):
    args = process_args(raw_args)

    global parallel_executor
    parallel_executor = futures.ThreadPoolExecutor(max_workers=args.threads)
    global serial_executor
    serial_executor = futures.ThreadPoolExecutor(max_workers=1)

    global unit_adjustments
    unit_adjustments = {key: args.units_base ** value for key, value in UNIT_ADJUSTMENTS_TEMPLATE.items()}

    global no_ok
    no_ok = args.no_ok

    global no_performance
    no_performance = args.no_ok

    if socketfile_permissions_failure(args):
        unknown("Cannot access docker socket file. User ID={}, socket file={}".format(os.getuid(), args.connection))
        return

    if args.containers == ["all"] and args.present:
        unknown("You can not use --present without --containers")
        return

    if no_checks_present(args):
        unknown("No checks specified.")
        return

    # Here is where all the work happens
    #############################################################################################
    containers = get_containers(args.containers, args.present)

    if len(containers) == 0 and not args.present:
        unknown("No containers names found matching criteria")
        return

    for container in containers:

        # Check status
        if args.status:
            check_status(container, args.status)

        # Check version
        if args.version:
            check_version(container, args.insecure_registries)

        # below are checks that require a 'running' status

        # Check status
        if args.health:
            check_health(container)

        # Check cpu usage
        if args.cpu:
            check_cpu(container, parse_thresholds(args.cpu, units_required=False))

        # Check memory usage
        if args.memory:
            check_memory(container, parse_thresholds(args.memory, units_required=False))

        # Check uptime
        if args.uptime:
            check_uptime(container, parse_thresholds(args.uptime, include_units=False))

        # Check image age
        if args.image_age:
            check_image_age(container, parse_thresholds(args.image_age, include_units=False))

        # Check restart count
        if args.restarts:
            check_restarts(container, parse_thresholds(args.restarts, include_units=False))


def main():
    try:
        perform_checks(argv[1:])

        # get results to let exceptions in threads bubble out
        [x.result() for x in futures.as_completed(threads)]

    except Exception as e:
        traceback.print_exc()
        unknown("Exception raised during check': {}".format(repr(e)))
    print_results()
    exit(rc)


if __name__ == '__main__':
    main()