ansible/roles/prometheus/templates/prometheus.yml.j2

# vim:ft=ansible:
---
global:
  scrape_interval: 15s
  evaluation_interval: 15s

scrape_configs:
    # Default Prometheus job to monitor itself
  - job_name: "prometheus"
    static_configs:
      - targets: ['localhost:9090']

    # This is shipped by the Ansible role that deploys Prometheus
  - job_name: "blackbox-ssh"
    metrics_path: /probe
    params:
      module: [ssh_banner]
    static_configs:
      - targets:
{% for host in groups['tags_nagios'] %}
        - "{{ host }}:22"
{% endfor %}
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115
  - job_name: "blackbox-http"
    metrics_path: /probe
    params:
      module: [http_2xx]
    static_configs:
      - targets:
{% for host in groups['tags_nagios'] %}
{% set vars = hostvars[host] %}
{% for service in vars.services %}
{% for tag in service.tags %}
{# #}
{% if tag.slug == "nagios-checkhttps" %}
{% for port in service.ports %}
        - "https://{{ service.name }}:{{ port }}"
{% endfor %}
{% endif %}
{# #}
{% if tag.slug == "nagios-checkmatrix" %}
{% for port in service.ports %}
        - "https://{{ service.name }}:{{ port }}/health"
{% endfor %}
{% endif %}
{# #}
{% endfor %}
{% endfor %}
{% endfor %}
    relabel_configs:
      - source_labels: [__address__]
        target_label: __param_target
      - source_labels: [__param_target]
        target_label: instance
      - target_label: __address__
        replacement: blackbox:9115
  - job_name: "blackbox-exporter"
    static_configs:
      - targets: ['blackbox:9115']

    # This job manages Minecraft servers, when we have them
  - job_name: "minecraft"
    static_configs:
      - targets:
{% for host in groups['tags_nagios'] %}
{% set vars = hostvars[host] %}
{% for service in vars.services %}
{% for tag in service.tags %}
{# #}
{% if tag.slug == "nagios-checkminecraft" %}
{% for port in service.ports %}
        - "{{ host }}:{{ port }}"
{% endfor %}
{% endif %}
{# #}
{% endfor %}
{% endfor %}
{% endfor %}

    # This job takes in information from Netbox on the generic "prom-metrics" tag
    # It's useful for all sorts of stuff
  - job_name: "generic"
    scheme: "https"
    static_configs:
      - targets:
{% for host in groups['tags_nagios'] %}
{% set vars = hostvars[host] %}
{% for service in vars.services %}
{% for tag in service.tags %}
{# #}
{% if tag.slug == "prom-metrics" %}
{% for port in service.ports %}
        - "{{ service.name }}:{{ port }}"
{% endfor %}
{% endif %}
{# #}
{% endfor %}
{% endfor %}
{% endfor %}

    # This one does the same thing but for HTTP-only clients
  - job_name: "generic-http"
    scheme: "http"
    static_configs:
      - targets:
{% for host in groups['tags_nagios'] %}
{% set vars = hostvars[host] %}
{% for service in vars.services %}
{% for tag in service.tags %}
{# #}
{% if tag.slug == "prom-metrics-http" %}
{% for port in service.ports %}
        - "{{ service.name }}:{{ port }}"
{% endfor %}
{% endif %}
{# #}
{% endfor %}
{% endfor %}
{% endfor %}
    # These two jobs are included for every node in our inventory
  - job_name: "node-exporter"
    static_configs:
      - targets:
{% for host in groups['tags_nagios'] %}
          - '{{ host }}:9100'
{% endfor %}
  - job_name: "cadvisor-exporter"
    static_configs:
      - targets:
{% for host in groups['tags_nagios'] %}
          - '{{ host }}:9101'
{% endfor %}

    # This container is used on every psql machine
  - job_name: "psql-exporter"
    static_configs:
      - targets:
{% for host in groups['tags_nagios-checkpgsql'] %}
          - '{{ host }}:9102'
{% endfor %}