Add Minecraft monitoring!

This also fixes some duplicate alert structures we saw earlier

inventories/production/group_vars/all.yml

@@ -202,8 +202,6 @@ nagios_commands:
   - name: notify-service-by-matrix
     command: "/usr/bin/printf \"%b\" \"$NOTIFICATIONTYPE$\\nService $HOSTALIAS$ - $SERVICEDESC$ is $SERVICESTATE$\\nInfo: $SERVICEOUTPUT$\\nDate/Time: $LONGDATETIME$\" | /opt/Custom-Nagios-Plugins/notify-by-matrix"
 nagios_services:
-  - name: SSH
-    command: check_ssh
   # check_by_ssh checks
   - name: Last Ansible Play
     command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_file_age /var/lib/ansible-last-run -w 432000 -c 604800

playbooks/tasks/game/minecraft-createfarming.yml

@@ -16,6 +16,12 @@
       - "25565:25565/tcp"
       - "25565:25565/udp"
       - "24454:24454/udp"
+      # Prometheus exporter for Forge
+      # https://www.curseforge.com/minecraft/mc-mods/prometheus-exporter
+      #- "19565:19565/tcp"
+      # Prometheus exporter for Fabric
+      # https://modrinth.com/mod/fabricexporter
+      - "19565:25585/tcp"
     volumes:
       - /data/minecraft/createfarming:/data
   tags: [ docker, minecraft, create, createfarming ]

roles/prometheus/templates/blackbox.yml.j2

@@ -10,3 +10,10 @@ modules:
       valid_http_versions: ["HTTP/1.1", "HTTP/2.0"]
       valid_status_codes: []  # Defaults to 2xx
       method: GET
+  ssh_banner:
+    prober: tcp
+    timeout: 3s
+    tcp:
+      query_response:
+        - expect: "^SSH-2.0-"
+        - send: "SSH-2.0-blackbox-ssh-check"

roles/prometheus/templates/prometheus.yml.j2

@@ -11,13 +11,30 @@ scrape_configs:
       - targets: ['localhost:9090']
 
     # This is shipped by the Ansible role that deploys Prometheus
-  - job_name: "blackbox"
+  - job_name: "blackbox-ssh"
+    metrics_path: /probe
+    params:
+      module: [ssh_banner]
+    static_configs:
+      - targets:
+{% for host in groups['tags_nagios'] %}
+        - "{{ host }}:22"
+{% endfor %}
+    relabel_configs:
+      - source_labels: [__address__]
+        target_label: __param_target
+      - source_labels: [__param_target]
+        target_label: instance
+      - target_label: __address__
+        replacement: blackbox:9115
+  - job_name: "blackbox-http"
     metrics_path: /probe
     params:
       module: [http_2xx]
     static_configs:
       - targets:
 {% for host in groups['tags_nagios'] %}
+{% set vars = hostvars[host] %}
 {% for service in vars.services %}
 {% for tag in service.tags %}
 {# #}
@@ -47,6 +64,25 @@ scrape_configs:
     static_configs:
       - targets: ['blackbox:9115']
 
+    # This job manages Minecraft servers, when we have them
+  - job_name: "minecraft"
+    static_configs:
+      - targets:
+{% for host in groups['tags_nagios'] %}
+{% set vars = hostvars[host] %}
+{% for service in vars.services %}
+{% for tag in service.tags %}
+{# #}
+{% if tag.slug == "nagios-checkminecraft" %}
+{% for port in service.ports %}
+        - "{{ host }}:{{ port }}"
+{% endfor %}
+{% endif %}
+{# #}
+{% endfor %}
+{% endfor %}
+{% endfor %}
+
     # These two jobs are included for every node in our inventory
   - job_name: "node-exporter"
     static_configs: