Create an Ansible contactgroup in Nagios and tie it to all alerts, enroll our Matrix user in that group

inventory/group_vars/all.yml

@@ -138,6 +138,14 @@ secret_nagios_admin_pass: !vault |
           34616338636630633539353335336631313361656633333539323130626132356263653436343363
           3930323538613137370a373861376566376631356564623665313662636562626234643862343863
           61326232633266633262613931303631396163326266386363366639366639613938
+secret_nagios_matrix_token: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          66366665666437643765366533646666386162393038653262333461376566333366363332643135
+          6233376362633566303939623832636366333330393238370a323766366164393733383736633435
+          37633137626634643530653665613166633439376333633663633561313864396465623036653063
+          6433376138386531380a383762393137613738643538343438633730313135613730613139393536
+          35666133666262383862663637623738643836383633653864626231623034613662646563623936
+          3763356331333561383833386162616664376335333139376363
 
 # For Netbox
 secret_netbox_user_pass: !vault |

playbooks/prod_web.yml

@@ -140,9 +140,18 @@
       # TODO: Replace this with Naemon(?)
     - role: nagios
       vars:
+        nagios_matrix_server: "https://matrix.desu.ltd"
+        nagios_matrix_room: "!QJpSrEHPWSAHwdUMgy:desu.ltd"
+        nagios_matrix_token: "{{ secret_nagios_matrix_token }}"
         nagios_data_dir: /data/nagios
         nagios_admin_pass: "{{ secret_nagios_admin_pass }}"
         nagios_contacts:
+          - name: matrix
+            host_notification_commands: notify-host-by-matrix
+            service_notification_commands: notify-service-by-matrix
+            extra:
+              - key: contactgroups
+                value: ansible
           - name: salt
             host_notification_commands: notify-host-by-email
             service_notification_commands: notify-service-by-email
@@ -155,6 +164,10 @@
             command: "$USER1$/check_nrpe -H $HOSTADDRESS$ -c $ARG1$"
           - name: check_by_ssh
             command: "$USER1$/check_by_ssh -H $HOSTADDRESS$ -F /opt/nagios/etc/ssh_config -t 30 -q -i /opt/nagios/etc/id_ed25519 -l nagios-checker -C \"$ARG1$\""
+          - name: notify-host-by-matrix
+            command: "/usr/bin/printf \"%b\" \"$NOTIFICATIONTYPE$ - $HOSTNAME$ is $HOSTSTATE$\\nAddress: $HOSTADDRESS$\\nInfo: $HOSTOUTPUT$\\nDate/Time: $LONGDATETIME$\" | /opt/Custom-Nagios-Plugins/notify-by-matrix"
+          - name: notify-service-by-matrix
+            command: "/usr/bin/printf \"%b\" \"$NOTIFICATIONTYPE$ - Service $HOSTALIAS$/$SERVICEDESC$ is $SERVICESTATE$\\nInfo: $SERVICEOUTPUT$\\nDate/Time: $LONGDATETIME$\" | /opt/Custom-Nagios-Plugins/notify-by-matrix"
         nagios_services:
           # Agentless checks
           - name: HTTP

roles/nagios/tasks/main.yml

@@ -27,6 +27,11 @@
       - /dev/null:/opt/nagios/bin/nsca
       - /dev/null:/opt/nagios/bin/send_nsca
   tags: [ docker, nagios ]
+- name: template out scripts for nagios
+  template: src="{{ item }}" dest="{{ nagios_data_dir }}/plugins/{{ item }}" owner=root group=root mode=0755
+  with_items:
+    - notify-by-matrix
+  tags: [ nagios, template, plugins ]
 - name: template out config for nagios
   template: src=nagios-ansible.cfg.j2 dest="{{ nagios_data_dir }}/etc/objects/ansible.cfg" owner=root group=root mode=0644
   tags: [ nagios, template ]

roles/nagios/templates/nagios-ansible.cfg.j2

@@ -54,6 +54,13 @@ define contact {
 {% endfor %}
 {% endif %}
 
+# And a contactgroup
+define contactgroup {
+	contactgroup_name		ansible
+	alias				Ansible notification contacts
+	members				nagiosadmin
+}
+
 {% if nagios_commands is defined %}
 # Commands
 # Everything here is defined in nagios_commands
@@ -79,6 +86,7 @@ define service {
 	service_description	{{ service.name }}
 	check_command		{{ service.command }}
 	hostgroup_name		{{ service.hostgroup | default('ansible', true) }}
+	contact_groups		ansible
 	{% if service.extra is defined %}
 	{% for kvp in service.extra %}
 	{{ kvp.key }} {{ kvp.value }}
@@ -121,6 +129,7 @@ define host {
 	address			{{ host.value.primary_ip.address.split('/',1)[0] }}
 	hostgroups		ansible{% for tag in host.value.tags %},tag-{{ tag.slug }}{% endfor %}{% if host.value.device_role is defined -%},role-{{ host.value.device_role.slug }}{% endif %}{% if host.value.role is defined %},role-{{ host.value.role.slug }}{% endif %}
 
+	contact_groups		ansible
 }
 {% if host.value.config_context.extra_checks is defined %}
 {% for check in host.value.config_context.extra_checks %}
@@ -130,6 +139,7 @@ define host {
 		service_description	{{ check.description }}
 		check_command		{{ check.command }}
 		host_name		{{ host.value.name }}
+		contact_groups		ansible
 	}
 {% endfor %}
 {% endif %}
@@ -159,6 +169,7 @@ define service {
 	service_description	Minecraft - {{ service.value.name }} - {{ port }}
 	check_command		check_by_ssh!/usr/local/bin/monitoring-scripts/check_minecraft -H {{ host_name }} -p {{ port }} -m "{{ service.value.description }}" -f -w 3 -c 5
 	host_name		{{ host_name }}
+	contact_groups		ansible
 }
 {% endfor %}
 {% endif %}
@@ -170,6 +181,7 @@ define service {
 	service_description	HTTP - {{ service.value.name }} - {{ port }}
 	check_command		check_http!-H {{ service.value.name }} -p {{ port }} -f sticky
 	host_name		{{ host_name }}
+	contact_groups		ansible
 }
 {% endfor %}
 {% endif %}
@@ -181,6 +193,7 @@ define service {
 	service_description	HTTPS - {{ service.value.name }} - {{ port }}
 	check_command		check_http!--ssl -H {{ service.value.name }} -p {{ port }} -f sticky
 	host_name		{{ host_name }}
+	contact_groups		ansible
 }
 {% endfor %}
 {% endif %}
@@ -192,6 +205,7 @@ define service {
 	service_description	TCP {{ service.value.name }} - {{ port }}
 	check_command		check_tcp!{{ port }}
 	host_name		{{ host_name }}
+	contact_groups		ansible
 }
 {% endfor %}
 {% endif %}

roles/nagios/templates/notify-by-matrix

@@ -0,0 +1,34 @@
+#! /bin/sh
+#
+# notify-by-matrix
+# Copyright (C) 2021 Vintage Salt <rehashedsalt@cock.li>
+#
+# Distributed under terms of the MIT license.
+#
+
+set -e
+
+# Set our Matrix-related vars here
+MX_TOKEN="{{ nagios_matrix_token }}"
+MX_SERVER="{{ nagios_matrix_server }}"
+MX_ROOM="{{ nagios_matrix_room }}"
+
+# Get a TXN to prefix this particular message with
+MX_TXN="$(date "+%s")$(( RANDOM % 9999 ))"
+
+# Read a message from STDIN
+# NOTE: This is dangerous and stupid and unsanitized
+read message
+while read line; do
+	message="${message}\n${line}"
+done
+
+# Push it to the channel
+curl -X PUT \
+	--header 'Content-Type: application/json' \
+	--header 'Accept: application/json' \
+	-d "{
+		\"msgtype\": \"m.text\",
+		\"body\": \"$message\"
+	}" \
+	"$MX_SERVER/_matrix/client/unstable/rooms/$MX_ROOM/send/m.room.message/$MX_TXN?access_token=$MX_TOKEN"