#!/usr/bin/env ansible-playbook # vim:ft=ansible: # Webservers --- - hosts: vm-general-1.ashburn.mgmt.desu.ltd gather_facts: no module_defaults: docker_container: restart_policy: unless-stopped pull: yes pre_tasks: - name: ensure docker network docker_network: name=web tags: [ docker ] tasks: - name: include tasks for applications include_tasks: tasks/{{ item }} with_items: - app/gitlab-runner.yml - app/redis.yml - web/9iron.yml - web/desultd.yml - web/element-web.yml - web/gitea.yml - web/netbox.yml - web/nextcloud.yml - web/prowlarr.yml - web/radarr.yml - web/sonarr.yml - web/srv.yml - web/synapse.yml - web/transmission.yml - game/factorio.yml - game/minecraft-vanilla.yml - game/minecraft-direwolf20.yml - game/zomboid.yml tags: [ always ] roles: - role: backup vars: backup_s3backup_list_extra: - /app/gitea/gitea - /data backup_s3backup_exclude_list_extra: - /var/lib/gitea/log - /data/gitea/data/gitea/log - /data/minecraft/oldpack/backups - /data/minecraft/stoneblock/backups - /data/minecraft/create-extra/backups - /data/minecraft/direwolf20/backups - /data/minecraft/prominence/FeedTheBeast/world/.git - /data/sb-mirror - /data/shared/media - /data/shared/downloads - /data/terraria/generic/backups tags: [ backup ] # - role: docker-tmodloader14 # tags: [ terraria, tmodloader ] # - role: docker-tmodloader14 # vars: # tmodloader_external_port: "7778" # tmodloader_name: "test" # tags: [ terraria-test, tmodloader-test ] - role: git vars: git_repos: - repo: https://git.desu.ltd/salt/gitea-custom dest: /data/gitea/data/gitea/custom tags: [ web, git ] - role: nagios vars: nagios_matrix_server: "https://matrix.desu.ltd" nagios_matrix_room: "!NWNCKlNmOTcarMcMIh:desu.ltd" nagios_matrix_token: "{{ secret_nagios_matrix_token }}" nagios_data_dir: /data/nagios nagios_admin_pass: "{{ secret_nagios_admin_pass }}" nagios_contacts: - name: matrix host_notification_commands: notify-host-by-matrix service_notification_commands: notify-service-by-matrix host_notification_period: ansible-not-late-at-night service_notification_period: ansible-not-late-at-night extra: - key: contactgroups value: ansible - name: salt host_notification_commands: notify-host-by-email service_notification_commands: notify-service-by-email extra: - key: email value: alerts@babor.tech nagios_commands: # This command is included in the container image - name: check_nrpe command: "$USER1$/check_nrpe -H $HOSTADDRESS$ -c $ARG1$" - name: check_by_ssh command: "$USER1$/check_by_ssh -H $HOSTADDRESS$ -F /opt/nagios/etc/ssh_config -t 30 -q -i /opt/nagios/etc/id_ed25519 -l nagios-checker -C \"$ARG1$\"" - name: notify-host-by-matrix command: "/usr/bin/printf \"%b\" \"$NOTIFICATIONTYPE$\\n$HOSTNAME$ is $HOSTSTATE$\\nAddress: $HOSTADDRESS$\\nInfo: $HOSTOUTPUT$\\nDate/Time: $LONGDATETIME$\" | /opt/Custom-Nagios-Plugins/notify-by-matrix" - name: notify-service-by-matrix command: "/usr/bin/printf \"%b\" \"$NOTIFICATIONTYPE$\\nService $HOSTALIAS$ - $SERVICEDESC$ is $SERVICESTATE$\\nInfo: $SERVICEOUTPUT$\\nDate/Time: $LONGDATETIME$\" | /opt/Custom-Nagios-Plugins/notify-by-matrix" nagios_services: # Agentless checks - name: HTTP command: check_http hostgroup: tag-nagios-checkhttp - name: HTTPS command: check_http!--ssl hostgroup: tag-nagios-checkhttp - name: SSH command: check_ssh # check_by_ssh checks - name: CPU Utilization command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_cpu_stats -w 75 -c 90 - name: DNS Resolution command: check_by_ssh!/usr/lib/nagios/plugins/check_etc_resolv - name: Executables in tmp command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_executables_in_tmp - name: Last Ansible Play command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_file_age /var/lib/ansible-last-run -w 432000 -c 604800 - name: Memory Usage command: check_by_ssh!/usr/lib/nagios/plugins/check_memory -w 10% -c 5% hostgroup: "ansible,!tag-prov-zfs" - name: Ping Self over DNS command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_ping_by_hostname - name: Reboot Required command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_reboot_required - name: Unit atd.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit atd.service - name: Unit backup.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit backup.service hostgroup: "ansible,!role-hypervisor" - name: Unit backup.timer command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit backup.timer hostgroup: "ansible,!role-hypervisor" - name: Unit cron.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit cron.service - name: Unit dbus.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit dbus.service - name: Unit ssh.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit ssh.service - name: Unit systemd-resolved.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit systemd-resolved.service hostgroup: "ansible,!role-hypervisor" - name: Users command: check_by_ssh!/usr/lib/nagios/plugins/check_users -w 3 -c 5 # Privileged checks # Required because check_disk may attempt to get the free space of # restricted mountpoints - name: Disk Usage command: check_by_ssh!/usr/bin/sudo /usr/lib/nagios/plugins/check_disk -M -u GB -X nfs -X tracefs -X cgroup -X tmpfs -X overlay -X shm -w 15% -c 10% -W 15% -K 10% -A -I '^/run/' -I '^udev$' -I '^/var/lib/kubelet/' -I '^/tmp/.mount_' -I '^/dev/loop' # Device type checks # R720 - name: CPU0 Temperature command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_temp -n -w 65 -c 75 --sensor coretemp-isa-0000 hostgroup: device-type-r720 - name: CPU1 Temperature command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_temp -n -w 65 -c 75 --sensor coretemp-isa-0001 hostgroup: device-type-r720 # Pi 4 4G - name: CPU Temperature command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_temp -n -w 65 -c 75 --sensor cpu_thermal-virtual-0 hostgroup: device-type-pi4b-2g,device-type-pi4b-4g,device-type-pi4b-4g-storage # Device role checks # hypervisor (which is assumed to be Proxmox) - name: PVE Unit pve-firewall.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pve-firewall.service hostgroup: role-hypervisor - name: PVE Unit spiceproxy.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit spiceproxy.service hostgroup: role-hypervisor - name: PVE Unit pve-ha-crm.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pve-ha-crm.service hostgroup: role-hypervisor - name: PVE Unit pvedaemon.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pvedaemon.service hostgroup: role-hypervisor - name: PVE Unit pvefw-logger.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pvefw-logger.service hostgroup: role-hypervisor - name: PVE Unit pveproxy.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pveproxy.service hostgroup: role-hypervisor - name: PVE Unit pve-cluster.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pve-cluster.service hostgroup: role-hypervisor - name: PVE Unit pvestatd.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pvestatd.service hostgroup: role-hypervisor # Tag-specific checks # docker - name: Unit docker.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit docker.service hostgroup: "ansible,!tag-no-docker" - name: Docker Status command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_docker --no-ok --status running hostgroup: tag-nagios-checkdocker # nagios-checkpgsql - name: PSQL command: "check_by_ssh!/usr/lib/nagios/plugins/check_pgsql -H localhost -l nagios -p {{ secret_postgresql_monitoring_password }} -w 2 -c 5" hostgroup: tag-nagios-checkpgsql - name: PSQL Connections command: "check_by_ssh!/usr/lib/nagios/plugins/check_pgsql -H localhost -l nagios -p {{ secret_postgresql_monitoring_password }} -w 2 -c 5 -q 'select (select count(*)::float used from pg_stat_activity) / (select setting::int max_conn from pg_settings where name=\\$\\$max_connections\\$\\$)' -W 0.7-0.8 -C 0.8-1.0" hostgroup: tag-nagios-checkpgsql # https://rhaas.blogspot.com/2020/02/useless-vacuuming.html - name: PSQL Old Xacts command: "check_by_ssh!/usr/lib/nagios/plugins/check_pgsql -H localhost -l nagios -p {{ secret_postgresql_monitoring_password }} -w 2 -c 5 -q 'select count(*)::float from pg_prepared_xacts where age(transaction) > 5000000' -W 500-1000 -C 1000-1000000" hostgroup: tag-nagios-checkpgsql - name: Unit postgresql.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit postgresql.service hostgroup: tag-nagios-checkpgsql # nagios-checkswap - name: Swap Usage command: check_by_ssh!/usr/lib/nagios/plugins/check_swap -w 20% -c 10% hostgroup: tag-nagios-checkswap # zerotier - name: Unit zerotier-one.service command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit zerotier-one.service hostgroup: tag-zt-personal tags: [ nagios, no-auto ] - role: ingress vars: ingress_servers: # desu.ltd - name: desu.ltd proxy_pass: http://desultd:80 locations: - location: /.well-known/matrix/server contents: | default_type application/json; return 200 '{"m.server":"matrix.desu.ltd:443"}'; - location: /.well-known/matrix/client contents: | default_type application/json; return 200 '{"m.homeserver":{"base_url":"https://matrix.desu.ltd"}}'; - name: git.desu.ltd proxy_pass: http://gitea:3000 - name: matrix.desu.ltd proxies: - location: "~* ^(\/_matrix|\/_synapse|\/client|\/health)" pass: http://synapse:8008 - location: / pass: http://element:80 directives: - "client_max_body_size 0" - name: nagios.desu.ltd proxy_pass: http://nagios:80 - name: nc.desu.ltd directives: - "add_header Strict-Transport-Security \"max-age=31536000\"" - "client_max_body_size 0" proxy_pass: http://nextcloud:80 locations: - location: "^~ /.well-known" contents: | location = /.well-known/carddav { return 301 /remote.php/dav/; } location = /.well-known/caldav { return 301 /remote.php/dav/; } location ^~ /.well-known { return 301 /index.php$uri; } try_files $uri $uri/ =404; - name: netbox.desu.ltd proxy_pass: http://netbox:8080 # desu.ltd media bullshit - name: prowlarr.media.desu.ltd directives: - "allow {{ common_home_address }}/{{ common_home_address_mask }}" - "deny all" proxy_pass: http://prowlarr:9696 - name: sonarr.media.desu.ltd directives: - "allow {{ common_home_address }}/{{ common_home_address_mask }}" - "deny all" proxy_pass: http://sonarr:8989 - name: radarr.media.desu.ltd directives: - "allow {{ common_home_address }}/{{ common_home_address_mask }}" - "deny all" proxy_pass: http://radarr:7878 - name: transmission.media.desu.ltd directives: - "allow {{ common_home_address }}/{{ common_home_address_mask }}" - "deny all" proxy_pass: http://transmission:9091 # 9iron - name: www.9iron.club directives: - "return 301 $scheme://9iron.club$request_uri" - name: 9iron.club proxy_pass: http://9iron:80 - name: srv.9iron.club proxy_pass: http://srv:80 tags: [ web, docker, ingress ]