ansible/playbooks/prod_web.yml

329 lines
15 KiB
YAML
Raw Normal View History

#!/usr/bin/env ansible-playbook
# vim:ft=ansible:
# Webservers
---
2021-10-15 19:03:55 -05:00
- hosts: web1.dallas.mgmt.desu.ltd
gather_facts: no
2021-06-18 12:39:21 -05:00
module_defaults:
docker_container:
state: started
restart_policy: unless-stopped
pull: yes
2021-03-25 12:59:40 -05:00
tasks:
- name: ensure docker network
docker_network: name=web
tags: [ docker ]
2021-09-18 07:23:03 -05:00
- name: include tasks for applications
include_tasks: tasks/{{ item }}
2021-08-07 16:49:24 -05:00
with_items:
2021-09-18 07:23:03 -05:00
- app/redis.yml
- web/9iron.yml
- web/desultd.yml
- web/gitea.yml
- web/nextcloud.yml
- web/srv.yml
2021-08-07 16:49:24 -05:00
tags: [ always ]
roles:
2020-12-24 09:19:12 -06:00
- role: backup
vars:
backup_s3backup_list_extra:
2021-06-14 17:16:48 -05:00
- /app/gitea/gitea
- /data
- /srv/desu.ltd
2021-01-14 18:06:13 -06:00
backup_s3backup_exclude_list_extra:
- /var/lib/gitea/log
- /data/gitea/data/gitea/log
2020-12-24 09:19:12 -06:00
tags: [ backup ]
- role: git
vars:
git_repos:
- repo: https://git.desu.ltd/salt/gitea-custom
dest: /data/gitea/data/gitea/custom
tags: [ web, git ]
- role: ingress
vars:
ingress_servers:
# desu.ltd
- name: desu.ltd
proxy_pass: http://desultd:80
2021-10-02 21:53:44 -05:00
locations:
- location: /.well-known/matrix/server
contents: |
2021-10-02 21:53:44 -05:00
default_type application/json;
return 200 '{"m.server":"matrix.desu.ltd:443"}';
- location: /.well-known/matrix/client
contents: |
2021-10-02 21:53:44 -05:00
default_type application/json;
return 200 '{"m.homeserver":{"base_url":"https://matrix.desu.ltd"}}';
- name: git.desu.ltd
proxy_pass: http://gitea:3000
- name: nc.desu.ltd
directives:
- "add_header Strict-Transport-Security \"max-age=31536000\""
- "client_max_body_size 0"
proxy_pass: http://nextcloud:80
locations:
- location: "^~ /.well-known"
contents: |
location = /.well-known/carddav { return 301 /remote.php/dav/; }
location = /.well-known/caldav { return 301 /remote.php/dav/; }
location ^~ /.well-known { return 301 /index.php$uri; }
try_files $uri $uri/ =404;
# 9iron
- name: www.9iron.club
directives:
- "return 301 $scheme://9iron.club$request_uri"
- name: 9iron.club
proxy_pass: http://9iron:80
- name: srv.9iron.club
proxy_pass: http://srv:80
tags: [ web, docker, ingress ]
2021-10-15 19:03:55 -05:00
- hosts: web2.dallas.mgmt.desu.ltd
gather_facts: no
2021-06-07 00:45:52 -05:00
module_defaults:
docker_container:
state: started
restart_policy: unless-stopped
pull: yes
2021-11-29 23:33:56 -06:00
pre_tasks:
- name: ensure docker network
docker_network: name=web
tags: [ docker ]
2021-09-18 07:23:03 -05:00
- name: include tasks for applications
include_tasks: tasks/{{ item }}
2021-08-07 16:49:24 -05:00
with_items:
2021-09-18 07:23:03 -05:00
- app/redis.yml
2022-01-22 09:48:47 -06:00
- web/jenkins.yml
2021-09-18 07:23:03 -05:00
- web/pleroma.yml
2021-08-07 16:49:24 -05:00
tags: [ always ]
2021-01-25 22:19:31 -06:00
roles:
- role: backup
vars:
backup_s3backup_list_extra:
- /data
2021-01-25 22:19:31 -06:00
tags: [ backup ]
- role: ingress
vars:
ingress_servers:
- name: cowfee.moe
proxy_pass: http://pleroma:4000
2022-01-22 09:48:47 -06:00
- name: jenkins.desu.ltd
locations:
- location: "/"
contents: |
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-Proto https;
proxy_set_header X-Forwarded-Port 443;
proxy_pass http://jenkins:8080;
2022-01-22 09:48:47 -06:00
- location: |
~ "^/static/[0-9a-fA-F]{8}\/(.*)$"
contents: |
rewrite "^/static/[0-9a-fA-F]{8}\/(.*)" /$1 last;
- location: "/userContent"
contents: |
root /data/jenkins/home/;
if (!-f $request_filename) {
rewrite (.*) /$1 last;
break;
}
tags: [ web, docker, ingress ]
2021-10-15 19:03:55 -05:00
- hosts: web3.dallas.mgmt.desu.ltd
module_defaults:
docker_container:
state: started
restart_policy: unless-stopped
pull: yes
tasks:
- name: ensure docker network
docker_network: name=web
tags: [ docker ]
2021-09-18 07:23:03 -05:00
- name: include tasks for applications
include_tasks: tasks/{{ item }}
2021-08-07 16:49:24 -05:00
with_items:
2021-09-18 07:23:03 -05:00
- app/redis.yml
- web/netbox.yml
2022-04-23 10:54:26 -05:00
- web/ara.yml
2021-08-07 16:49:24 -05:00
tags: [ always ]
roles:
- role: backup
vars:
backup_s3backup_list_extra:
- /data
tags: [ backup ]
# TODO: Replace this with Naemon(?)
- role: nagios
vars:
nagios_matrix_server: "https://matrix.desu.ltd"
2022-01-27 14:31:58 -06:00
nagios_matrix_room: "!NWNCKlNmOTcarMcMIh:desu.ltd"
nagios_matrix_token: "{{ secret_nagios_matrix_token }}"
nagios_data_dir: /data/nagios
nagios_admin_pass: "{{ secret_nagios_admin_pass }}"
nagios_contacts:
- name: matrix
host_notification_commands: notify-host-by-matrix
service_notification_commands: notify-service-by-matrix
host_notification_period: ansible-not-late-at-night
service_notification_period: ansible-not-late-at-night
extra:
- key: contactgroups
value: ansible
- name: salt
host_notification_commands: notify-host-by-email
service_notification_commands: notify-service-by-email
extra:
- key: email
2021-12-24 15:36:18 -06:00
value: alerts@babor.tech
nagios_commands:
# This command is included in the container image
- name: check_nrpe
command: "$USER1$/check_nrpe -H $HOSTADDRESS$ -c $ARG1$"
- name: check_by_ssh
command: "$USER1$/check_by_ssh -H $HOSTADDRESS$ -F /opt/nagios/etc/ssh_config -t 30 -q -i /opt/nagios/etc/id_ed25519 -l nagios-checker -C \"$ARG1$\""
- name: notify-host-by-matrix
command: "/usr/bin/printf \"%b\" \"$NOTIFICATIONTYPE$\\n$HOSTNAME$ is $HOSTSTATE$\\nAddress: $HOSTADDRESS$\\nInfo: $HOSTOUTPUT$\\nDate/Time: $LONGDATETIME$\" | /opt/Custom-Nagios-Plugins/notify-by-matrix"
- name: notify-service-by-matrix
command: "/usr/bin/printf \"%b\" \"$NOTIFICATIONTYPE$\\nService $HOSTALIAS$ - $SERVICEDESC$ is $SERVICESTATE$\\nInfo: $SERVICEOUTPUT$\\nDate/Time: $LONGDATETIME$\" | /opt/Custom-Nagios-Plugins/notify-by-matrix"
nagios_services:
# Agentless checks
- name: HTTP
command: check_http
hostgroup: tag-nagios-checkhttp
- name: HTTPS
command: check_http!--ssl
hostgroup: tag-nagios-checkhttp
- name: SSH
command: check_ssh
# check_by_ssh checks
- name: CPU Utilization
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_cpu_stats -w 75 -c 90
- name: DNS Resolution
command: check_by_ssh!/usr/lib/nagios/plugins/check_etc_resolv
2022-04-27 17:58:00 -05:00
- name: Executables in tmp
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_executables_in_tmp
- name: Last Ansible Play
2022-03-26 19:57:37 -05:00
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_file_age /var/lib/ansible-last-run -w 432000 -c 604800
- name: Memory Usage
command: check_by_ssh!/usr/lib/nagios/plugins/check_memory -w 10% -c 5%
hostgroup: "ansible,!tag-prov-zfs"
- name: Ping Self over DNS
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_ping_by_hostname
- name: Reboot Required
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_reboot_required
- name: Unit atd.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit atd.service
- name: Unit backup.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit backup.service
hostgroup: "ansible,!role-hypervisor"
- name: Unit backup.timer
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit backup.timer
hostgroup: "ansible,!role-hypervisor"
- name: Unit cron.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit cron.service
- name: Unit dbus.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit dbus.service
- name: Unit ssh.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit ssh.service
- name: Unit systemd-resolved.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit systemd-resolved.service
hostgroup: "ansible,!role-hypervisor"
- name: Users
command: check_by_ssh!/usr/lib/nagios/plugins/check_users -w 3 -c 5
# Privileged checks
# Required because check_disk may attempt to get the free space of
# restricted mountpoints
- name: Disk Usage
command: check_by_ssh!/usr/bin/sudo /usr/lib/nagios/plugins/check_disk -M -u GB -X nfs -X tracefs -X cgroup -X tmpfs -X overlay -X shm -w 15% -c 10% -W 15% -K 10% -A -I '^/run/' -I '^udev$' -I '^/var/lib/kubelet/' -I '^/tmp/.mount_' -I '^/dev/loop'
# Device type checks
# R720
- name: CPU0 Temperature
command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_temp -n -w 65 -c 75 --sensor coretemp-isa-0000
2022-01-10 23:46:26 -06:00
hostgroup: device-type-r720
- name: CPU1 Temperature
command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_temp -n -w 65 -c 75 --sensor coretemp-isa-0001
2022-01-10 23:46:26 -06:00
hostgroup: device-type-r720
2022-01-11 00:16:54 -06:00
# Pi 4 4G
- name: CPU Temperature
command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_temp -n -w 65 -c 75 --sensor cpu_thermal-virtual-0
hostgroup: device-type-pi4b-2g,device-type-pi4b-4g,device-type-pi4b-4g-storage
# Device role checks
# hypervisor (which is assumed to be Proxmox)
- name: PVE Unit pve-firewall.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pve-firewall.service
hostgroup: role-hypervisor
- name: PVE Unit spiceproxy.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit spiceproxy.service
hostgroup: role-hypervisor
- name: PVE Unit pve-ha-crm.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pve-ha-crm.service
hostgroup: role-hypervisor
- name: PVE Unit pvedaemon.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pvedaemon.service
hostgroup: role-hypervisor
- name: PVE Unit pvefw-logger.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pvefw-logger.service
hostgroup: role-hypervisor
- name: PVE Unit pveproxy.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pveproxy.service
hostgroup: role-hypervisor
- name: PVE Unit pve-cluster.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pve-cluster.service
hostgroup: role-hypervisor
- name: PVE Unit pvestatd.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit pvestatd.service
hostgroup: role-hypervisor
# Tag-specific checks
# ansible-pull
- name: Unit ansible-pull.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit ansible-pull.service
hostgroup: tag-ansible-pull
- name: Unit ansible-pull.timer
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit ansible-pull.timer
hostgroup: tag-ansible-pull
# docker
2022-02-13 10:11:45 -06:00
- name: Unit docker.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit docker.service
hostgroup: "ansible,!tag-no-docker"
- name: Docker Status
command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_docker --no-ok --status running
hostgroup: tag-nagios-checkdocker
- name: Docker CPU Usage
command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_docker --no-ok --cpu 85:90
hostgroup: tag-nagios-checkdocker
- name: Docker Memory Usage
command: check_by_ssh!/usr/bin/sudo /usr/local/bin/monitoring-scripts/check_docker --no-ok --memory 90:95:%
hostgroup: tag-nagios-checkdocker
# nagios-checkpgsql
- name: PSQL
command: "check_by_ssh!/usr/lib/nagios/plugins/check_pgsql -H localhost -l nagios -p {{ secret_postgresql_monitoring_password }} -w 2 -c 5"
hostgroup: tag-nagios-checkpgsql
- name: PSQL Connections
command: "check_by_ssh!/usr/lib/nagios/plugins/check_pgsql -H localhost -l nagios -p {{ secret_postgresql_monitoring_password }} -w 2 -c 5 -q 'select (select count(*)::float used from pg_stat_activity) / (select setting::int max_conn from pg_settings where name=\\$\\$max_connections\\$\\$)' -W 0.7-0.8 -C 0.8-1.0"
hostgroup: tag-nagios-checkpgsql
# https://rhaas.blogspot.com/2020/02/useless-vacuuming.html
- name: PSQL Old Xacts
command: "check_by_ssh!/usr/lib/nagios/plugins/check_pgsql -H localhost -l nagios -p {{ secret_postgresql_monitoring_password }} -w 2 -c 5 -q 'select count(*)::float from pg_prepared_xacts where age(transaction) > 5000000' -W 500-1000 -C 1000-1000000"
hostgroup: tag-nagios-checkpgsql
- name: Unit postgresql.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit postgresql.service
hostgroup: tag-nagios-checkpgsql
# nagios-checkswap
- name: Swap Usage
command: check_by_ssh!/usr/lib/nagios/plugins/check_swap -w 20% -c 10%
hostgroup: tag-nagios-checkswap
# zerotier
- name: Unit zerotier-one.service
command: check_by_ssh!/usr/local/bin/monitoring-scripts/check_systemd_unit zerotier-one.service
hostgroup: tag-zt-personal
tags: [ nagios, no-auto ]
- role: ingress
vars:
ingress_servers:
2022-04-23 10:54:26 -05:00
- name: ara.desu.ltd
proxy_pass: http://ara:8000
- name: netbox.desu.ltd
proxy_pass: http://netbox:8080
- name: nagios.desu.ltd
proxy_pass: http://nagios:80
tags: [ web, docker, ingress ]