Work on Kopia

This doesn't work. The repo must be initialized beforehand which is going to require a bunch of setup
2024-07-10 13:00:17 -05:00
27 changed files with 266 additions and 317 deletions
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@ -81,6 +81,11 @@ Common:
  stage: play-main
  script:
    - ansible-playbook --skip-tags no-auto playbooks/site_common.yml --ssh-common-args='-o ProxyCommand="ssh -W %h:%p -q ansible@bastion1.dallas.mgmt.desu.ltd"' --vault-password-file ~/.vault_pass
+Nagios:
+  stage: play-main
+  retry: 1
+  script:
+    - ansible-playbook -l vm-general-1.ashburn.mgmt.desu.ltd playbooks/prod_web.yml --tags nagios --ssh-common-args='-o ProxyCommand="ssh -W %h:%p -q ansible@bastion1.dallas.mgmt.desu.ltd"' --vault-password-file ~/.vault_pass

 # CLEANUP
 Cleanup:
--- a/inventories/production/group_vars/all.yml
+++ b/inventories/production/group_vars/all.yml
@ -50,7 +50,29 @@ backup_s3_aws_secret_access_key: !vault |
          3635616437373236650a353661343131303332376161316664333833393833373830623130666633
          66356130646434653039363863346630363931383832353637636131626530616434
 backup_s3_aws_endpoint_url: "https://s3.us-east-005.backblazeb2.com"
-
+backup_kopia_bucket_name: desultd-kopia
+backup_kopia_access_key_id: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          34633366656134376166636164643233353461396263313237653032353764613737393865373763
+          6665633239396333633132323936343030346362333734640a356631373230383663383530333434
+          32386639393135373236373263363365366163346234643135363766666666373938373135653663
+          3836623735393563610a613332623965633032356266643638386230323965366233353930313239
+          38666562326232353165323934303966643630383235393830613939616330333839
+backup_kopia_secret_access_key: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          31373662326464396136346663626635363332303862613466316236333431636136373038666531
+          6630616565613431323464373862373963356335643435360a353665356163313635393137363330
+          66383531326535653066386432646464346161336363373334313064303261616238613564396439
+          6439333432653862370a303461346438623263636364633437356432613831366462666666303633
+          63643862643033376363353836616137366432336339383931363837353161373036
+backup_kopia_password: !vault |
+          $ANSIBLE_VAULT;1.1;AES256
+          34306564393161336162633833356464373065643633343935373566316465373939663838343537
+          3831343963666432323538636665663733353435636337340a633738306463646133643730333032
+          33303962306136636163623930306238666633333738373435636366666339623562323531323732
+          3330633238386336330a346431383233383533303131323736306636353033356538303264383963
+          37306461613834643063383965356664326265383431336332303333636365316163363437343634
+          6439613537396535656361616365386261336139366133393637

 # For zerotier
 zerotier_personal_network_id: !vault |
--- a/playbooks/device_roles_workstation.yml
+++ b/playbooks/device_roles_workstation.yml
@ -21,3 +21,9 @@
 - hosts: dsk-ryzen-1.ws.mgmt.desu.ltd
  roles:
    - role: desktop
+    - role: backup
+      vars:
+        backup_s3backup_tar_args_extra: h
+        backup_s3backup_list_extra:
+          - /home/salt/.backup/
+      tags: [ backup ]
--- a/playbooks/local_dns.yml
+++ b/playbooks/local_dns.yml
@ -109,10 +109,6 @@
        - record: prometheus.desu.ltd
          value: vm-general-1.ashburn.mgmt.desu.ltd
        # Public media stuff
-        - record: music.desu.ltd
-          value: vm-general-1.ashburn.mgmt.desu.ltd
-        - record: lidarr.media.desu.ltd
-          value: vm-general-1.ashburn.mgmt.desu.ltd
        - record: prowlarr.media.desu.ltd
          value: vm-general-1.ashburn.mgmt.desu.ltd
        - record: sonarr.media.desu.ltd
--- a/playbooks/prod_db.yml
+++ b/playbooks/prod_db.yml
@ -8,7 +8,6 @@
      ansible.builtin.docker_container:
        name: prometheus-psql-exporter
        image: quay.io/prometheuscommunity/postgres-exporter
-        restart_policy: unless-stopped
        env:
          DATA_SOURCE_URI: "10.0.0.2:5432/postgres"
          DATA_SOURCE_USER: "nagios"
--- a/playbooks/prod_web.yml
+++ b/playbooks/prod_web.yml
@ -29,8 +29,6 @@
        - web/nextcloud.yml
        - web/synapse.yml
        # Backend web services
-        - web/lidarr.yml
-        - web/navidrome.yml
        - web/prowlarr.yml
        - web/radarr.yml
        - web/sonarr.yml
@ -39,10 +37,8 @@
        # Games
        - game/factorio.yml
        - game/minecraft-createfarming.yml
-        - game/minecraft-magicpack.yml
-        - game/minecraft-weedie.yml
+        - game/minecraft-direwolf20.yml
        - game/zomboid.yml
-        - game/satisfactory.yml
      tags: [ always ]
  roles:
    - role: backup
@ -51,9 +47,7 @@
          - /app/gitea/gitea
          - /data
        backup_s3backup_exclude_list_extra:
-          - /data/minecraft/magicpack/backups
          - /data/minecraft/direwolf20/backups
-          - /data/minecraft/weedie/backups
          - /data/shared/media
          - /data/shared/downloads
          - /data/zomboid/ZomboidDedicatedServer/steamapps/workshop
@ -117,12 +111,12 @@
                pass: http://element:80
            directives:
              - "client_max_body_size 0"
+          - name: nagios.desu.ltd
+            proxy_pass: http://nagios:80
          - name: nc.desu.ltd
            directives:
              - "add_header Strict-Transport-Security \"max-age=31536000\""
              - "client_max_body_size 0"
-              - "keepalive_requests 99999"
-              - "keepalive_timeout 600"
            proxy_pass: http://nextcloud:80
            locations:
              - location: "^~ /.well-known"
--- a/playbooks/tags_autoreboot.yml
+++ b/playbooks/tags_autoreboot.yml
@ -3,11 +3,34 @@
 ---
 - hosts: tags_autoreboot
  gather_facts: no
+  module_defaults:
+    nagios:
+      author: Ansible
+      action: downtime
+      cmdfile: /data/nagios/var/rw/nagios.cmd
+      comment: "Ansible tags_autoreboot task"
+      host: "{{ inventory_hostname }}"
+      minutes: 10
  serial: 1
  tasks:
    - name: check for reboot-required
      ansible.builtin.stat: path=/var/run/reboot-required
      register: s
+    - name: reboot
+      block:
+        - name: attempt to schedule downtime
+          block:
+            - name: register nagios host downtime
+              nagios:
+                service: host
+              delegate_to: vm-general-1.ashburn.mgmt.desu.ltd
+            - name: register nagios service downtime
+              nagios:
+                service: all
+              delegate_to: vm-general-1.ashburn.mgmt.desu.ltd
+          rescue:
+            - name: notify of failure to reboot
+              ansible.builtin.debug: msg="Miscellaneous failure when scheduling downtime"
        - name: reboot
          ansible.builtin.reboot: reboot_timeout=600
      when: s.stat.exists
--- a/playbooks/tags_nagios.yml
+++ b/playbooks/tags_nagios.yml
@ -2,22 +2,44 @@
 # vim:ft=ansible:
 ---
 - hosts: tags_nagios
-  gather_facts: yes
+  gather_facts: no
+  roles:
+    - role: git
+      vars:
+        git_repos:
+          - repo: https://git.desu.ltd/salt/monitoring-scripts
+            dest: /usr/local/bin/monitoring-scripts
+      tags: [ nagios, git ]
  tasks:
+    - name: assure nagios plugin packages
+      ansible.builtin.apt: name=monitoring-plugins,nagios-plugins-contrib
+      tags: [ nagios ]
    - name: assure nagios user
-      ansible.builtin.user: name=nagios-checker state=absent remove=yes
+      ansible.builtin.user: name=nagios-checker state=present system=yes
+      tags: [ nagios ]
+    - name: assure nagios user ssh key
+      authorized_key:
+        user: nagios-checker
+        state: present
+        key: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKNavw28C0mKIQVRLQDW2aoovliU1XCGaenDhIMwumK/ Nagios monitoring"
      tags: [ nagios ]
    - name: assure nagios user sudo rule file
-      ansible.builtin.file: path=/etc/sudoers.d/50-nagios-checker state=absent
+      ansible.builtin.file: path=/etc/sudoers.d/50-nagios-checker mode=0750 owner=root group=root state=touch modification_time=preserve access_time=preserve
+      tags: [ nagios, sudo ]
+    - name: assure nagios user sudo rules
+      ansible.builtin.lineinfile:
+        path: /etc/sudoers.d/50-nagios-checker
+        line: "nagios-checker ALL = (root) NOPASSWD: {{ item }}"
+      with_items:
+        - /usr/lib/nagios/plugins/check_disk
+        - /usr/local/bin/monitoring-scripts/check_docker
+        - /usr/local/bin/monitoring-scripts/check_temp
      tags: [ nagios, sudo ]
-    - name: assure prometheus containers for docker hosts
-      block:
    - name: assure prometheus node exporter
      # https://github.com/prometheus/node_exporter
      ansible.builtin.docker_container:
        name: prometheus-node-exporter
        image: quay.io/prometheus/node-exporter:latest
-            restart_policy: unless-stopped
        command:
          - '--path.rootfs=/host'
          - '--collector.interrupts'
@ -31,7 +53,6 @@
      ansible.builtin.docker_container:
        name: prometheus-cadvisor-exporter
        image: gcr.io/cadvisor/cadvisor:latest
-            restart_policy: unless-stopped
        ports:
          - 9101:8080/tcp
        volumes:
@ -42,25 +63,6 @@
          - /dev/disk:/dev/disk:ro
        devices:
          - /dev/kmsg
-      when: ansible_pkg_mgr != "atomic_container"
-    - name: assure prometheus containers for coreos
-      block:
-        - name: assure prometheus node exporter
-          # https://github.com/prometheus/node_exporter
-          containers.podman.podman_container:
-            name: prometheus-node-exporter
-            image: quay.io/prometheus/node-exporter:latest
-            restart_policy: unless-stopped
-            command:
-              - '--path.rootfs=/host'
-              - '--collector.interrupts'
-              - '--collector.processes'
-            network_mode: host
-            pid_mode: host
-            volumes:
-              - /:/host:ro,rslave
-          tags: [ prometheus ]
-      when: ansible_pkg_mgr == "atomic_container"
 - hosts: all
  gather_facts: no
  tasks:
--- a/playbooks/tasks/game/minecraft-createfarming.yml
+++ b/playbooks/tasks/game/minecraft-createfarming.yml
@ -2,28 +2,16 @@
 - name: docker deploy minecraft - create farming and delights
  docker_container:
    name: minecraft-createfarming
-    state: absent
+    state: started
    image: itzg/minecraft-server:latest
+    restart_policy: unless-stopped
+    pull: yes
    env:
-      # Common envvars
      EULA: "true"
-      OPS: "VintageSalt"
-      SNOOPER_ENABLED: "false"
-      SPAWN_PROTECTION: "0"
-      USE_AIKAR_FLAGS: "true"
-      RCON_CMDS_STARTUP: |-
-        scoreboard objectives add Deaths deathCount
-        #scoreboard objectives add Health health {"text":"❤","color":"red"}
-      RCON_CMDS_ON_CONNECT: |-
-        scoreboard objectives setdisplay list Deaths
-        #scoreboard objectives setdisplay belowName Health
-      # Pack-specific stuff
      MODRINTH_PROJECT: "https://modrinth.com/modpack/create-farmersdelight/version/1.0.0"
-      MOTD: "Create Farming and Delights! Spinny trains!"
      TYPE: "MODRINTH"
      VERSION: "1.20.1"
      MAX_MEMORY: "6G"
-      #VIEW_DISTANCE: "10"
    ports:
      - "25565:25565/tcp"
      - "25565:25565/udp"
--- a/playbooks/tasks/game/minecraft-direwolf20.yml
+++ b/playbooks/tasks/game/minecraft-direwolf20.yml
@ -0,0 +1,34 @@
+# vim:ft=ansible:
+- name: docker deploy minecraft - direwolf20
+  docker_container:
+    name: minecraft-direwolf20
+    state: absent
+    image: itzg/minecraft-server:latest
+    restart_policy: unless-stopped
+    pull: yes
+    env:
+      EULA: "true"
+      GENERIC_PACK: "/modpacks/1.20.1-direwolf20/Da Bois.zip"
+      TYPE: "NEOFORGE"
+      VERSION: "1.20.1"
+      FORGE_VERSION: "47.1.105"
+      MEMORY: "8G"
+      MOTD: "Tannerite Dog Edition\\n#abolishtheatf"
+      OPS: "VintageSalt"
+      RCON_CMDS_STARTUP: |-
+        scoreboard objectives add Deaths deathCount
+        scoreboard objectives add Health health {"text":"❤","color":"red"}
+      RCON_CMDS_ON_CONNECT: |-
+        scoreboard objectives setdisplay list Deaths
+        scoreboard objectives setdisplay belowName Health
+      SNOOPER_ENABLED: "false"
+      SPAWN_PROTECTION: "0"
+      USE_AIKAR_FLAGS: "true"
+      VIEW_DISTANCE: "10"
+    ports:
+      - "25567:25565/tcp"
+      - "25567:25565/udp"
+    volumes:
+      - /data/srv/packs:/modpacks
+      - /data/minecraft/direwolf20:/data
+  tags: [ docker, minecraft, direwolf20 ]
--- a/playbooks/tasks/game/minecraft-magicpack.yml
+++ b/playbooks/tasks/game/minecraft-magicpack.yml
@ -1,50 +0,0 @@
-# vim:ft=ansible:
- name: docker deploy minecraft - magicpack
-  docker_container:
-    name: minecraft-magicpack
-    state: absent
-    image: itzg/minecraft-server:java8
-    env:
-      # Common envvars
-      EULA: "true"
-      OPS: "VintageSalt"
-      SNOOPER_ENABLED: "false"
-      SPAWN_PROTECTION: "0"
-      USE_AIKAR_FLAGS: "true"
-      #
-      # This enables the use of Ely.by as an auth and skin server
-      # Comment this and the above line out if you'd like to use Mojang's
-      # https://docs.ely.by/en/authlib-injector.html
-      #
-      # All players should register on Ely.by in order for this to work.
-      # They should also use Fjord Launcher by Unmojang:
-      # https://github.com/unmojang/FjordLauncher
-      #
-      JVM_OPTS: "-javaagent:/authlib-injector.jar=ely.by"
-      RCON_CMDS_STARTUP: |-
-        scoreboard objectives add Deaths deathCount
-        #scoreboard objectives add Health health {"text":"❤","color":"red"}
-      RCON_CMDS_ON_CONNECT: |-
-        scoreboard objectives setdisplay list Deaths
-        #scoreboard objectives setdisplay belowName Health
-      # Pack-specific stuff
-      MODRINTH_PROJECT: "https://srv.9iron.club/files/packs/1.7.10-magicpack/server.mrpack"
-      MOTD: "It's ya boy, uh, skrunkly modpack"
-      TYPE: "MODRINTH"
-      VERSION: "1.7.10"
-      MAX_MEMORY: "6G"
-      #VIEW_DISTANCE: "10"
-    ports:
-      - "25565:25565/tcp"
-      - "25565:25565/udp"
-      - "24454:24454/udp"
-      # Prometheus exporter for Forge
-      # https://www.curseforge.com/minecraft/mc-mods/prometheus-exporter
-      - "19565:19565/tcp"
-      # Prometheus exporter for Fabric
-      # https://modrinth.com/mod/fabricexporter
-      #- "19565:25585/tcp"
-    volumes:
-      - /data/minecraft/magicpack:/data
-      - /data/minecraft/authlib-injector-1.2.5.jar:/authlib-injector.jar
-  tags: [ docker, minecraft, magicpack ]
--- a/playbooks/tasks/game/minecraft-vanilla.yml
+++ b/playbooks/tasks/game/minecraft-vanilla.yml
@ -0,0 +1,33 @@
+# vim:ft=ansible:
+- name: docker deploy minecraft - vanilla
+  docker_container:
+    name: minecraft-vanilla
+    state: absent
+    image: itzg/minecraft-server:latest
+    restart_policy: unless-stopped
+    pull: yes
+    env:
+      DIFFICULTY: "normal"
+      ENABLE_COMMAND_BLOCK: "true"
+      EULA: "true"
+      MAX_PLAYERS: "8"
+      MODRINTH_PROJECT: "https://modrinth.com/modpack/adrenaserver"
+      MOTD: "Tannerite Dog Edition\\n#abolishtheatf"
+      OPS: "VintageSalt"
+      RCON_CMDS_STARTUP: |-
+        scoreboard objectives add Deaths deathCount
+        scoreboard objectives add Health health {"text":"❤","color":"red"}
+      RCON_CMDS_ON_CONNECT: |-
+        scoreboard objectives setdisplay list Deaths
+        scoreboard objectives setdisplay belowName Health
+      SNOOPER_ENABLED: "false"
+      SPAWN_PROTECTION: "0"
+      TYPE: "MODRINTH"
+      USE_AIKAR_FLAGS: "true"
+      VIEW_DISTANCE: "12"
+    ports:
+      - "26565:25565/tcp"
+      - "26565:25565/udp"
+    volumes:
+      - /data/minecraft/vanilla:/data
+  tags: [ docker, minecraft ]
--- a/playbooks/tasks/game/minecraft-weedie.yml
+++ b/playbooks/tasks/game/minecraft-weedie.yml
@ -1,44 +0,0 @@
-# vim:ft=ansible:
- name: docker deploy minecraft - weediewack next gen pack
-  docker_container:
-    name: minecraft-weedie
-    state: started
-    image: itzg/minecraft-server:latest
-    env:
-      # Common envvars
-      EULA: "true"
-      OPS: "VintageSalt"
-      SNOOPER_ENABLED: "false"
-      SPAWN_PROTECTION: "0"
-      USE_AIKAR_FLAGS: "true"
-      ALLOW_FLIGHT: "true"
-      RCON_CMDS_STARTUP: |-
-        scoreboard objectives add Deaths deathCount
-        scoreboard objectives add Health health {"text":"❤","color":"red"}
-      RCON_CMDS_ON_CONNECT: |-
-        scoreboard objectives setdisplay list Deaths
-        scoreboard objectives setdisplay belowName Health
-      # Pack-specific stuff
-      TYPE: "Forge"
-      MOTD: "We're doing it a-fucking-gain!"
-      VERSION: "1.20.1"
-      FORGE_VERSION: "47.3.11"
-      MAX_MEMORY: "8G"
-      #GENERIC_PACKS: "Server Files 1.3.7"
-      #GENERIC_PACKS_PREFIX: "https://mediafilez.forgecdn.net/files/5832/451/"
-      #GENERIC_PACKS_SUFFIX: ".zip"
-      #SKIP_GENERIC_PACK_UPDATE_CHECK: "true"
-      #VIEW_DISTANCE: "10"
-    ports:
-      - "25565:25565/tcp"
-      - "25565:25565/udp"
-      - "24454:24454/udp"
-      # Prometheus exporter for Forge
-      # https://www.curseforge.com/minecraft/mc-mods/prometheus-exporter
-      - "19566:19565/tcp"
-      # Prometheus exporter for Fabric
-      # https://modrinth.com/mod/fabricexporter
-      #- "19565:25585/tcp"
-    volumes:
-      - /data/minecraft/weedie:/data
-  tags: [ docker, minecraft, weedie ]
--- a/playbooks/tasks/game/satisfactory.yml
+++ b/playbooks/tasks/game/satisfactory.yml
@ -1,47 +0,0 @@
-# vim:ft=ansible:
- name: ensure docker network
-  docker_network: name=satisfactory
-  tags: [ satisfactory, docker, network ]
- name: docker deploy satisfactory
-  docker_container:
-    name: satisfactory
-    state: absent
-    image: wolveix/satisfactory-server:latest
-    restart_policy: unless-stopped
-    pull: yes
-    networks:
-      - name: satisfactory
-        aliases: [ "gameserver" ]
-    env:
-      MAXPLAYERS: "8"
-      # We have this turned on for modding's sake
-      #SKIPUPDATE: "true"
-    ports:
-      - '7777:7777/udp'
-      - '7777:7777/tcp'
-    volumes:
-      - /data/satisfactory/config:/config
-  tags: [ docker, satisfactory ]
- name: docker deploy satisfactory sftp
-  docker_container:
-    name: satisfactory-sftp
-    state: started
-    image: atmoz/sftp
-    restart_policy: unless-stopped
-    pull: yes
-    ulimits:
-      - 'nofile:262144:262144'
-    ports:
-      - '7776:22/tcp'
-    volumes:
-      - /data/satisfactory/config:/home/servermgr/game
-    command: 'servermgr:{{ server_password }}:1000'
-  vars:
-    server_password: !vault |
-          $ANSIBLE_VAULT;1.1;AES256
-          33336138656461646462323661363336623235333861663730373535656331623230313334353239
-          6535623833343237626161383833663435643262376133320a616634613764396661316332373339
-          33633662366666623931643635313162366339306539666632643437396637616632633432326631
-          3038333932623638390a386362653463306338326436396230633562313466336464663764643461
-          3134
-  tags: [ docker, satisfactory, sidecar, sftp ]
--- a/playbooks/tasks/web/grafana.yml
+++ b/playbooks/tasks/web/grafana.yml
@ -31,7 +31,7 @@
 - name: docker deploy grafana matrix bridge
  docker_container:
    name: grafana-matrix-bridge
-    image: registry.gitlab.com/hctrdev/grafana-matrix-forwarder:latest
+    image: registry.gitlab.com/hectorjsmith/grafana-matrix-forwarder:latest
    env:
      GMF_MATRIX_USER: "@grafana:desu.ltd"
      GMF_MATRIX_PASSWORD: "{{ secret_grafana_matrix_token }}"
--- a/playbooks/tasks/web/lidarr.yml
+++ b/playbooks/tasks/web/lidarr.yml
@ -2,7 +2,6 @@
 - name: docker deploy lidarr
  docker_container:
    name: lidarr
-    state: absent
    image: linuxserver/lidarr:latest
    networks:
      - name: web
@ -10,10 +9,7 @@
    env:
      TZ: "America/Chicago"
    volumes:
-      # https://github.com/RandomNinjaAtk/arr-scripts?tab=readme-ov-file
      - /data/lidarr/config:/config
-      - /data/lidarr/custom-services.d:/custom-services.d
-      - /data/lidarr/custom-cont-init.d:/custom-cont-init.d
      - /data/shared/downloads:/data
      - /data/shared/media/music:/music
  tags: [ docker, lidarr ]
--- a/playbooks/tasks/web/navidrome.yml
+++ b/playbooks/tasks/web/navidrome.yml
@ -1,18 +0,0 @@
-# vim:ft=ansible:
- name: docker deploy navidrome
-  docker_container:
-    name: navidrome
-    state: absent
-    image: deluan/navidrome:latest
-    user: 911:911
-    env:
-      ND_BASEURL: "https://music.desu.ltd"
-      ND_PROMETHEUS_ENABLED: "true"
-      ND_LOGLEVEL: "info"
-    networks:
-      - name: web
-        aliases: [ "navidrome" ]
-    volumes:
-      - /data/navidrome/data:/data
-      - /data/shared/media/music:/music:ro
-  tags: [ docker, navidrome ]
--- a/playbooks/tasks/web/synapse.yml
+++ b/playbooks/tasks/web/synapse.yml
@ -2,7 +2,7 @@
 - name: docker deploy synapse
  docker_container:
    name: synapse
-    image: matrixdotorg/synapse:latest
+    image: ghcr.io/element-hq/synapse:latest
    env:
      TZ: "America/Chicago"
      SYNAPSE_SERVER_NAME: matrix.desu.ltd
--- a/roles/backup/tasks/main.yml
+++ b/roles/backup/tasks/main.yml
@ -14,3 +14,50 @@
  notify: restart backup timer
 - name: enable timer
  ansible.builtin.systemd: name=backup.timer state=started enabled=yes daemon_reload=yes
+- name: deploy kopia
+  block:
+    - name: ensure kopia dirs
+      ansible.builtin.file:
+        state: directory
+        owner: root
+        group: root
+        mode: "0750"
+        path: "{{ item }}"
+      with_items:
+        - /data/kopia/config
+        - /data/kopia/cache
+        - /data/kopia/logs
+    - name: template out password file
+      copy:
+        content: "{{ backup_kopia_password }}"
+        owner: root
+        group: root
+        mode: "0600"
+        dest: /data/kopia/config/repository.config.kopia-password
+    - name: template out configuration file
+      template:
+        src: repository.config.j2
+        owner: root
+        group: root
+        mode: "0600"
+        dest: /data/kopia/config/repository.config
+    - name: deploy kopia
+      community.docker.docker_container:
+        name: kopia
+        image: kopia/kopia:latest
+        env:
+          KOPIA_PASSWORD: "{{ backup_kopia_password }}"
+        command:
+          - "repository"
+          - "connect"
+          - "from-config"
+          - "--file"
+          - "/app/config/repository.config"
+        volumes:
+          - /data/kopia/config:/app/config
+          - /data/kopia/cache:/app/cache
+          - /data/kopia/logs:/app/logs
+          # Shared tmp so Kopia can dump restorable backups to the host
+          - /tmp:/tmp:shared
+          # And a RO mount for the host so it can be backed up
+          - /:/host:ro,rslave
--- a/roles/backup/templates/backup.service
+++ b/roles/backup/templates/backup.service
@ -3,15 +3,11 @@
 Description=Nightly backup service
 After=network-online.target
 Wants=network-online.target
-StartLimitInterval=600
-StartLimitBurst=5

 [Service]
 Type=oneshot
 MemoryMax=256M
 ExecStart=/opt/backup.sh
-Restart=on-failure
-RestartSec=5

 [Install]
 WantedBy=multi-user.target
--- a/roles/backup/templates/repository.config.j2
+++ b/roles/backup/templates/repository.config.j2
@ -0,0 +1,21 @@
+{
+  "storage": {
+    "type": "b2",
+    "config": {
+      "bucket": "desultd-kopia",
+      "keyID": "{{ backup_kopia_access_key_id }}",
+      "key": "{{ backup_kopia_secret_access_key }}"
+    }
+  },
+  "caching": {
+    "cacheDirectory": "/app/cache/cachedir",
+    "maxCacheSize": 5242880000,
+    "maxMetadataCacheSize": 5242880000,
+    "maxListCacheDuration": 30
+  },
+  "hostname": "{{ inventory_hostname }}",
+  "username": "salt",
+  "description": "Desu LTD Backups",
+  "enableActions": false,
+  "formatBlobCacheDuration": 900000000000
+}
--- a/roles/common/tasks/packages.yml
+++ b/roles/common/tasks/packages.yml
@ -44,8 +44,10 @@
 - name: configure rpm-ostree packages
  community.general.rpm_ostree_pkg:
    name:
+      - awscli
      - htop
      - ibm-plex-fonts-all
      - ncdu
+      - screen
      - vim
  when: ansible_os_family == "RedHat" and ansible_pkg_mgr == "atomic_container"
--- a/roles/desktop/defaults/main.yml
+++ b/roles/desktop/defaults/main.yml
@ -153,31 +153,17 @@ desktop_flatpak_remotes:
    url: "https://dl.flathub.org/repo/flathub.flatpakrepo"
  - name: flathub-beta
    url: "https://flathub.org/beta-repo/flathub-beta.flatpakrepo"
-#  - name: unmojang
-#    url: "https://unmojang.github.io/unmojang-flatpak/index.flatpakrepo"
 desktop_flatpak_remotes_extra: []

 desktop_flatpak_packages:
  - remote: flathub
    packages:
-      - com.github.KRTirtho.Spotube
-      - com.github.Matoking.protontricks
-      - com.github.tchx84.Flatseal
-      - com.nextcloud.desktopclient.nextcloud
-      - com.valvesoftware.Steam
-      - dev.vencord.Vesktop
-      - im.riot.Riot
-      - io.freetubeapp.FreeTube
-      - io.kopia.KopiaUI
-      - io.mpv.Mpv
+      - com.discordapp.Discord
+      - com.obsproject.Studio
      - net.minetest.Minetest
      - org.DolphinEmu.dolphin-emu
-      - org.gnucash.GnuCash
      - org.mozilla.firefox
-      - org.mozilla.Thunderbird
-      - org.openscad.OpenSCAD
-      - org.qbittorrent.qBittorrent
-#  - remote: unmojang
-#    packages:
-#      - org.unmojang.FjordLauncher
+  - remote: flathub-beta
+    packages:
+      - net.lutris.Lutris
 desktop_flatpak_packages_extra: []
--- a/roles/desktop/tasks/packages.yml
+++ b/roles/desktop/tasks/packages.yml
@ -29,7 +29,7 @@
  when: ansible_pkg_mgr == "apt"
 - name: configure pip3 packages
  ansible.builtin.pip: executable=/usr/bin/pip3 state=latest name="{{ desktop_pip3_packages + desktop_pip3_packages_extra }}"
-  when: ansible_pkg_mgr == "apt"
+  when: ansible_os_family != "Gentoo"
 - name: configure flatpak
  block:
    - name: configure flatpak remotes
--- a/roles/nagios/tasks/main.yml
+++ b/roles/nagios/tasks/main.yml
@ -9,7 +9,7 @@
    image: manios/nagios:latest
    pull: yes
    restart_policy: unless-stopped
-    state: absent
+    state: started
    env:
      NAGIOSADMIN_USER: admin
      NAGIOSADMIN_PASS: "{{ nagios_admin_pass }}"
--- a/roles/prometheus/tasks/main.yml
+++ b/roles/prometheus/tasks/main.yml
@ -24,7 +24,6 @@
      community.docker.docker_container:
        name: prometheus
        image: prom/prometheus:latest
-        restart_policy: unless-stopped
        user: 5476:5476
        env:
          TZ: "America/Chicago"
@ -56,7 +55,6 @@
      community.docker.docker_container:
        name: prometheus-blackbox
        image: quay.io/prometheus/blackbox-exporter:latest
-        restart_policy: unless-stopped
        user: 5476:5476
        command:
          - '--config.file=/config/blackbox.yml'
--- a/roles/prometheus/templates/prometheus.yml.j2
+++ b/roles/prometheus/templates/prometheus.yml.j2
@ -83,46 +83,6 @@ scrape_configs:
 {% endfor %}
 {% endfor %}

-    # This job takes in information from Netbox on the generic "prom-metrics" tag
-    # It's useful for all sorts of stuff
-  - job_name: "generic"
-    scheme: "https"
-    static_configs:
-      - targets:
-{% for host in groups['tags_nagios'] %}
-{% set vars = hostvars[host] %}
-{% for service in vars.services %}
-{% for tag in service.tags %}
-{# #}
-{% if tag.slug == "prom-metrics" %}
-{% for port in service.ports %}
-        - "{{ service.name }}:{{ port }}"
-{% endfor %}
-{% endif %}
-{# #}
-{% endfor %}
-{% endfor %}
-{% endfor %}
-
-    # This one does the same thing but for HTTP-only clients
-  - job_name: "generic-http"
-    scheme: "http"
-    static_configs:
-      - targets:
-{% for host in groups['tags_nagios'] %}
-{% set vars = hostvars[host] %}
-{% for service in vars.services %}
-{% for tag in service.tags %}
-{# #}
-{% if tag.slug == "prom-metrics-http" %}
-{% for port in service.ports %}
-        - "{{ service.name }}:{{ port }}"
-{% endfor %}
-{% endif %}
-{# #}
-{% endfor %}
-{% endfor %}
-{% endfor %}
    # These two jobs are included for every node in our inventory
  - job_name: "node-exporter"
    static_configs: