Back up Pi k8s nodes in some rudimentary way

We forkin

playbooks/nrpe.yml

@@ -3,6 +3,12 @@
 ---
 - hosts: tags_nagios-nrpe
   roles:
+    - role: git
+      vars:
+        git_repos:
+          - repo: https://git.desu.ltd/salt/monitoring-scripts
+            dest: /usr/local/bin/monitoring-scripts
+      tags: [ nrpe ]
     - role: nrpe
       vars:
         nrpe_server_bind_address: 0.0.0.0
@@ -28,6 +34,22 @@
           check_users:
             script: check_users
             option: -w 3 -c 5
+        nrpe_command_extra:
+          check_systemd_ansiblepull_service:
+            script: /usr/local/bin/monitoring-scripts/check_systemd_unit
+            option: ansible-pull.service
+          check_systemd_ansiblepull_timer:
+            script: /usr/local/bin/monitoring-scripts/check_systemd_unit
+            option: ansible-pull.timer
+          check_systemd_backup_service:
+            script: /usr/local/bin/monitoring-scripts/check_systemd_unit
+            option: backup.service
+          check_systemd_backup_timer:
+            script: /usr/local/bin/monitoring-scripts/check_systemd_unit
+            option: backup.timer
+          check_systemd_docker_service:
+            script: /usr/local/bin/monitoring-scripts/check_systemd_unit
+            option: docker.service
       tags: [ nrpe ]
 - hosts: all
   tasks:
@@ -36,4 +58,4 @@
       with_items:
         - nagios-nrpe-server.service
       when: "'tags_nagios-nrpe' not in group_names and item in services"
-      tags: [ zerotier ]
+      tags: [ nrpe, zerotier ]

playbooks/pik8s.yml

@@ -7,6 +7,11 @@
   roles:
     - role: k8s
       tags: [ k8s, skip-pull ]
+    - role: backup
+      vars:
+        backup_s3backup_list_extra:
+          - /etc/kubernetes
+      tags: [ k8s, backup ]
 - hosts: tags_pik8s-master
   gather_facts: no
   tasks:

playbooks/tasks/web/nagios-hosts.cfg.j2

@@ -108,6 +108,38 @@ define service {
 	check_command		check_nrpe!check_load
 	hostgroup_name		nagios-nrpe
 }
+# Systemd unit checks
+define service {
+	use			ansible-generic-service
+	service_description	NRPE Check Unit backup.service
+	check_command		check_nrpe!check_systemd_backup_service
+	hostgroup_name		nagios-nrpe
+}
+define service {
+	use			ansible-generic-service
+	service_description	NRPE Check Unit backup.timer
+	check_command		check_nrpe!check_systemd_backup_timer
+	hostgroup_name		nagios-nrpe
+}
+define service {
+	use			ansible-generic-service
+	service_description	NRPE Check Unit ansible-pull.service
+	check_command		check_nrpe!check_systemd_ansiblepull_service
+	hostgroup_name		ansible-pull
+}
+define service {
+	use			ansible-generic-service
+	service_description	NRPE Check Unit ansible-pull.timer
+	check_command		check_nrpe!check_systemd_ansiblepull_timer
+	hostgroup_name		ansible-pull
+}
+define service {
+	use			ansible-generic-service
+	service_description	NRPE Check Unit docker.service
+	check_command		check_nrpe!check_systemd_docker_service
+	hostgroup_name		nagios-nrpe
+}
+# Swap
 define service {
 	use			ansible-generic-service
 	service_description	NRPE Check Swap

requirements.yml

@@ -57,7 +57,3 @@ roles:
   - name: snmpd
     src: oefenweb.snmpd
     version: master
-  # Upstream: https://github.com/hspaans/ansible-role-nrpe
-  - name: nrpe
-    src: hspaans.nrpe
-    version: master

roles/nrpe/.editorconfig

@@ -1,46 +0,0 @@
-# https://editorconfig.org/
-
-root = true
-
-[*]
-indent_style = space
-indent_size = 4
-trim_trailing_whitespace = true
-end_of_line = lf
-charset = utf-8
-
-# Docstrings and comments use max_line_length = 79
-[*.py]
-insert_final_newline = true
-max_line_length = 119
-
-# Use 2 spaces for the HTML files
-[*.html]
-indent_size = 2
-
-# Use 2 spaces for the Markdown files
-[*.md]
-indent_size = 2
-insert_final_newline = true
-
-# The JSON files contain newlines inconsistently
-[*.json]
-indent_size = 2
-
-# Makefiles always use tabs for indentation
-[Makefile]
-indent_style = tab
-insert_final_newline = true
-
-# Batch files use tabs for indentation
-[*.bat]
-indent_style = tab
-insert_final_newline = true
-
-[docs/**.txt]
-insert_final_newline = true
-max_line_length = 79
-
-[*.yml]
-indent_size = 2
-insert_final_newline = true

roles/nrpe/.flake8

@@ -1,2 +0,0 @@
-[flake8]
-max-line-length = 120

roles/nrpe/.github/dependabot.yml

@@ -1,8 +0,0 @@
----
-version: 2
-updates:
-  - package-ecosystem: github-actions
-    directory: /
-    schedule:
-      interval: monthly
-    open-pull-requests-limit: 10

roles/nrpe/.github/settings.yml

@@ -1,6 +0,0 @@
----
-repository:
-  description: Ansible Role for Nagios Remote Plugin Executor
-  topics: ansible, ansible-role, nagios, nrpe, agent, monitoring
-  delete_branch_on_merge: true
-  has_wiki: false

roles/nrpe/.github/stale.yml

@@ -1,2 +0,0 @@
----
-_extends: .github:.github/stale.yml

roles/nrpe/.github/workflows/ansible-ci.yml

@@ -1,46 +0,0 @@
----
-name: CI
-
-on:
-  pull_request:
-  schedule:
-    - cron: '30 5 15 * *'
-
-jobs:
-
-  lint:
-    name: Lint Code Base
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Checkout Code
-        uses: actions/checkout@v2.3.4
-
-      - name: Lint Code Base
-        uses: github/super-linter@v4.6.0
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  molecule:
-    needs: lint
-    name: Molecule
-    runs-on: ubuntu-latest
-
-    steps:
-      - name: Check out the codebase
-        uses: actions/checkout@v2.3.4
-
-      - name: Set up Python 3
-        uses: actions/setup-python@v2.2.2
-        with:
-          python-version: "3.x"
-
-      - name: Install Tox
-        run: |
-          pip3 install tox
-
-      - name: Run Molecule tests
-        run: tox
-        env:
-          PY_COLORS: "1"
-          ANSIBLE_FORCE_COLOR: "1"

roles/nrpe/.github/workflows/ansible-release.yml

@@ -1,21 +0,0 @@
----
-name: Release to Ansible Galaxy
-
-on:
-  push:
-    tags:
-      - v*
-    branches:
-      - master
-
-jobs:
-  release:
-    name: Release to Ansible Galaxy
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Release Ansible Role to Galaxy
-        uses: hspaans/ansible-galaxy-action@v0.3.3
-        with:
-          api_key: ${{ secrets.galaxy_api_key }}

roles/nrpe/.github/workflows/codeql-analysis.yml

@@ -1,59 +0,0 @@
----
-name: "CodeQL"
-
-on:
-  pull_request:
-    branches:
-      - master
-  schedule:
-    - cron: "35 22 * * 4"
-
-jobs:
-  analyze:
-    name: Analyze
-    runs-on: ubuntu-latest
-    permissions:
-      actions: read
-      contents: read
-      security-events: write
-
-    strategy:
-      fail-fast: false
-      matrix:
-        language: ["python"]
-        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
-        # Learn more:
-        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v2
-
-      # Initializes the CodeQL tools for scanning.
-      - name: Initialize CodeQL
-        uses: github/codeql-action/init@v1
-        with:
-          languages: ${{ matrix.language }}
-          # If you wish to specify custom queries, you can do so here or in a config file.
-          # By default, queries listed here will override any specified in a config file.
-          # Prefix the list here with "+" to use these queries and those in the config file.
-          # queries: ./path/to/local/query, your-org/your-repo/queries@main
-
-      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
-      # If this step fails, then you should remove it and run the build manually (see below)
-      - name: Autobuild
-        uses: github/codeql-action/autobuild@v1
-
-      # ℹ️ Command-line programs to run using the OS shell.
-      # 📚 https://git.io/JvXDl
-
-      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
-      #    and modify them (or add more) to build your code if your project
-      #    uses a compiled language
-
-      #- run: |
-      #   make bootstrap
-      #   make release
-
-      - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@v1

roles/nrpe/.gitignore

@@ -1,15 +0,0 @@
-## Ansible
-*.retry
-.ansible_cache
-
-## Molecule
-.cache
-.molecule
-.tox
-.vagrant
-
-## Python
-.pytest_cache/
-__pycache__/
-*.py[cod]
-*$py.class

roles/nrpe/.yamllint.yml

@@ -1,12 +0,0 @@
----
-extends: default
-
-rules:
-  braces:
-    max-spaces-inside: 1
-    level: error
-  brackets:
-    max-spaces-inside: 1
-    level: error
-  line-length: disable
-  truthy: disable

roles/nrpe/README.md

@@ -1,51 +0,0 @@
-# Role Name
-
-Install and configure NRPE
-
-## Requirements
-
-None as the nrpe package is being installed.
-
-## Role Variables
-
-Default variables are set in `defaults/main.yml`.
-
-## Dependencies
-
-No dependency on other Ansible Galaxy roles.
-
-## Example Playbook
-
-```yaml
----
-- hosts: servers
-  vars:
-    nrpe_server_allowed_hosts:
-      - 10.0.10.0/24
-      - 10.0.11.2
-      - 127.0.0.1
-    nrpe_plugin_packages:
-      - nagios-plugins-disk
-      - nagios-plugins-nagios
-      - nagios-plugins-users
-    nrpe_command:
-      check_disk_all:
-        script: check_disk
-        option: -w 80 -c 90
-      check_users:
-        script: check_users2
-        option: -w 1 -c 1
-      check_nagios:
-        script: check_nagios
-        option: -F /var/log/nagios/nagios.log -e 15 -C nagios
-  roles:
-    - { role: hspaans.nrpe, become: true }
-```
-
-## License
-
-MIT
-
-## Author Information
-
-This role was created in 2020 by [Hans Spaans](https://github.com/hspaans).

roles/nrpe/molecule/os-family-debian/INSTALL.rst

@@ -1,22 +0,0 @@
-*******
-Docker driver installation guide
-*******
-
-Requirements
-============
-
-* Docker Engine
-
-Install
-=======
-
-Please refer to the `Virtual environment`_ documentation for installation best
-practices. If not using a virtual environment, please consider passing the
-widely recommended `'--user' flag`_ when invoking ``pip``.
-
-.. _Virtual environment: https://virtualenv.pypa.io/en/latest/
-.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site
-
-.. code-block:: bash
-
-    $ pip install 'molecule[docker]'

roles/nrpe/molecule/os-family-debian/converge.yml

@@ -1,5 +0,0 @@
----
-- name: Converge
-  hosts: all
-  roles:
-    - role: hspaans.nrpe

roles/nrpe/molecule/os-family-debian/molecule.yml

@@ -1,47 +0,0 @@
----
-dependency:
-  name: galaxy
-driver:
-  name: docker
-lint: |
-  set -e
-  yamllint `git ls-files '*.yaml' '*.yml'`
-  ansible-lint
-  # flake8
-platforms:
-  - name: debian-10
-    image: "ghcr.io/hspaans/molecule-container-debian:10"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-  - name: debian-11
-    image: "ghcr.io/hspaans/molecule-container-debian:11"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-  - name: ubuntu-18.04
-    image: "ghcr.io/hspaans/molecule-container-ubuntu:18.04"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-  - name: ubuntu-20.04
-    image: "ghcr.io/hspaans/molecule-container-ubuntu:20.04"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-provisioner:
-  name: ansible
-  inventory:
-    group_vars:
-      all:
-        nrpe_plugin_packages: []
-verifier:
-  name: testinfra

roles/nrpe/molecule/os-family-debian/prepare.yml

@@ -1,27 +0,0 @@
----
-- name: Prepare
-  hosts: all
-
-  pre_tasks:
-    - name: Update apt cache (on Debian).
-      apt:
-        update_cache: true
-        cache_valid_time: 3600
-      when: ansible_os_family == 'Debian'
-
-    - name: Remove packages
-      package:
-        name: apt-utils
-        state: present
-      when: ansible_os_family == 'Debian'
-
-    - name: Update yum cache (on Redhat).
-      yum:
-        update_cache: true
-      when: ansible_os_family == 'RedHat'
-
-  tasks:
-    - name: Remove packages
-      package:
-        name: git
-        state: absent

roles/nrpe/molecule/os-family-debian/tests/conftest.py

@@ -1,23 +0,0 @@
-"""PyTest Fixtures."""
-from __future__ import absolute_import
-
-import os
-
-import pytest
-
-
-def pytest_runtest_setup():
-    """Run tests only when under molecule with testinfra installed."""
-    try:
-        # pylint: disable = import-outside-toplevel
-        import testinfra
-    except ImportError:
-        pytest.skip("Test requires testinfra", allow_module_level=True)
-    if "MOLECULE_INVENTORY_FILE" in os.environ:
-        pytest.testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-            os.environ["MOLECULE_INVENTORY_FILE"]
-        ).get_hosts("all")
-    else:
-        pytest.skip(
-            "Test should run only from inside molecule.", allow_module_level=True
-        )

roles/nrpe/molecule/os-family-debian/tests/test_default.py

@@ -1,11 +0,0 @@
-"""Role testing files using testinfra."""
-
-import pytest
-
-
-@pytest.mark.parametrize("pkg", ["nagios-nrpe-server"])
-def test_pkg_installed(host, pkg):
-    """Test if package installed."""
-    package = host.package(pkg)
-
-    assert package.is_installed

roles/nrpe/molecule/os-family-redhat/INSTALL.rst

@@ -1,22 +0,0 @@
-*******
-Docker driver installation guide
-*******
-
-Requirements
-============
-
-* Docker Engine
-
-Install
-=======
-
-Please refer to the `Virtual environment`_ documentation for installation best
-practices. If not using a virtual environment, please consider passing the
-widely recommended `'--user' flag`_ when invoking ``pip``.
-
-.. _Virtual environment: https://virtualenv.pypa.io/en/latest/
-.. _'--user' flag: https://packaging.python.org/tutorials/installing-packages/#installing-to-the-user-site
-
-.. code-block:: bash
-
-    $ pip install 'molecule[docker]'

roles/nrpe/molecule/os-family-redhat/converge.yml

@@ -1,5 +0,0 @@
----
-- name: Converge
-  hosts: all
-  roles:
-    - role: hspaans.nrpe

roles/nrpe/molecule/os-family-redhat/molecule.yml

@@ -1,47 +0,0 @@
----
-dependency:
-  name: galaxy
-driver:
-  name: docker
-lint: |
-  set -e
-  yamllint `git ls-files '*.yaml' '*.yml'`
-  ansible-lint
-  # flake8
-platforms:
-  - name: centos-7
-    image: "ghcr.io/hspaans/molecule-container-centos:7"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-  - name: centos-8
-    image: "ghcr.io/hspaans/molecule-container-centos:8"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-  - name: fedora-33
-    image: "ghcr.io/hspaans/molecule-container-fedora:33"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-  - name: fedora-34
-    image: "ghcr.io/hspaans/molecule-container-fedora:34"
-    command: ""
-    volumes:
-      - /sys/fs/cgroup:/sys/fs/cgroup:ro
-    privileged: true
-    pre_build_image: true
-provisioner:
-  name: ansible
-  inventory:
-    group_vars:
-      all:
-        nrpe_plugin_packages: []
-verifier:
-  name: testinfra

roles/nrpe/molecule/os-family-redhat/prepare.yml

@@ -1,27 +0,0 @@
----
-- name: Prepare
-  hosts: all
-
-  pre_tasks:
-    - name: Update apt cache (on Debian).
-      apt:
-        update_cache: true
-        cache_valid_time: 3600
-      when: ansible_os_family == 'Debian'
-
-    - name: Remove packages
-      package:
-        name: apt-utils
-        state: present
-      when: ansible_os_family == 'Debian'
-
-    - name: Update yum cache (on Redhat).
-      yum:
-        update_cache: true
-      when: ansible_os_family == 'RedHat'
-
-  tasks:
-    - name: Remove packages
-      package:
-        name: git
-        state: absent

roles/nrpe/molecule/os-family-redhat/tests/conftest.py

@@ -1,23 +0,0 @@
-"""PyTest Fixtures."""
-from __future__ import absolute_import
-
-import os
-
-import pytest
-
-
-def pytest_runtest_setup():
-    """Run tests only when under molecule with testinfra installed."""
-    try:
-        # pylint: disable = import-outside-toplevel
-        import testinfra
-    except ImportError:
-        pytest.skip("Test requires testinfra", allow_module_level=True)
-    if "MOLECULE_INVENTORY_FILE" in os.environ:
-        pytest.testinfra_hosts = testinfra.utils.ansible_runner.AnsibleRunner(
-            os.environ["MOLECULE_INVENTORY_FILE"]
-        ).get_hosts("all")
-    else:
-        pytest.skip(
-            "Test should run only from inside molecule.", allow_module_level=True
-        )

roles/nrpe/molecule/os-family-redhat/tests/test_default.py

@@ -1,11 +0,0 @@
-"""Role testing files using testinfra."""
-
-import pytest
-
-
-@pytest.mark.parametrize("pkg", ["nrpe"])
-def test_pkg_installed(host, pkg):
-    """Test if package installed."""
-    package = host.package(pkg)
-
-    assert package.is_installed

roles/nrpe/templates/nrpe_local.cfg.j2

@@ -9,3 +9,9 @@ allowed_hosts={{ nrpe_server_allowed_hosts | join(',') }}
 command[{{ command }}]={{ nrpe_server_plugins_dir }}/{{ nrpe_command[command]["script"] }} {{ nrpe_command[command]["option"] }}
 {% endfor %}
 {% endif %}
+{% if nrpe_command_extra is defined %}
+# Extra Commands
+{% for command in nrpe_command_extra %}
+command[{{ command }}]={{ nrpe_command_extra[command]["script"] }} {{ nrpe_command_extra[command]["option"] }}
+{% endfor %}
+{% endif %}

roles/nrpe/tests/inventory

@@ -1 +0,0 @@
-localhost

roles/nrpe/tests/test.yml

@@ -1,5 +0,0 @@
----
-- hosts: localhost
-  remote_user: root
-  roles:
-    - hspaans.nrpe

roles/nrpe/tox.ini

@@ -1,20 +0,0 @@
-[tox]
-minversion = 3.23.0
-envlist = py3
-skipsdist = true
-
-[testenv]
-passenv = *
-deps =
-    ansible-lint
-    molecule
-    molecule-docker
-    docker
-    flake8
-    flake8-docstrings
-    flake8-pylint
-    pytest
-    pytest-testinfra
-    anisble
-commands =
-    molecule test --all