Add Ansible-driven DR bootstrap (#9) + PVC capture in backups
restore-k8s-apps.sh's own header explicitly scoped out fresh-cluster
provisioning ("assumes the target cluster/namespaces/PVCs already
exist"). This adds exactly that missing piece as an Ansible playbook
rather than another bash script — a better fit for "provision a target
into a working state" than for the actual data-restore logic, which stays
in restore-k8s-apps.sh unchanged and is just invoked as the final step.
dr-bootstrap.yml, given an app name + a backup archive:
1. Checks the target's environment (disk/mem/cpu) against k3s minimums
before touching anything
2. Installs k3s only if not already present (checked, not assumed)
3. Creates the namespace, PVC(s), Secret, and Deployments/Services/
Ingress from the backup's captured manifests — sanitized first
(sanitize_k8s_manifest.py strips resourceVersion/uid/status/
volumeName, the same class of bug as the earlier Secret-apply fix,
generalized) since a raw `kubectl get -o yaml` dump can't be
reapplied to a different cluster as-is
4. Hands off to the existing, already-validated restore-k8s-apps.sh for
the actual data population
backup-k8s-apps.sh gets a small but necessary addition: PVC specs were
never captured at all before (only Deployment/Service/ConfigMap/Ingress),
so there was nothing for a fresh-cluster bootstrap to provision the PVC
from. Captured with the same label selector as the existing manifests
capture, so it stays in lockstep automatically.
Verified for real, not just syntax-checked: ran the full playbook against
the standby VM (freshly installed k3s, empty namespace) using a live n8n
backup. Namespace/PVCs/Secret/Deployments provisioned from nothing, data
restored, pod healthy, and the workflow count in the restored Postgres DB
(11) matched the real source server exactly.
This commit is contained in:
root
179
ansible/dr-bootstrap.yml
Normal file
179
ansible/dr-bootstrap.yml
Normal file
@@ -0,0 +1,179 @@
|
||||
---
|
||||
# dr-bootstrap.yml — minimum-viable cross-server disaster recovery.
|
||||
#
|
||||
# Provisions a target server into a working k3s node (checking the
|
||||
# environment first, installing only if missing — safe to re-run) and
|
||||
# restores one app from a captured backup-k8s-apps.sh archive onto it,
|
||||
# from scratch: creates the namespace, the PVC(s) (never captured by the
|
||||
# backup before this — see sanitize_k8s_manifest.py's header) and the
|
||||
# Deployments/Services/Secret, then hands off to the existing
|
||||
# restore-k8s-apps.sh for the actual data population (PVC contents + DB),
|
||||
# which already assumes namespace/PVC/Deployments exist — this playbook is
|
||||
# what makes that assumption true on a target that starts with nothing.
|
||||
#
|
||||
# Usage:
|
||||
# ansible-playbook -i inventory.ini dr-bootstrap.yml \
|
||||
# -e dr_app=n8n -e dr_backup_archive=/root/backups/myapps-k8s-backup-XXXXXXXX_XXXXXX.tar.gz
|
||||
#
|
||||
# NOT in scope (same cut as restore-k8s-apps.sh's own header): this
|
||||
# targets a server that already has network/DNS/SSH access sorted out —
|
||||
# it provisions the CLUSTER and the APP, not the surrounding
|
||||
# infrastructure.
|
||||
|
||||
- name: DR Bootstrap
|
||||
hosts: dr_target
|
||||
gather_facts: yes
|
||||
vars:
|
||||
remote_work_dir: "/root/dr-bootstrap-{{ dr_app }}"
|
||||
k3s_version: "v1.35.4+k3s1"
|
||||
|
||||
tasks:
|
||||
- name: Fail fast if required vars are missing
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- dr_app is defined
|
||||
- dr_backup_archive is defined
|
||||
fail_msg: "Pass -e dr_app=<name> -e dr_backup_archive=<local path to a myapps-k8s-backup-*.tar.gz>"
|
||||
|
||||
# ── 1. Environment check — before touching anything ──────────────
|
||||
- name: Check free disk space and memory
|
||||
ansible.builtin.set_fact:
|
||||
disk_free_gb: "{{ ((ansible_facts['mounts'] | selectattr('mount', 'equalto', '/') | first).size_available / (1024**3)) | round(1) }}"
|
||||
|
||||
- name: Assert environment meets k3s minimums
|
||||
ansible.builtin.assert:
|
||||
that:
|
||||
- disk_free_gb | float > 5
|
||||
- ansible_facts['memtotal_mb'] > 1500
|
||||
fail_msg: >
|
||||
Environment below k3s minimums on {{ inventory_hostname }}:
|
||||
{{ disk_free_gb }}GB free disk / {{ ansible_facts['memtotal_mb'] }}MB RAM
|
||||
(need >5GB / >1.5GB). Not proceeding.
|
||||
success_msg: >
|
||||
Environment OK on {{ inventory_hostname }}: {{ disk_free_gb }}GB free disk,
|
||||
{{ ansible_facts['memtotal_mb'] }}MB RAM, {{ ansible_facts['processor_vcpus'] }} vCPUs.
|
||||
|
||||
# ── 2. k3s — install only if missing, never re-run blindly ───────
|
||||
- name: Check if k3s is already installed
|
||||
ansible.builtin.command: which k3s
|
||||
register: k3s_check
|
||||
ignore_errors: true
|
||||
changed_when: false
|
||||
|
||||
- name: Install k3s (skipped if already present)
|
||||
ansible.builtin.shell: >
|
||||
curl -sfL https://get.k3s.io |
|
||||
INSTALL_K3S_VERSION="{{ k3s_version }}" sh -s - --write-kubeconfig-mode 644
|
||||
when: k3s_check.rc != 0
|
||||
|
||||
- name: Wait for the k3s node to report Ready
|
||||
ansible.builtin.shell: k3s kubectl get nodes --no-headers | awk '{print $2}'
|
||||
register: node_status
|
||||
until: "'Ready' in node_status.stdout"
|
||||
retries: 24
|
||||
delay: 5
|
||||
changed_when: false
|
||||
|
||||
# ── 3. Stage the backup + tooling on the target ───────────────────
|
||||
- name: Create working directory
|
||||
ansible.builtin.file:
|
||||
path: "{{ remote_work_dir }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Copy backup archive to target
|
||||
ansible.builtin.copy:
|
||||
src: "{{ dr_backup_archive }}"
|
||||
dest: "{{ remote_work_dir }}/backup.tar.gz"
|
||||
|
||||
- name: Extract backup archive
|
||||
ansible.builtin.unarchive:
|
||||
src: "{{ remote_work_dir }}/backup.tar.gz"
|
||||
dest: "{{ remote_work_dir }}"
|
||||
remote_src: true
|
||||
extra_opts: ['--strip-components=1']
|
||||
|
||||
- name: Copy sanitizer script
|
||||
ansible.builtin.copy:
|
||||
src: sanitize_k8s_manifest.py
|
||||
dest: "{{ remote_work_dir }}/sanitize_k8s_manifest.py"
|
||||
mode: '0755'
|
||||
|
||||
- name: Copy restore-k8s-apps.sh
|
||||
ansible.builtin.copy:
|
||||
src: "{{ playbook_dir }}/../backup/restore-k8s-apps.sh"
|
||||
dest: "{{ remote_work_dir }}/restore-k8s-apps.sh"
|
||||
mode: '0755'
|
||||
|
||||
- name: Determine the app's namespace from the captured manifests
|
||||
ansible.builtin.shell: >
|
||||
python3 -c "import yaml; d=yaml.safe_load(open('{{ remote_work_dir }}/{{ dr_app }}/manifests.yaml'));
|
||||
print(d['items'][0]['metadata']['namespace'])"
|
||||
register: ns_result
|
||||
changed_when: false
|
||||
|
||||
- name: Set dr_namespace fact
|
||||
ansible.builtin.set_fact:
|
||||
dr_namespace: "{{ ns_result.stdout }}"
|
||||
|
||||
# ── 4. Provision namespace + PVC + Secret + Deployments/Services ──
|
||||
- name: Create namespace (idempotent)
|
||||
ansible.builtin.shell: >
|
||||
k3s kubectl get namespace {{ dr_namespace }} ||
|
||||
k3s kubectl create namespace {{ dr_namespace }}
|
||||
changed_when: false
|
||||
|
||||
- name: Check for a captured PVC spec
|
||||
ansible.builtin.stat:
|
||||
path: "{{ remote_work_dir }}/{{ dr_app }}/pvc.yaml"
|
||||
register: pvc_file
|
||||
|
||||
- name: Sanitize + apply PVC spec
|
||||
when: pvc_file.stat.exists
|
||||
block:
|
||||
- ansible.builtin.command: >
|
||||
python3 {{ remote_work_dir }}/sanitize_k8s_manifest.py
|
||||
{{ remote_work_dir }}/{{ dr_app }}/pvc.yaml
|
||||
{{ remote_work_dir }}/{{ dr_app }}/pvc.clean.yaml
|
||||
- ansible.builtin.command: k3s kubectl apply -f {{ remote_work_dir }}/{{ dr_app }}/pvc.clean.yaml
|
||||
|
||||
- name: Sanitize + apply Secret
|
||||
block:
|
||||
- ansible.builtin.command: >
|
||||
python3 {{ remote_work_dir }}/sanitize_k8s_manifest.py
|
||||
{{ remote_work_dir }}/{{ dr_app }}/secret.yaml
|
||||
{{ remote_work_dir }}/{{ dr_app }}/secret.clean.yaml
|
||||
- ansible.builtin.command: k3s kubectl apply -f {{ remote_work_dir }}/{{ dr_app }}/secret.clean.yaml
|
||||
|
||||
- name: Sanitize + apply Deployments/Services/Ingress
|
||||
block:
|
||||
- ansible.builtin.command: >
|
||||
python3 {{ remote_work_dir }}/sanitize_k8s_manifest.py
|
||||
{{ remote_work_dir }}/{{ dr_app }}/manifests.yaml
|
||||
{{ remote_work_dir }}/{{ dr_app }}/manifests.clean.yaml
|
||||
- ansible.builtin.command: k3s kubectl apply -f {{ remote_work_dir }}/{{ dr_app }}/manifests.clean.yaml
|
||||
|
||||
- name: Wait for PVC(s) to be Bound
|
||||
when: pvc_file.stat.exists
|
||||
ansible.builtin.shell: >
|
||||
k3s kubectl get pvc -n {{ dr_namespace }} -o jsonpath='{.items[*].status.phase}'
|
||||
register: pvc_status
|
||||
until: "'Pending' not in pvc_status.stdout"
|
||||
retries: 12
|
||||
delay: 5
|
||||
changed_when: false
|
||||
|
||||
# ── 5. Data population — reuse the existing, already-validated
|
||||
# restore pipeline. It handles PVC contents + DB restore; steps
|
||||
# 1-4 above are only what's needed to make its own assumption
|
||||
# (namespace/PVC/Deployments already exist) true on a target
|
||||
# that started with nothing. ──────────────────────────────────
|
||||
- name: Run restore-k8s-apps.sh to populate data
|
||||
ansible.builtin.command: bash restore-k8s-apps.sh --apps {{ dr_app }}
|
||||
args:
|
||||
chdir: "{{ remote_work_dir }}"
|
||||
register: restore_output
|
||||
|
||||
- name: Show restore output
|
||||
ansible.builtin.debug:
|
||||
var: restore_output.stdout_lines
|
||||
Reference in New Issue
Block a user