diff --git a/ansible/dr-bootstrap.yml b/ansible/dr-bootstrap.yml new file mode 100644 index 0000000..422d1cc --- /dev/null +++ b/ansible/dr-bootstrap.yml @@ -0,0 +1,179 @@ +--- +# dr-bootstrap.yml — minimum-viable cross-server disaster recovery. +# +# Provisions a target server into a working k3s node (checking the +# environment first, installing only if missing — safe to re-run) and +# restores one app from a captured backup-k8s-apps.sh archive onto it, +# from scratch: creates the namespace, the PVC(s) (never captured by the +# backup before this — see sanitize_k8s_manifest.py's header) and the +# Deployments/Services/Secret, then hands off to the existing +# restore-k8s-apps.sh for the actual data population (PVC contents + DB), +# which already assumes namespace/PVC/Deployments exist — this playbook is +# what makes that assumption true on a target that starts with nothing. +# +# Usage: +# ansible-playbook -i inventory.ini dr-bootstrap.yml \ +# -e dr_app=n8n -e dr_backup_archive=/root/backups/myapps-k8s-backup-XXXXXXXX_XXXXXX.tar.gz +# +# NOT in scope (same cut as restore-k8s-apps.sh's own header): this +# targets a server that already has network/DNS/SSH access sorted out — +# it provisions the CLUSTER and the APP, not the surrounding +# infrastructure. + +- name: DR Bootstrap + hosts: dr_target + gather_facts: yes + vars: + remote_work_dir: "/root/dr-bootstrap-{{ dr_app }}" + k3s_version: "v1.35.4+k3s1" + + tasks: + - name: Fail fast if required vars are missing + ansible.builtin.assert: + that: + - dr_app is defined + - dr_backup_archive is defined + fail_msg: "Pass -e dr_app= -e dr_backup_archive=" + + # ── 1. Environment check — before touching anything ────────────── + - name: Check free disk space and memory + ansible.builtin.set_fact: + disk_free_gb: "{{ ((ansible_facts['mounts'] | selectattr('mount', 'equalto', '/') | first).size_available / (1024**3)) | round(1) }}" + + - name: Assert environment meets k3s minimums + ansible.builtin.assert: + that: + - disk_free_gb | float > 5 + - ansible_facts['memtotal_mb'] > 1500 + fail_msg: > + Environment below k3s minimums on {{ inventory_hostname }}: + {{ disk_free_gb }}GB free disk / {{ ansible_facts['memtotal_mb'] }}MB RAM + (need >5GB / >1.5GB). Not proceeding. + success_msg: > + Environment OK on {{ inventory_hostname }}: {{ disk_free_gb }}GB free disk, + {{ ansible_facts['memtotal_mb'] }}MB RAM, {{ ansible_facts['processor_vcpus'] }} vCPUs. + + # ── 2. k3s — install only if missing, never re-run blindly ─────── + - name: Check if k3s is already installed + ansible.builtin.command: which k3s + register: k3s_check + ignore_errors: true + changed_when: false + + - name: Install k3s (skipped if already present) + ansible.builtin.shell: > + curl -sfL https://get.k3s.io | + INSTALL_K3S_VERSION="{{ k3s_version }}" sh -s - --write-kubeconfig-mode 644 + when: k3s_check.rc != 0 + + - name: Wait for the k3s node to report Ready + ansible.builtin.shell: k3s kubectl get nodes --no-headers | awk '{print $2}' + register: node_status + until: "'Ready' in node_status.stdout" + retries: 24 + delay: 5 + changed_when: false + + # ── 3. Stage the backup + tooling on the target ─────────────────── + - name: Create working directory + ansible.builtin.file: + path: "{{ remote_work_dir }}" + state: directory + mode: '0755' + + - name: Copy backup archive to target + ansible.builtin.copy: + src: "{{ dr_backup_archive }}" + dest: "{{ remote_work_dir }}/backup.tar.gz" + + - name: Extract backup archive + ansible.builtin.unarchive: + src: "{{ remote_work_dir }}/backup.tar.gz" + dest: "{{ remote_work_dir }}" + remote_src: true + extra_opts: ['--strip-components=1'] + + - name: Copy sanitizer script + ansible.builtin.copy: + src: sanitize_k8s_manifest.py + dest: "{{ remote_work_dir }}/sanitize_k8s_manifest.py" + mode: '0755' + + - name: Copy restore-k8s-apps.sh + ansible.builtin.copy: + src: "{{ playbook_dir }}/../backup/restore-k8s-apps.sh" + dest: "{{ remote_work_dir }}/restore-k8s-apps.sh" + mode: '0755' + + - name: Determine the app's namespace from the captured manifests + ansible.builtin.shell: > + python3 -c "import yaml; d=yaml.safe_load(open('{{ remote_work_dir }}/{{ dr_app }}/manifests.yaml')); + print(d['items'][0]['metadata']['namespace'])" + register: ns_result + changed_when: false + + - name: Set dr_namespace fact + ansible.builtin.set_fact: + dr_namespace: "{{ ns_result.stdout }}" + + # ── 4. Provision namespace + PVC + Secret + Deployments/Services ── + - name: Create namespace (idempotent) + ansible.builtin.shell: > + k3s kubectl get namespace {{ dr_namespace }} || + k3s kubectl create namespace {{ dr_namespace }} + changed_when: false + + - name: Check for a captured PVC spec + ansible.builtin.stat: + path: "{{ remote_work_dir }}/{{ dr_app }}/pvc.yaml" + register: pvc_file + + - name: Sanitize + apply PVC spec + when: pvc_file.stat.exists + block: + - ansible.builtin.command: > + python3 {{ remote_work_dir }}/sanitize_k8s_manifest.py + {{ remote_work_dir }}/{{ dr_app }}/pvc.yaml + {{ remote_work_dir }}/{{ dr_app }}/pvc.clean.yaml + - ansible.builtin.command: k3s kubectl apply -f {{ remote_work_dir }}/{{ dr_app }}/pvc.clean.yaml + + - name: Sanitize + apply Secret + block: + - ansible.builtin.command: > + python3 {{ remote_work_dir }}/sanitize_k8s_manifest.py + {{ remote_work_dir }}/{{ dr_app }}/secret.yaml + {{ remote_work_dir }}/{{ dr_app }}/secret.clean.yaml + - ansible.builtin.command: k3s kubectl apply -f {{ remote_work_dir }}/{{ dr_app }}/secret.clean.yaml + + - name: Sanitize + apply Deployments/Services/Ingress + block: + - ansible.builtin.command: > + python3 {{ remote_work_dir }}/sanitize_k8s_manifest.py + {{ remote_work_dir }}/{{ dr_app }}/manifests.yaml + {{ remote_work_dir }}/{{ dr_app }}/manifests.clean.yaml + - ansible.builtin.command: k3s kubectl apply -f {{ remote_work_dir }}/{{ dr_app }}/manifests.clean.yaml + + - name: Wait for PVC(s) to be Bound + when: pvc_file.stat.exists + ansible.builtin.shell: > + k3s kubectl get pvc -n {{ dr_namespace }} -o jsonpath='{.items[*].status.phase}' + register: pvc_status + until: "'Pending' not in pvc_status.stdout" + retries: 12 + delay: 5 + changed_when: false + + # ── 5. Data population — reuse the existing, already-validated + # restore pipeline. It handles PVC contents + DB restore; steps + # 1-4 above are only what's needed to make its own assumption + # (namespace/PVC/Deployments already exist) true on a target + # that started with nothing. ────────────────────────────────── + - name: Run restore-k8s-apps.sh to populate data + ansible.builtin.command: bash restore-k8s-apps.sh --apps {{ dr_app }} + args: + chdir: "{{ remote_work_dir }}" + register: restore_output + + - name: Show restore output + ansible.builtin.debug: + var: restore_output.stdout_lines diff --git a/ansible/inventory.ini b/ansible/inventory.ini new file mode 100644 index 0000000..f107546 --- /dev/null +++ b/ansible/inventory.ini @@ -0,0 +1,2 @@ +[dr_target] +178.18.243.51 ansible_user=root ansible_ssh_private_key_file=/root/.ssh/id_rsa ansible_ssh_common_args='-o StrictHostKeyChecking=no' diff --git a/ansible/sanitize_k8s_manifest.py b/ansible/sanitize_k8s_manifest.py new file mode 100755 index 0000000..952617b --- /dev/null +++ b/ansible/sanitize_k8s_manifest.py @@ -0,0 +1,60 @@ +#!/usr/bin/env python3 +"""sanitize_k8s_manifest.py — strip source-cluster-specific fields from a +captured `kubectl get ... -o yaml` dump so it can be applied fresh onto a +different (DR/standby) cluster. + +backup-k8s-apps.sh's manifests.yaml/pvc.yaml/secret.yaml are raw `kubectl +get -o yaml` output: server-assigned fields like resourceVersion/uid are +always present, and PVCs additionally carry spec.volumeName + a Bound +status pointing at a specific PersistentVolume that only exists on the +ORIGINAL cluster. Applying any of that verbatim onto a fresh cluster either +fails outright (a stale resourceVersion trips optimistic-concurrency +checks — the exact bug fixed in restore-k8s-apps.sh's Secret handling) or +silently tries to bind a PVC to a PV that was never created there. + +Usage: sanitize_k8s_manifest.py +""" +import sys +import yaml + +DROP_METADATA_KEYS = ( + 'resourceVersion', 'uid', 'creationTimestamp', 'selfLink', + 'generation', 'finalizers', 'managedFields', 'annotations', +) + + +def sanitize_item(item): + meta = item.get('metadata') or {} + for key in DROP_METADATA_KEYS: + meta.pop(key, None) + + item.pop('status', None) + + if item.get('kind') == 'PersistentVolumeClaim': + spec = item.get('spec') or {} + spec.pop('volumeName', None) + + return item + + +def main(): + if len(sys.argv) != 3: + print(f"usage: {sys.argv[0]} ", file=sys.stderr) + sys.exit(1) + + with open(sys.argv[1]) as f: + doc = yaml.safe_load(f) + + if doc is None: + items = [] + elif doc.get('kind') == 'List': + items = [sanitize_item(i) for i in doc.get('items', [])] + else: + items = [sanitize_item(doc)] + + with open(sys.argv[2], 'w') as f: + yaml.safe_dump_all(items, f, default_flow_style=False) + + +if __name__ == '__main__': + main() diff --git a/backup/backup-k8s-apps.sh b/backup/backup-k8s-apps.sh index 93acb00..68e2d1a 100755 --- a/backup/backup-k8s-apps.sh +++ b/backup/backup-k8s-apps.sh @@ -306,6 +306,19 @@ for app in $ALL_APPS; do && { echo "✅"; rm -f "$APP_DIR/manifests.err"; } \ || { echo "⚠️ FAILED (see manifests.err)"; ERRORS+=("$app:manifests:$(tr '\n' ' ' < "$APP_DIR/manifests.err" | cut -c1-200)"); } + # ---- 1b. PVC specs — NOT covered by the manifests capture above, and + # needed to provision a namespace from scratch on a fresh/DR + # cluster (restore-k8s-apps.sh assumes the PVC already exists; + # this is what actually creates it there). Same label selector + # as the manifests capture, so this stays in lockstep with + # whatever PVCs are actually labeled as ours — no separate + # per-app name table to keep in sync. ---- + echo -n " 💾 PVC specs ... " + kubectl get pvc -n "$ns" -l owner=ameni-boukattaya -o yaml \ + > "$APP_DIR/pvc.yaml" 2>"$APP_DIR/pvc.err" \ + && { echo "✅"; rm -f "$APP_DIR/pvc.err"; } \ + || { echo "⚠️ FAILED (see pvc.err)"; ERRORS+=("$app:pvc_spec:$(tr '\n' ' ' < "$APP_DIR/pvc.err" | cut -c1-200)"); } + # ---- 2. Secret (plaintext, encryption-at-rest deferred) ---- echo -n " 🔑 Secret ($secret_name) ... " kubectl get secret "$secret_name" -n "$ns" -o yaml \