diff --git a/ansible/dr-bootstrap.yml b/ansible/dr-bootstrap.yml index 422d1cc..abccf45 100644 --- a/ansible/dr-bootstrap.yml +++ b/ansible/dr-bootstrap.yml @@ -21,7 +21,7 @@ # infrastructure. - name: DR Bootstrap - hosts: dr_target + hosts: all gather_facts: yes vars: remote_work_dir: "/root/dr-bootstrap-{{ dr_app }}" diff --git a/platform/app.py b/platform/app.py index 2590420..39a5330 100644 --- a/platform/app.py +++ b/platform/app.py @@ -3,6 +3,7 @@ from flask import Flask, render_template, request, redirect, url_for, session, j import os import re import subprocess +import sys import threading import uuid import time @@ -869,36 +870,74 @@ def restore_start(): if not remote_ip: return jsonify({'error': 'remote_ip required'}), 400 - base_opts = "-o StrictHostKeyChecking=no -o ConnectTimeout=15" + # Unified path: every "External Machine" restore goes through the + # same Ansible playbook used for from-scratch DR (ansible/dr- + # bootstrap.yml) rather than a bare scp+ssh+restore-k8s-apps.sh + # dance. It's safe for BOTH cases, not just the empty-server one: + # the playbook's own steps (create-namespace-if-missing, `kubectl + # apply` for the sanitized PVC/Secret/manifests) are no-ops against + # a target that already has this app running with a matching spec + # — apply only reconciles differences, and a backup's own captured + # manifests are by definition identical to what's already live — + # so "restore onto an existing cluster" and "restore onto nothing" + # are the same command now; the playbook's own checks-then-acts + # steps decide how much of it actually needs to do anything. + ansible_dir = '/root/CloudOps/ansible' + playbook_path = f'{ansible_dir}/dr-bootstrap.yml' + if not os.path.exists(playbook_path): + return jsonify({'error': f'Ansible playbook not found at {playbook_path}'}), 500 + + # ANSIBLE_HOST_KEY_CHECKING=False (env var, below) rather than + # -e ansible_ssh_common_args='-o StrictHostKeyChecking=no ...': + # passing that as an extra-var hits what looks like a real bug in + # this ansible-core version's SSH connection plugin (its own + # internal tty-detection re-parses the string with a strict + # argparse and throws "argument -o: expected one argument" even + # for a single well-formed -o KEY=VALUE) — reproduced directly on + # the CLI, not a quoting artifact from this code. The env var + # achieves the same effect (disables host-key checking) through + # ansible's own dedicated option instead, bypassing that path + # entirely. + common_evars = [ + f"ansible_user={remote_user}", + f"ansible_port={remote_port}", + ] if auth_method == 'key': if not ssh_key_path: return jsonify({'error': 'ssh_key_path required'}), 400 - ssh_prefix = f"ssh -p {remote_port} -i {ssh_key_path} {base_opts}" - scp_prefix = f"scp -P {remote_port} -i {ssh_key_path} {base_opts}" + common_evars.append(f"ansible_ssh_private_key_file={ssh_key_path}") else: if not ssh_password: return jsonify({'error': 'ssh_password required'}), 400 - ssh_prefix = f"sshpass -p '{ssh_password}' ssh -p {remote_port} {base_opts}" - scp_prefix = f"sshpass -p '{ssh_password}' scp -P {remote_port} {base_opts}" + common_evars.append(f"ansible_password='{ssh_password}'") - remote_dest = f"/backups/restore-session-{uuid.uuid4().hex[:8]}" + evars_str = " ".join(f"-e {v}" for v in common_evars) + # ansible-playbook is a pip console-script installed alongside + # whichever python is actually running this process — the system + # one in the main pod's Docker image (no venv there), or this + # server's own venv/bin on the standby. A bare "ansible-playbook" + # in the command string only resolves on the main pod (system PATH + # already includes it); on the standby, running a venv's python + # directly (not through `activate`) never puts venv/bin on PATH, + # so it'd fail with "command not found" there. Resolving it + # relative to sys.executable works in both. + ansible_playbook_bin = os.path.join(os.path.dirname(sys.executable), 'ansible-playbook') + + # dr-bootstrap.yml restores one app per run (it reads that app's own + # manifests.yaml to work out the namespace) — loop for a multi-app + # selection, each run against the same already-staged backup_path. + apps_for_ansible = selected_apps if selected_apps else sorted(ALLOWED_RESTORE_APPS) + ansible_env = "ANSIBLE_HOST_KEY_CHECKING=False ANSIBLE_TIMEOUT=15" + per_app_cmds = [ + f"echo '=== [{a}] provisioning (if needed) + restore via Ansible ===' && " + f"{ansible_env} {ansible_playbook_bin} -i '{remote_ip},' {evars_str} " + f"-e dr_app={a} -e dr_backup_archive={backup_path} {playbook_path}" + for a in apps_for_ansible + ] cmd = ( - f"echo 'Connecting to {remote_user}@{remote_ip}:{remote_port}...' && " + f"echo 'Target: {remote_user}@{remote_ip}:{remote_port}' && " f"echo 'Apps: {apps_label}' && " - f"{ssh_prefix} {remote_user}@{remote_ip} 'mkdir -p {remote_dest}' && " - f"echo 'Connected.' && " - f"echo 'Copying backup archive...' && " - f"{scp_prefix} {backup_path} {remote_user}@{remote_ip}:{remote_dest}/{backup_file} && " - f"echo 'Copying restore script...' && " - f"{scp_prefix} {restore_script_local} {remote_user}@{remote_ip}:{remote_dest}/restore-k8s-apps.sh && " - f"echo 'Running restore on {remote_ip}:{remote_port}...' && " - f"{ssh_prefix} {remote_user}@{remote_ip} " - f"'set -e && cd {remote_dest} && " - f"tar -xzf {backup_file} --strip-components=1 && " - f"chmod +x restore-k8s-apps.sh && bash restore-k8s-apps.sh{apps_flag}' ; " - f"EXIT=$? ; " - f"{ssh_prefix} {remote_user}@{remote_ip} 'rm -rf {remote_dest}' 2>/dev/null ; " - f"exit $EXIT" + + " && ".join(per_app_cmds) ) job_id = str(uuid.uuid4()) diff --git a/platform/requirements.txt b/platform/requirements.txt index 6d82a78..eccabfa 100644 --- a/platform/requirements.txt +++ b/platform/requirements.txt @@ -1,4 +1,5 @@ boto3==1.43.5 flask psutil -kubernetes \ No newline at end of file +kubernetes +ansible-core \ No newline at end of file diff --git a/scripts/sync-standby-platform.sh b/scripts/sync-standby-platform.sh index 94637b8..401f530 100755 --- a/scripts/sync-standby-platform.sh +++ b/scripts/sync-standby-platform.sh @@ -44,6 +44,13 @@ LIVE_CONFIG="/root/management-platform/config.py" # here to scp across, so mirror it too, not just platform/. LOCAL_BACKUP_SRC="/root/CloudOps/backup/" VM_BACKUP_DIR="/root/CloudOps/backup" + +# Same reason as backup/ above — restore_start() references this as a +# fixed absolute path (/root/CloudOps/ansible/dr-bootstrap.yml) to drive +# every "External Machine" restore now, not just true from-scratch DR. +LOCAL_ANSIBLE_SRC="/root/CloudOps/ansible/" +VM_ANSIBLE_DIR="/root/CloudOps/ansible" + LOG="/root/CloudOps/scripts/sync-standby-platform.log" SSH_OPTS="-i $VM_KEY -p $VM_PORT -o StrictHostKeyChecking=no -o ConnectTimeout=15" @@ -82,6 +89,17 @@ if [ $? -ne 0 ]; then fi CHANGES="${CHANGES}${BACKUP_CHANGES}" +ssh $SSH_OPTS "$VM_USER@$VM_HOST" "mkdir -p $VM_ANSIBLE_DIR" +ANSIBLE_CHANGES=$(rsync -a --delete -e "ssh $SSH_OPTS" \ + --exclude tmp --exclude '*.retry' \ + --itemize-changes \ + "$LOCAL_ANSIBLE_SRC" "$VM_USER@$VM_HOST:$VM_ANSIBLE_DIR/" 2>>"$LOG") +if [ $? -ne 0 ]; then + log "❌ rsync of ansible/ failed" + exit 1 +fi +CHANGES="${CHANGES}${ANSIBLE_CHANGES}" + CONFIG_CHANGED="" if ! scp $SCP_OPTS "$LIVE_CONFIG" "$VM_USER@$VM_HOST:$VM_DEPLOY_DIR/config.py.new" &>>"$LOG"; then log "❌ Could not copy live config.py to VM"