CloudOps/scripts/sync-standby-platform.sh

#!/bin/bash
# ─────────────────────────────────────────────────────────────
# sync-standby-platform.sh
# Keeps the warm-standby copy of management-platform on the VM
# (178.18.243.51) in sync with the real source of truth on this
# server (/root/CloudOps/platform), pushed directly over SSH.
#
# Replaces the old pull-platform-backup.sh + deploy-platform.sh
# pair on the VM, which pulled/redeployed a platform-backup-*.tar.gz
# tarball of /root/management-platform (the pre-migration Docker-era
# deploy dir). That tarball generation was disabled on 2026-08-14
# when the platform's source of truth moved to git-tracked
# /root/CloudOps + Jenkins — nothing replaced it on the VM side, so
# the standby silently kept re-deploying its last cached snapshot
# (found stale at 2+ months old during PFE failover investigation).
#
# config.py is intentionally NOT sourced from /root/CloudOps/platform
# (gitignored there, and can drift — see that file's own history).
# The live /root/management-platform/config.py on THIS server is the
# real one actually mounted into the running pod; it already contains
# the RUNNING_ON_MAIN_SERVER auto-detect logic that makes the exact
# same file behave correctly on both this server and the VM, so it's
# copied byte-for-byte rather than templated.
#
# Run hourly via cron on this server (NOT on the VM — the VM has no
# reason to pull, this server already pushes everywhere else it
# manages, e.g. backup-k8s-apps.sh's VM tier).
# ─────────────────────────────────────────────────────────────

VM_HOST="178.18.243.51"
VM_PORT="22"
VM_USER="root"
VM_KEY="/root/.ssh/id_rsa"
VM_DEPLOY_DIR="/root/management-platform"

LOCAL_SRC="/root/CloudOps/platform/"
LIVE_CONFIG="/root/management-platform/config.py"

# restore_start()/api_backup_run() in app.py reference this as a fixed
# absolute path (/root/CloudOps/backup/restore-k8s-apps.sh) — it's a
# sibling of platform/, never baked into the image, reached via the same
# rw hostPath mount as everything else. Restoring FROM the standby (target
# = remote, pointed at the main server over the tunnel) needs a local copy
# here to scp across, so mirror it too, not just platform/.
LOCAL_BACKUP_SRC="/root/CloudOps/backup/"
VM_BACKUP_DIR="/root/CloudOps/backup"

# Same reason as backup/ above — restore_start() references this as a
# fixed absolute path (/root/CloudOps/ansible/dr-bootstrap.yml) to drive
# every "External Machine" restore now, not just true from-scratch DR.
LOCAL_ANSIBLE_SRC="/root/CloudOps/ansible/"
VM_ANSIBLE_DIR="/root/CloudOps/ansible"

LOG="/root/CloudOps/scripts/sync-standby-platform.log"

SSH_OPTS="-i $VM_KEY -p $VM_PORT -o StrictHostKeyChecking=no -o ConnectTimeout=15"
SCP_OPTS="-i $VM_KEY -P $VM_PORT -o StrictHostKeyChecking=no -o ConnectTimeout=15"

log() { echo "$(date '+%Y-%m-%d %H:%M:%S') $1" | tee -a "$LOG"; }

log "🔄 Syncing platform code → $VM_USER@$VM_HOST:$VM_DEPLOY_DIR"

ssh $SSH_OPTS "$VM_USER@$VM_HOST" "mkdir -p $VM_DEPLOY_DIR" || {
    log "❌ Cannot reach VM — aborting"
    exit 1
}

# --itemize-changes so we can tell below whether anything actually
# changed (drives the restart-only-if-needed decision).
CHANGES=$(rsync -a --delete -e "ssh $SSH_OPTS" \
    --exclude venv --exclude __pycache__ --exclude '*.pyc' \
    --exclude config.py --exclude '*.log' \
    --itemize-changes \
    "$LOCAL_SRC" "$VM_USER@$VM_HOST:$VM_DEPLOY_DIR/" 2>>"$LOG")
RSYNC_RC=$?

if [ $RSYNC_RC -ne 0 ]; then
    log "❌ rsync failed (rc=$RSYNC_RC)"
    exit 1
fi

ssh $SSH_OPTS "$VM_USER@$VM_HOST" "mkdir -p $VM_BACKUP_DIR"
BACKUP_CHANGES=$(rsync -a --delete -e "ssh $SSH_OPTS" \
    --itemize-changes \
    "$LOCAL_BACKUP_SRC" "$VM_USER@$VM_HOST:$VM_BACKUP_DIR/" 2>>"$LOG")
if [ $? -ne 0 ]; then
    log "❌ rsync of backup/ failed"
    exit 1
fi
CHANGES="${CHANGES}${BACKUP_CHANGES}"

ssh $SSH_OPTS "$VM_USER@$VM_HOST" "mkdir -p $VM_ANSIBLE_DIR"
ANSIBLE_CHANGES=$(rsync -a --delete -e "ssh $SSH_OPTS" \
    --exclude tmp --exclude '*.retry' \
    --itemize-changes \
    "$LOCAL_ANSIBLE_SRC" "$VM_USER@$VM_HOST:$VM_ANSIBLE_DIR/" 2>>"$LOG")
if [ $? -ne 0 ]; then
    log "❌ rsync of ansible/ failed"
    exit 1
fi
CHANGES="${CHANGES}${ANSIBLE_CHANGES}"

CONFIG_CHANGED=""
if ! scp $SCP_OPTS "$LIVE_CONFIG" "$VM_USER@$VM_HOST:$VM_DEPLOY_DIR/config.py.new" &>>"$LOG"; then
    log "❌ Could not copy live config.py to VM"
    exit 1
fi
if ! ssh $SSH_OPTS "$VM_USER@$VM_HOST" \
    "cmp -s $VM_DEPLOY_DIR/config.py $VM_DEPLOY_DIR/config.py.new 2>/dev/null"; then
    CONFIG_CHANGED="yes"
fi
ssh $SSH_OPTS "$VM_USER@$VM_HOST" "mv $VM_DEPLOY_DIR/config.py.new $VM_DEPLOY_DIR/config.py"

if [ -z "$CHANGES" ] && [ -z "$CONFIG_CHANGED" ]; then
    log "✅ Already up to date — nothing changed, no restart needed"
    exit 0
fi

log "📝 Changes detected:"
echo "$CHANGES" | tee -a "$LOG"
[ -n "$CONFIG_CHANGED" ] && log "   (config.py changed too)"

log "📦 Syncing venv deps (requirements.txt) ..."
ssh $SSH_OPTS "$VM_USER@$VM_HOST" "
    cd $VM_DEPLOY_DIR
    chmod +x *.sh 2>/dev/null
    chmod +x *.py 2>/dev/null
    if [ ! -d venv ]; then
        python3 -m venv venv
        venv/bin/python -m ensurepip --upgrade 2>/dev/null || true
    fi
    venv/bin/pip install --quiet -r requirements.txt
" >>"$LOG" 2>&1

log "🔁 Restarting management-platform service on VM ..."
ssh $SSH_OPTS "$VM_USER@$VM_HOST" "systemctl restart management-platform"
sleep 3
STATUS=$(ssh $SSH_OPTS "$VM_USER@$VM_HOST" "systemctl is-active management-platform 2>/dev/null")

if [ "$STATUS" = "active" ]; then
    log "✅ Standby platform restarted and running latest code"
else
    log "⚠️  Service not active after restart (status: $STATUS) — check journalctl -u management-platform on the VM"
    exit 1
fi