#!/bin/bash
# =============================================
# backup-k8s-apps.sh — k8s-native replacement for backup-myapps.sh
# Backs up: n8n, Odoo, Mautic, Nextcloud, Frappe/ERPNext (all live in k3s)
#
# Per app, captures:
#   - k8s manifests (Deployment/Service/ConfigMap/Ingress, via `kubectl get -o yaml`)
#   - Secret manifest (plaintext for now — encryption-at-rest explicitly deferred)
#   - DB dump (pg_dump / mysqldump via `kubectl exec`, never a raw PGDATA/datadir copy)
#   - App-data PVC contents (via `kubectl exec ... tar`, never raw filesystem access —
#     see migration-status.md §5, sandbox-filesystem-divergence rule)
#
# Storage tiers (logic reused verbatim from backup-myapps.sh, just repointed at the
# new archive name/contents):
#   1. Local  → /root/backups/          (CRITICAL — always runs)
#   2. VM     → SSH tunnel → /backups/cloudproject/
#   3. Cloud  → Cloudflare R2 (S3-compatible, optional)
#
# NOTE: R2 and VM failures do NOT stop the backup. Local backup always completes first.
#
# Archive name is deliberately NOT "myapps-backup-*" (the legacy Docker-era pattern
# still produced nightly by /root/backup-myapps.sh via cron) — using a distinct
# "myapps-k8s-backup-*" prefix keeps the two backup lineages from mixing in the same
# retention pool / R2 prefix until the legacy cron job is explicitly retired.
# =============================================

set -uo pipefail

# When run inside the management-platform pod, only the static k3s binary is
# hostPath-mounted (no separate kubectl binary) — fall back to `k3s kubectl`.
# IMPORTANT: this pod also has a pre-existing rw hostPath mount of /root (for
# unrelated file-browsing features), which means the host's own root-owned
# ~/.kube/config (full cluster-admin) is silently visible inside the pod too.
# `--kubeconfig=/dev/null` is REQUIRED here to stop k3s kubectl from picking
# that up as a base config — without it, every command below silently runs
# as cluster-admin instead of the scoped management-platform-sa token,
# defeating the RBAC in management-platform-backup-rbac.yaml entirely.
if ! command -v kubectl &>/dev/null && command -v k3s &>/dev/null; then
    kubectl() {
        k3s kubectl \
            --kubeconfig=/dev/null \
            --server=https://kubernetes.default.svc \
            --certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \
            --token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
            "$@"
    }
fi

# --------------------------------------------------
# Parse --apps
# --------------------------------------------------
APPS_FILTER="all"   # "all" or comma-separated: frappe,odoo,nextcloud,mautic,n8n
VALID_APPS="frappe odoo nextcloud mautic n8n"

while [[ $# -gt 0 ]]; do
    case "$1" in
        --apps)
            APPS_FILTER="$2"
            shift 2
            ;;
        *)
            shift
            ;;
    esac
done

if [ "$APPS_FILTER" != "all" ]; then
    APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
    IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER"
    for _a in "${_APP_CHECK[@]}"; do
        if [[ " $VALID_APPS " != *" $_a "* ]]; then
            echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS"
            exit 1
        fi
    done
fi

app_selected() {
    local app="$1"
    [ "$APPS_FILTER" = "all" ] && return 0
    [[ ",$APPS_FILTER," == *",${app},"* ]]
}

# --------------------------------------------------
# Per-app metadata table
# app-key : namespace : app-deployment : db-deployment : db-engine : db-name(or DYNAMIC) : pvc-mount-path : container-name(for exec) : secret-name : db-user-key : db-pass-key
# --------------------------------------------------
declare -A NS=(          [n8n]=n8n            [odoo]=odoo            [mautic]=mautic         [nextcloud]=nextcloud         [frappe]=erpnext )
declare -A APP_DEPLOY=(  [n8n]=n8n            [odoo]=odoo            [mautic]=mautic         [nextcloud]=nextcloud         [frappe]=erpnext-web )
declare -A APP_CTR=(     [n8n]=n8n            [odoo]=odoo            [mautic]=mautic         [nextcloud]=nextcloud         [frappe]=erpnext-web )
declare -A DB_DEPLOY=(   [n8n]=n8n-postgres   [odoo]=odoo-postgres   [mautic]=mautic-mariadb [nextcloud]=nextcloud-postgres [frappe]=erpnext-mariadb )
declare -A DB_ENGINE=(   [n8n]=postgres       [odoo]=postgres        [mautic]=mariadb        [nextcloud]=postgres          [frappe]=mariadb )
declare -A DB_NAME=(     [n8n]=n8n            [odoo]=odoo            [mautic]=mautic         [nextcloud]=nextcloud         [frappe]=DYNAMIC )
declare -A PVC_PATH=(    [n8n]="/home/node/.n8n" [odoo]="/var/lib/odoo" [mautic]="/var/www/html" [nextcloud]="/var/www/html" [frappe]="/home/frappe/frappe-bench/sites" )
declare -A SECRET_NAME=( [n8n]=n8n-secrets    [odoo]=odoo-secrets    [mautic]=mautic-secrets [nextcloud]=nextcloud-secrets [frappe]=erpnext-secrets )
declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" )
declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD )

# Reproducible framework/vendor code that lives on the same PVC as real user
# data for these two apps (single shared mount, no separate volume). Both
# images bake these dirs into their own filesystem layer independent of any
# volume — Mautic has no logic to reconstruct them, so restore-k8s-apps.sh
# re-seeds them straight from the app's own image instead of from backup.
# Never pull request/exclude changes here without updating the matching
# RECONSTRUCT_DIRS table in restore-k8s-apps.sh.
declare -A TAR_EXCLUDES=( [mautic]="node_modules vendor" [nextcloud]="apps core dist 3rdparty lib" )

ALL_APPS="n8n odoo mautic nextcloud frappe"

BACKUP_DATE=$(date +%Y%m%d_%H%M%S)
BACKUP_NAME="myapps-k8s-backup-${BACKUP_DATE}"
BACKUP_DIR="/root/backups/${BACKUP_NAME}"
BACKUP_ARCHIVE="/root/backups/${BACKUP_NAME}.tar.gz"

VM_USER="root"
VM_HOST="178.18.243.51"
VM_PORT="22"
VM_KEY="/root/.ssh/id_rsa"
VM_DEST="/backups/cloudproject/"

# ── Load R2 credentials (identical to backup-myapps.sh) ────────────────────────
CREDENTIALS_FILE="/root/.r2-credentials"
if [ -f "$CREDENTIALS_FILE" ]; then
    set +u
    source "$CREDENTIALS_FILE"
    set -u
fi

if [ -z "${AWS_ACCESS_KEY_ID:-}" ]; then
    CONFIG_PY="/root/management-platform/config.py"
    if [ -f "$CONFIG_PY" ]; then
        _KEY=$(grep -oP '(?<=R2_ACCESS_KEY_ID",\s{5}")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
        _SEC=$(grep -oP '(?<=R2_SECRET_ACCESS_KEY", ")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
        _BKT=$(grep -oP '(?<=R2_BUCKET_NAME",\s{7}")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
        [ -n "$_KEY" ] && export AWS_ACCESS_KEY_ID="$_KEY"
        [ -n "$_SEC" ] && export AWS_SECRET_ACCESS_KEY="$_SEC"
        [ -n "$_BKT" ] && export R2_BUCKET_NAME="$_BKT"
    fi
fi

R2_ACCOUNT_ID="${R2_ACCOUNT_ID:-35e00c230cc8066252a2d9890b69aea2}"
R2_BUCKET="${R2_BUCKET_NAME:-navitrends-backups}"
R2_ENDPOINT="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"

BACKUP_LOG_FILE="/root/backups/k8s-backup-status.log"
MAX_BACKUPS=10
MAX_R2_BACKUPS=5

TIER1_STATUS="pending"
TIER2_STATUS="pending"
TIER3_STATUS="pending"

log_status() {
    local status="$1" name="$2" msg="${3:-}"
    echo "$(date '+%Y-%m-%d %H:%M:%S') | ${status} | ${name} | ${msg}" >> "$BACKUP_LOG_FILE"
}

# --------------------------------------------------
# Resource-safety gate (migration-status.md §5): abort before starting
# any heavy per-app work if we're already below threshold.
# --------------------------------------------------
check_resources() {
    local avail_mb avail_gb
    # /proc/meminfo instead of `free` — `free` isn't present in the
    # management-platform pod's minimal image, /proc/meminfo always is.
    avail_mb=$(awk '/^MemAvailable:/{printf "%d", $2/1024}' /proc/meminfo)
    avail_gb=$(df -BG / | awk 'NR==2{gsub("G","",$4); print $4}')
    # Threshold lowered 1536->1024MB 2026-08-21: this host's steady-state
    # available RAM sits ~1.3-1.5GB even when idle (k3s + Jenkins + Docker
    # monitoring/wazuh stack + dev tooling baseline) — investigated, no
    # single runaway process, just genuine multi-service baseline. 1536MB
    # was tripping on ordinary single-app backups. 1024MB still leaves
    # real headroom above what a pg_dump/mysqldump/tar op actually needs.
    if [ "$avail_mb" -lt 1024 ]; then
        echo "   ❌ RAM available ${avail_mb}MB < 1024MB threshold — aborting."
        log_status "FAILED" "$BACKUP_NAME" "resource_safety_ram_${avail_mb}mb"
        exit 1
    fi
    if [ "$avail_gb" -lt 10 ]; then
        echo "   ❌ Disk available ${avail_gb}GB < 10GB threshold — aborting."
        log_status "FAILED" "$BACKUP_NAME" "resource_safety_disk_${avail_gb}gb"
        exit 1
    fi
}

# --------------------------------------------------
# Per-backup status metadata sidecar (.meta.json) — powers the UI's
# green/yellow/red status dot without ever needing to decompress the
# (possibly 600MB+) archive itself. Written only after all 3 tiers are
# known, then best-effort mirrored to VM/R2 alongside the archive, same
# as the .sha256 sidecar already is — so status is readable from
# whichever tier the UI is listing, not just local.
# --------------------------------------------------
write_meta_sidecar() {
    local size_bytes meta_path
    size_bytes=$(stat -c%s "$BACKUP_ARCHIVE" 2>/dev/null || echo 0)
    meta_path="/root/backups/${BACKUP_NAME}.tar.gz.meta.json"

    python3 - "$BACKUP_NAME" "$size_bytes" "$TIER1_STATUS" "$TIER2_STATUS" "$TIER3_STATUS" \
        "${#INCLUDED_APPS[@]}" "${INCLUDED_APPS[@]}" "${ERRORS[@]}" > "$meta_path" <<'PYEOF'
import sys, json, datetime

backup_name  = sys.argv[1]
size_bytes   = int(sys.argv[2])
tier1, tier2, tier3 = sys.argv[3], sys.argv[4], sys.argv[5]
n_apps       = int(sys.argv[6])
apps         = sys.argv[7:7 + n_apps]
errors_raw   = sys.argv[7 + n_apps:]

def norm_tier(s):
    # "ok:572M" -> "ok"; "failed:rc=2" / "skipped:no_credentials" pass through
    # unchanged so the UI can tell a real failure apart from a deliberate skip.
    return 'ok' if s.startswith('ok') else s

errors = []
for e in errors_raw:
    parts = e.split(':', 2)
    errors.append({
        'app':    parts[0] if len(parts) > 0 else '',
        'step':   parts[1] if len(parts) > 1 else '',
        'detail': parts[2] if len(parts) > 2 else '',
    })

meta = {
    'backup_name': backup_name,
    'created_at':  datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds'),
    'apps':        apps,
    'apps_key':    ','.join(sorted(apps)),
    'size_bytes':  size_bytes,
    'errors':      errors,
    'tiers': {'local': norm_tier(tier1), 'vm': norm_tier(tier2), 'r2': norm_tier(tier3)},
}
json.dump(meta, sys.stdout, indent=2)
PYEOF

    if [ -s "$meta_path" ]; then
        echo "   ✅ Metadata sidecar written: $(basename "$meta_path")"
    else
        echo "   ⚠️  Metadata sidecar write failed"
        return 1
    fi

    # Best-effort mirror to whichever tiers actually succeeded — never fatal,
    # the local sidecar (already written above) is what matters most.
    if [ "$TIER2_STATUS" = "ok" ]; then
        scp -i "$VM_KEY" -P "$VM_PORT" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
            "$meta_path" "${VM_USER}@${VM_HOST}:${VM_DEST}" &>/dev/null || true
    fi
    if [ "$TIER3_STATUS" = "ok" ] && [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
        META_SCRIPT="/tmp/r2_meta_$$.py"
        cat > "$META_SCRIPT" << PYEOF2
import boto3
client = boto3.client('s3', endpoint_url="${R2_ENDPOINT}",
    aws_access_key_id="${AWS_ACCESS_KEY_ID}", aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}",
    region_name='auto')
client.upload_file("$meta_path", "${R2_BUCKET}", "backups/${BACKUP_NAME}.tar.gz.meta.json")
PYEOF2
        "$PYTHON_BIN" "$META_SCRIPT" &>/dev/null || true
        rm -f "$META_SCRIPT"
    fi
}

echo "========================================="
echo "📦 Starting k8s Backup: $BACKUP_NAME"
if [ "$APPS_FILTER" = "all" ]; then
    echo "   Apps: ALL ($ALL_APPS)"
else
    echo "   Apps: $APPS_FILTER"
fi
echo "   Tiers: Local → VM → ☁ Cloudflare R2"
echo "========================================="

check_resources
mkdir -p "$BACKUP_DIR" "/root/backups"
cd "$BACKUP_DIR"

APP_COUNT=0
INCLUDED_APPS=()   # apps actually attempted this run — feeds the .meta.json sidecar's apps_key
ERRORS=()          # "app:step:detail" entries — any entry here makes the sidecar's status red
for app in $ALL_APPS; do
    app_selected "$app" || continue
    INCLUDED_APPS+=("$app")
    ns="${NS[$app]}"
    app_deploy="${APP_DEPLOY[$app]}"
    app_ctr="${APP_CTR[$app]}"
    db_deploy="${DB_DEPLOY[$app]}"
    db_engine="${DB_ENGINE[$app]}"
    db_name="${DB_NAME[$app]}"
    pvc_path="${PVC_PATH[$app]}"
    secret_name="${SECRET_NAME[$app]}"
    db_user_key="${DB_USER_KEY[$app]}"
    db_pass_key="${DB_PASS_KEY[$app]}"

    echo ""
    echo "-----------------------------------------"
    echo "📁 [$app] namespace=$ns"
    echo "-----------------------------------------"

    APP_DIR="$BACKUP_DIR/$app"
    mkdir -p "$APP_DIR"

    # ---- 1. Manifests ----
    echo -n "   📄 Manifests (deployment/service/configmap/ingress) ... "
    kubectl get deployment,service,configmap,ingress -n "$ns" -l owner=ameni-boukattaya -o yaml \
        > "$APP_DIR/manifests.yaml" 2>"$APP_DIR/manifests.err" \
        && { echo "✅"; rm -f "$APP_DIR/manifests.err"; } \
        || { echo "⚠️  FAILED (see manifests.err)"; ERRORS+=("$app:manifests:$(tr '\n' ' ' < "$APP_DIR/manifests.err" | cut -c1-200)"); }

    # ---- 1b. PVC specs — NOT covered by the manifests capture above, and
    #          needed to provision a namespace from scratch on a fresh/DR
    #          cluster (restore-k8s-apps.sh assumes the PVC already exists;
    #          this is what actually creates it there). Same label selector
    #          as the manifests capture, so this stays in lockstep with
    #          whatever PVCs are actually labeled as ours — no separate
    #          per-app name table to keep in sync. ----
    echo -n "   💾 PVC specs ... "
    kubectl get pvc -n "$ns" -l owner=ameni-boukattaya -o yaml \
        > "$APP_DIR/pvc.yaml" 2>"$APP_DIR/pvc.err" \
        && { echo "✅"; rm -f "$APP_DIR/pvc.err"; } \
        || { echo "⚠️  FAILED (see pvc.err)"; ERRORS+=("$app:pvc_spec:$(tr '\n' ' ' < "$APP_DIR/pvc.err" | cut -c1-200)"); }

    # ---- 2. Secret (plaintext, encryption-at-rest deferred) ----
    echo -n "   🔑 Secret ($secret_name) ... "
    kubectl get secret "$secret_name" -n "$ns" -o yaml \
        > "$APP_DIR/secret.yaml" 2>"$APP_DIR/secret.err" \
        && { echo "✅"; rm -f "$APP_DIR/secret.err"; } \
        || { echo "⚠️  FAILED (see secret.err)"; ERRORS+=("$app:secret:$(tr '\n' ' ' < "$APP_DIR/secret.err" | cut -c1-200)"); }

    # ---- 3. DB dump via kubectl exec (never raw datadir copy) ----
    echo -n "   💾 DB dump ($db_engine) ... "
    case "$db_engine" in
        postgres)
            db_user=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_user_key}}" 2>/dev/null | base64 -d)
            db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
            if [ -n "$db_user" ] && [ -n "$db_pass" ]; then
                kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
                    pg_dump -U "$db_user" -d "$db_name" 2>"$APP_DIR/db-dump.err" \
                    | gzip > "$APP_DIR/db-dump.sql.gz"
                pg_dump_rc=${PIPESTATUS[0]}
                if [ "$pg_dump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then
                    echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1))"
                    rm -f "$APP_DIR/db-dump.err"
                else
                    echo "⚠️  FAILED (rc=$pg_dump_rc, see db-dump.err)"
                    ERRORS+=("$app:db_dump:rc=$pg_dump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)")
                fi
            else
                echo "⚠️  could not read DB credentials from secret"
                ERRORS+=("$app:db_dump:could not read DB credentials from secret")
            fi
            ;;
        mariadb)
            if [ "$db_name" = "DYNAMIC" ]; then
                # Frappe: real db_name/db_password live in site_config.json on the
                # sites PVC, not in the k8s Secret. Dump with root instead, but still
                # need the site's real db_name to target the right database.
                site_dir=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
                    sh -c 'for d in sites/*/; do [ -f "$d/site_config.json" ] && [ ! -L "${d%/}" ] && basename "$d" && break; done' 2>/dev/null)
                db_name=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
                    cat "sites/${site_dir}/site_config.json" 2>/dev/null \
                    | python3 -c 'import json,sys; print(json.load(sys.stdin)["db_name"])' 2>/dev/null)
            fi
            db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
            if [ -n "$db_name" ] && [ -n "$db_pass" ]; then
                kubectl exec -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \
                    mysqldump -uroot --databases "$db_name" 2>"$APP_DIR/db-dump.err" \
                    | gzip > "$APP_DIR/db-dump.sql.gz"
                mysqldump_rc=${PIPESTATUS[0]}
                if [ "$mysqldump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then
                    echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1), db=$db_name)"
                    echo "$db_name" > "$APP_DIR/db-name.txt"
                    rm -f "$APP_DIR/db-dump.err"
                else
                    echo "⚠️  FAILED (rc=$mysqldump_rc, see db-dump.err)"
                    ERRORS+=("$app:db_dump:rc=$mysqldump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)")
                fi
            else
                echo "⚠️  could not determine DB name/credentials"
                ERRORS+=("$app:db_dump:could not determine DB name/credentials")
            fi
            ;;
    esac

    # ---- 4. App-data PVC contents via kubectl exec (live tar, app-data only,
    #         never raw filesystem/PVC-path access — §5 sandbox-divergence rule) ----
    exclude_args=()
    if [ -n "${TAR_EXCLUDES[$app]:-}" ]; then
        for _excl_dir in ${TAR_EXCLUDES[$app]}; do
            exclude_args+=(--exclude="./$_excl_dir")
        done
    fi
    echo -n "   📦 PVC data ($pvc_path${exclude_args:+, excl: ${TAR_EXCLUDES[$app]}}) ... "
    kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
        tar czf - "${exclude_args[@]}" -C "$pvc_path" . 2>"$APP_DIR/pvc-data.err" \
        > "$APP_DIR/pvc-data.tar.gz"
    if [ -s "$APP_DIR/pvc-data.tar.gz" ]; then
        echo "✅ ($(du -h "$APP_DIR/pvc-data.tar.gz" | cut -f1))"
        rm -f "$APP_DIR/pvc-data.err"
    else
        echo "⚠️  FAILED (see pvc-data.err)"
        ERRORS+=("$app:pvc_data:$(tr '\n' ' ' < "$APP_DIR/pvc-data.err" 2>/dev/null | cut -c1-160)")
    fi

    APP_COUNT=$((APP_COUNT + 1))

    # Resource check between apps, not just at the start (§5: check every
    # 1-2 min during long-running/heavy ops; PVC tars can be sizeable).
    check_resources
done

# --------------------------------------------------
# Metadata + checksum
# --------------------------------------------------
echo ""
echo "📝 Writing backup metadata..."
cat > "$BACKUP_DIR/backup-info.txt" << EOF
Backup Name:  $BACKUP_NAME
Backup Date:  $(date)
Hostname:     $(hostname)
Apps:         $([ "$APPS_FILTER" = "all" ] && echo "$ALL_APPS" || echo "$APPS_FILTER")
Apps backed up: $APP_COUNT
kubectl:      $(kubectl version --client -o yaml 2>/dev/null | grep gitVersion | head -1 || echo 'N/A')
Storage Tiers:
  - Local:  /root/backups/
  - VM:     ${VM_HOST}:${VM_PORT} → ${VM_DEST}
  - Cloud:  Cloudflare R2 → s3://${R2_BUCKET}/backups/
EOF

echo "" >> "$BACKUP_DIR/backup-info.txt"
echo "Per-app checksums:" >> "$BACKUP_DIR/backup-info.txt"
find "$BACKUP_DIR" -type f \( -name "*.tar.gz" -o -name "*.sql.gz" \) -print0 \
    | xargs -0 -r sha256sum >> "$BACKUP_DIR/backup-info.txt"
echo "   ✅ Done"

# --------------------------------------------------
# Compress (identical logic to backup-myapps.sh)
# --------------------------------------------------
echo ""
echo "🗜️  Compressing backup..."
cd /root/backups

tar -czf "${BACKUP_NAME}.tar.gz" "${BACKUP_NAME}/" 2>/dev/null
if [ $? -ne 0 ]; then
    echo "   ❌ CRITICAL: Compression failed!"
    log_status "FAILED" "$BACKUP_NAME" "compression_failed"
    exit 1
fi

COMPRESSED_SIZE=$(du -h "${BACKUP_NAME}.tar.gz" | cut -f1)
echo "   ✅ Compressed size: $COMPRESSED_SIZE → $BACKUP_ARCHIVE"

sha256sum "${BACKUP_NAME}.tar.gz" > "${BACKUP_NAME}.tar.gz.sha256"
echo "   ✅ Checksum written"

rm -rf "$BACKUP_DIR"
TIER1_STATUS="ok:${COMPRESSED_SIZE}"
echo ""
echo "✅ [TIER 1] Local backup complete: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"

# --------------------------------------------------
# Retention — keep only MAX_BACKUPS locally (own glob, own lineage)
# --------------------------------------------------
echo ""
echo "🧹 [Retention] Keeping latest ${MAX_BACKUPS} local k8s backups..."
ARCHIVE_LIST=$(ls -t /root/backups/myapps-k8s-backup-*.tar.gz 2>/dev/null || true)
ARCHIVE_COUNT=$(echo "$ARCHIVE_LIST" | grep -c '.tar.gz' 2>/dev/null || echo "0")

if [ "$ARCHIVE_COUNT" -gt "$MAX_BACKUPS" ]; then
    TO_DELETE=$(echo "$ARCHIVE_LIST" | tail -n +$((MAX_BACKUPS + 1)))
    while IFS= read -r old_file; do
        [ -z "$old_file" ] && continue
        rm -f "$old_file" "${old_file}.sha256" "${old_file}.meta.json"
        echo "   🗑️  Deleted: $(basename "$old_file")"
    done <<< "$TO_DELETE"
else
    echo "   ✅ ${ARCHIVE_COUNT}/${MAX_BACKUPS} backups — nothing to prune"
fi

# --------------------------------------------------
# VM transfer [TIER 2] — non-fatal (identical to backup-myapps.sh)
# --------------------------------------------------
echo ""
echo "📤 [TIER 2] Sending backup to VM (${VM_HOST}:${VM_PORT})..."

scp -i "$VM_KEY" -P "$VM_PORT" \
    -o StrictHostKeyChecking=no -o ConnectTimeout=30 \
    "${BACKUP_NAME}.tar.gz" \
    "${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null
VM_SCP_RC=$?

if [ $VM_SCP_RC -eq 0 ]; then
    echo "   ✅ Backup sent to VM successfully!"
    scp -i "$VM_KEY" -P "$VM_PORT" \
        -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
        "${BACKUP_NAME}.tar.gz.sha256" \
        "${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null || true
    TIER2_STATUS="ok"

    # VM-side retention — this tier never had any pruning at all until
    # found accumulating 85 legacy pre-k8s archives (131GB) untouched
    # since May. Mirrors the local [TIER 1] policy above: same
    # MAX_BACKUPS, own glob/lineage only (never touches other
    # unrelated backup dirs also living on this VM).
    ssh -i "$VM_KEY" -p "$VM_PORT" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
        "${VM_USER}@${VM_HOST}" \
        "cd ${VM_DEST} && ls -t myapps-k8s-backup-*.tar.gz 2>/dev/null | tail -n +$((MAX_BACKUPS + 1)) | xargs -r -I{} rm -f {} {}.sha256 {}.meta.json" \
        &>/dev/null || true
else
    echo "   ⚠️  VM transfer failed (rc=$VM_SCP_RC) — local backup is safe, continuing..."
    TIER2_STATUS="failed:rc=${VM_SCP_RC}"
fi

# --------------------------------------------------
# Cloudflare R2 [TIER 3] — non-fatal (identical to backup-myapps.sh,
# key prefix stays "backups/" — shared with the legacy lineage in the
# same bucket, distinguished by filename prefix)
# --------------------------------------------------
echo ""
echo "☁️  [TIER 3] Uploading to Cloudflare R2..."

if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then
    echo "   ⚠️  R2 credentials not found — skipping cloud upload"
    TIER3_STATUS="skipped:no_credentials"
else
    echo "   Credentials: found (key=${AWS_ACCESS_KEY_ID:0:8}...)"
    echo "   Bucket:      ${R2_BUCKET}"
    echo "   Endpoint:    ${R2_ENDPOINT}"

    PYTHON_BIN="/root/management-platform/venv/bin/python3"
    if [ ! -f "$PYTHON_BIN" ]; then
        PYTHON_BIN="python3"
    fi

    R2_SCRIPT="/tmp/r2_upload_$$.py"
    cat > "$R2_SCRIPT" << PYEOF
import sys, os, boto3
from datetime import datetime, timezone

account_id  = "${R2_ACCOUNT_ID}"
bucket      = "${R2_BUCKET}"
endpoint    = "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
access_key  = "${AWS_ACCESS_KEY_ID}"
secret_key  = "${AWS_SECRET_ACCESS_KEY}"
archive     = "/root/backups/${BACKUP_NAME}.tar.gz"
sha_file    = archive + '.sha256'
key         = "backups/${BACKUP_NAME}.tar.gz"
max_r2      = ${MAX_R2_BACKUPS}
name_prefix = "backups/myapps-k8s-backup-"

try:
    client = boto3.client(
        's3',
        endpoint_url=endpoint,
        aws_access_key_id=access_key,
        aws_secret_access_key=secret_key,
        region_name='auto'
    )

    try:
        client.head_bucket(Bucket=bucket)
    except Exception:
        client.create_bucket(Bucket=bucket)
        print(f'   Created bucket: {bucket}')

    size_mb   = os.path.getsize(archive) / 1024 / 1024
    timestamp = datetime.now(timezone.utc).isoformat()
    print(f'   Uploading {size_mb:.1f} MB to r2://{bucket}/{key}')
    client.upload_file(
        archive, bucket, key,
        ExtraArgs={"Metadata": {
            "uploaded-by":   "backup-k8s-apps-script",
            "uploaded-at":   timestamp,
            "original-file": archive.split("/")[-1],
        }}
    )
    print('   ✅ R2 archive uploaded successfully')

    if os.path.exists(sha_file):
        client.upload_file(sha_file, bucket, key + '.sha256')
        print('   ✅ R2 SHA256 checksum uploaded')

    # Retention scoped to this script's own lineage (name_prefix) only —
    # never touches the legacy myapps-backup-* objects in the same bucket.
    print(f'   🧹 Enforcing R2 retention (max {max_r2}, prefix={name_prefix})...')
    resp    = client.list_objects_v2(Bucket=bucket, Prefix=name_prefix)
    objects = sorted(
        [o for o in resp.get('Contents', []) if not o['Key'].endswith('.sha256')],
        key=lambda x: x['LastModified'],
        reverse=True
    )
    to_delete = objects[max_r2:]
    if not to_delete:
        print(f'   ✅ {len(objects)}/{max_r2} R2 backups — nothing to prune')
    for obj in to_delete:
        old_key = obj['Key']
        client.delete_object(Bucket=bucket, Key=old_key)
        for suffix in ('.sha256', '.meta.json'):
            try:
                client.delete_object(Bucket=bucket, Key=old_key + suffix)
            except Exception:
                pass
        print(f'   🗑️  Deleted from R2: {old_key.replace("backups/", "")}')

    sys.exit(0)

except Exception as e:
    print(f'   ERROR: {e}')
    sys.exit(2)
PYEOF

    "$PYTHON_BIN" "$R2_SCRIPT"
    R2_RC=$?
    rm -f "$R2_SCRIPT"

    if [ $R2_RC -eq 0 ]; then
        echo "   ✅ R2 upload + retention complete"
        TIER3_STATUS="ok"
    else
        echo "   ⚠️  R2 upload failed (rc=$R2_RC) — local backup is safe at: $BACKUP_ARCHIVE"
        TIER3_STATUS="failed:rc=${R2_RC}"
    fi
fi

# --------------------------------------------------
# Status metadata sidecar — after all 3 tiers are known
# --------------------------------------------------
echo ""
echo "📝 Writing status metadata sidecar..."
write_meta_sidecar

# --------------------------------------------------
# Final summary + log
# --------------------------------------------------
echo ""
echo "========================================="
echo "✅ K8S BACKUP COMPLETE"
echo ""
echo "   📦 Name:  $BACKUP_NAME"
echo "   💾 Local: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"
echo "   🖥️  VM:    ${TIER2_STATUS}"
echo "   ☁️  R2:    ${TIER3_STATUS}"
echo "========================================="

log_status "SUCCESS" "$BACKUP_NAME" \
    "apps=${APP_COUNT} | size=${COMPRESSED_SIZE} | tier1=${TIER1_STATUS} | tier2=${TIER2_STATUS} | tier3=${TIER3_STATUS}"