diff --git a/backup/backup-k8s-apps.sh b/backup/backup-k8s-apps.sh new file mode 100755 index 0000000..7120c27 --- /dev/null +++ b/backup/backup-k8s-apps.sh @@ -0,0 +1,484 @@ +#!/bin/bash +# ============================================= +# backup-k8s-apps.sh — k8s-native replacement for backup-myapps.sh +# Backs up: n8n, Odoo, Mautic, Nextcloud, Frappe/ERPNext (all live in k3s) +# +# Per app, captures: +# - k8s manifests (Deployment/Service/ConfigMap/Ingress, via `kubectl get -o yaml`) +# - Secret manifest (plaintext for now — encryption-at-rest explicitly deferred) +# - DB dump (pg_dump / mysqldump via `kubectl exec`, never a raw PGDATA/datadir copy) +# - App-data PVC contents (via `kubectl exec ... tar`, never raw filesystem access — +# see migration-status.md §5, sandbox-filesystem-divergence rule) +# +# Storage tiers (logic reused verbatim from backup-myapps.sh, just repointed at the +# new archive name/contents): +# 1. Local → /root/backups/ (CRITICAL — always runs) +# 2. VM → SSH tunnel → /backups/cloudproject/ +# 3. Cloud → Cloudflare R2 (S3-compatible, optional) +# +# NOTE: R2 and VM failures do NOT stop the backup. Local backup always completes first. +# +# Archive name is deliberately NOT "myapps-backup-*" (the legacy Docker-era pattern +# still produced nightly by /root/backup-myapps.sh via cron) — using a distinct +# "myapps-k8s-backup-*" prefix keeps the two backup lineages from mixing in the same +# retention pool / R2 prefix until the legacy cron job is explicitly retired. +# ============================================= + +set -uo pipefail + +# -------------------------------------------------- +# Parse --apps +# -------------------------------------------------- +APPS_FILTER="all" # "all" or comma-separated: frappe,odoo,nextcloud,mautic,n8n +VALID_APPS="frappe odoo nextcloud mautic n8n" + +while [[ $# -gt 0 ]]; do + case "$1" in + --apps) + APPS_FILTER="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +if [ "$APPS_FILTER" != "all" ]; then + APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER" + for _a in "${_APP_CHECK[@]}"; do + if [[ " $VALID_APPS " != *" $_a "* ]]; then + echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS" + exit 1 + fi + done +fi + +app_selected() { + local app="$1" + [ "$APPS_FILTER" = "all" ] && return 0 + [[ ",$APPS_FILTER," == *",${app},"* ]] +} + +# -------------------------------------------------- +# Per-app metadata table +# app-key : namespace : app-deployment : db-deployment : db-engine : db-name(or DYNAMIC) : pvc-mount-path : container-name(for exec) : secret-name : db-user-key : db-pass-key +# -------------------------------------------------- +declare -A NS=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext ) +declare -A APP_DEPLOY=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web ) +declare -A APP_CTR=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web ) +declare -A DB_DEPLOY=( [n8n]=n8n-postgres [odoo]=odoo-postgres [mautic]=mautic-mariadb [nextcloud]=nextcloud-postgres [frappe]=erpnext-mariadb ) +declare -A DB_ENGINE=( [n8n]=postgres [odoo]=postgres [mautic]=mariadb [nextcloud]=postgres [frappe]=mariadb ) +declare -A DB_NAME=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=DYNAMIC ) +declare -A PVC_PATH=( [n8n]="/home/node/.n8n" [odoo]="/var/lib/odoo" [mautic]="/var/www/html" [nextcloud]="/var/www/html" [frappe]="/home/frappe/frappe-bench/sites" ) +declare -A SECRET_NAME=( [n8n]=n8n-secrets [odoo]=odoo-secrets [mautic]=mautic-secrets [nextcloud]=nextcloud-secrets [frappe]=erpnext-secrets ) +declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" ) +declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD ) + +ALL_APPS="n8n odoo mautic nextcloud frappe" + +BACKUP_DATE=$(date +%Y%m%d_%H%M%S) +BACKUP_NAME="myapps-k8s-backup-${BACKUP_DATE}" +BACKUP_DIR="/root/backups/${BACKUP_NAME}" +BACKUP_ARCHIVE="/root/backups/${BACKUP_NAME}.tar.gz" + +VM_USER="root" +VM_HOST="178.18.243.51" +VM_PORT="22" +VM_KEY="/root/.ssh/id_rsa" +VM_DEST="/backups/cloudproject/" + +# ── Load R2 credentials (identical to backup-myapps.sh) ──────────────────────── +CREDENTIALS_FILE="/root/.r2-credentials" +if [ -f "$CREDENTIALS_FILE" ]; then + set +u + source "$CREDENTIALS_FILE" + set -u +fi + +if [ -z "${AWS_ACCESS_KEY_ID:-}" ]; then + CONFIG_PY="/root/management-platform/config.py" + if [ -f "$CONFIG_PY" ]; then + _KEY=$(grep -oP '(?<=R2_ACCESS_KEY_ID",\s{5}")[^"]+' "$CONFIG_PY" 2>/dev/null || true) + _SEC=$(grep -oP '(?<=R2_SECRET_ACCESS_KEY", ")[^"]+' "$CONFIG_PY" 2>/dev/null || true) + _BKT=$(grep -oP '(?<=R2_BUCKET_NAME",\s{7}")[^"]+' "$CONFIG_PY" 2>/dev/null || true) + [ -n "$_KEY" ] && export AWS_ACCESS_KEY_ID="$_KEY" + [ -n "$_SEC" ] && export AWS_SECRET_ACCESS_KEY="$_SEC" + [ -n "$_BKT" ] && export R2_BUCKET_NAME="$_BKT" + fi +fi + +R2_ACCOUNT_ID="${R2_ACCOUNT_ID:-35e00c230cc8066252a2d9890b69aea2}" +R2_BUCKET="${R2_BUCKET_NAME:-navitrends-backups}" +R2_ENDPOINT="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" + +BACKUP_LOG_FILE="/root/backups/k8s-backup-status.log" +MAX_BACKUPS=10 +MAX_R2_BACKUPS=5 + +TIER1_STATUS="pending" +TIER2_STATUS="pending" +TIER3_STATUS="pending" + +log_status() { + local status="$1" name="$2" msg="${3:-}" + echo "$(date '+%Y-%m-%d %H:%M:%S') | ${status} | ${name} | ${msg}" >> "$BACKUP_LOG_FILE" +} + +# -------------------------------------------------- +# Resource-safety gate (migration-status.md §5): abort before starting +# any heavy per-app work if we're already below threshold. +# -------------------------------------------------- +check_resources() { + local avail_mb avail_gb + avail_mb=$(free -m | awk '/^Mem:/{print $7}') + avail_gb=$(df -BG / | awk 'NR==2{gsub("G","",$4); print $4}') + if [ "$avail_mb" -lt 1536 ]; then + echo " ❌ RAM available ${avail_mb}MB < 1536MB threshold — aborting." + log_status "FAILED" "$BACKUP_NAME" "resource_safety_ram_${avail_mb}mb" + exit 1 + fi + if [ "$avail_gb" -lt 10 ]; then + echo " ❌ Disk available ${avail_gb}GB < 10GB threshold — aborting." + log_status "FAILED" "$BACKUP_NAME" "resource_safety_disk_${avail_gb}gb" + exit 1 + fi +} + +echo "=========================================" +echo "📦 Starting k8s Backup: $BACKUP_NAME" +if [ "$APPS_FILTER" = "all" ]; then + echo " Apps: ALL ($ALL_APPS)" +else + echo " Apps: $APPS_FILTER" +fi +echo " Tiers: Local → VM → ☁ Cloudflare R2" +echo "=========================================" + +check_resources +mkdir -p "$BACKUP_DIR" "/root/backups" +cd "$BACKUP_DIR" + +APP_COUNT=0 +for app in $ALL_APPS; do + app_selected "$app" || continue + ns="${NS[$app]}" + app_deploy="${APP_DEPLOY[$app]}" + app_ctr="${APP_CTR[$app]}" + db_deploy="${DB_DEPLOY[$app]}" + db_engine="${DB_ENGINE[$app]}" + db_name="${DB_NAME[$app]}" + pvc_path="${PVC_PATH[$app]}" + secret_name="${SECRET_NAME[$app]}" + db_user_key="${DB_USER_KEY[$app]}" + db_pass_key="${DB_PASS_KEY[$app]}" + + echo "" + echo "-----------------------------------------" + echo "📁 [$app] namespace=$ns" + echo "-----------------------------------------" + + APP_DIR="$BACKUP_DIR/$app" + mkdir -p "$APP_DIR" + + # ---- 1. Manifests ---- + echo -n " 📄 Manifests (deployment/service/configmap/ingress) ... " + kubectl get deployment,service,configmap,ingress -n "$ns" -l owner=ameni-boukattaya -o yaml \ + > "$APP_DIR/manifests.yaml" 2>"$APP_DIR/manifests.err" \ + && { echo "✅"; rm -f "$APP_DIR/manifests.err"; } || echo "⚠️ FAILED (see manifests.err)" + + # ---- 2. Secret (plaintext, encryption-at-rest deferred) ---- + echo -n " 🔑 Secret ($secret_name) ... " + kubectl get secret "$secret_name" -n "$ns" -o yaml \ + > "$APP_DIR/secret.yaml" 2>"$APP_DIR/secret.err" \ + && { echo "✅"; rm -f "$APP_DIR/secret.err"; } || echo "⚠️ FAILED (see secret.err)" + + # ---- 3. DB dump via kubectl exec (never raw datadir copy) ---- + echo -n " 💾 DB dump ($db_engine) ... " + case "$db_engine" in + postgres) + db_user=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_user_key}}" 2>/dev/null | base64 -d) + db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d) + if [ -n "$db_user" ] && [ -n "$db_pass" ]; then + kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \ + pg_dump -U "$db_user" -d "$db_name" 2>"$APP_DIR/db-dump.err" \ + | gzip > "$APP_DIR/db-dump.sql.gz" + pg_dump_rc=${PIPESTATUS[0]} + if [ "$pg_dump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then + echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1))" + rm -f "$APP_DIR/db-dump.err" + else + echo "⚠️ FAILED (rc=$pg_dump_rc, see db-dump.err)" + fi + else + echo "⚠️ could not read DB credentials from secret" + fi + ;; + mariadb) + if [ "$db_name" = "DYNAMIC" ]; then + # Frappe: real db_name/db_password live in site_config.json on the + # sites PVC, not in the k8s Secret. Dump with root instead, but still + # need the site's real db_name to target the right database. + site_dir=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \ + sh -c 'for d in sites/*/; do [ -f "$d/site_config.json" ] && [ ! -L "${d%/}" ] && basename "$d" && break; done' 2>/dev/null) + db_name=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \ + cat "sites/${site_dir}/site_config.json" 2>/dev/null \ + | python3 -c 'import json,sys; print(json.load(sys.stdin)["db_name"])' 2>/dev/null) + fi + db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d) + if [ -n "$db_name" ] && [ -n "$db_pass" ]; then + kubectl exec -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \ + mysqldump -uroot --databases "$db_name" 2>"$APP_DIR/db-dump.err" \ + | gzip > "$APP_DIR/db-dump.sql.gz" + mysqldump_rc=${PIPESTATUS[0]} + if [ "$mysqldump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then + echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1), db=$db_name)" + echo "$db_name" > "$APP_DIR/db-name.txt" + rm -f "$APP_DIR/db-dump.err" + else + echo "⚠️ FAILED (rc=$mysqldump_rc, see db-dump.err)" + fi + else + echo "⚠️ could not determine DB name/credentials" + fi + ;; + esac + + # ---- 4. App-data PVC contents via kubectl exec (live tar, app-data only, + # never raw filesystem/PVC-path access — §5 sandbox-divergence rule) ---- + echo -n " 📦 PVC data ($pvc_path) ... " + kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \ + tar czf - -C "$pvc_path" . 2>"$APP_DIR/pvc-data.err" \ + > "$APP_DIR/pvc-data.tar.gz" + if [ -s "$APP_DIR/pvc-data.tar.gz" ]; then + echo "✅ ($(du -h "$APP_DIR/pvc-data.tar.gz" | cut -f1))" + rm -f "$APP_DIR/pvc-data.err" + else + echo "⚠️ FAILED (see pvc-data.err)" + fi + + APP_COUNT=$((APP_COUNT + 1)) + + # Resource check between apps, not just at the start (§5: check every + # 1-2 min during long-running/heavy ops; PVC tars can be sizeable). + check_resources +done + +# -------------------------------------------------- +# Metadata + checksum +# -------------------------------------------------- +echo "" +echo "📝 Writing backup metadata..." +cat > "$BACKUP_DIR/backup-info.txt" << EOF +Backup Name: $BACKUP_NAME +Backup Date: $(date) +Hostname: $(hostname) +Apps: $([ "$APPS_FILTER" = "all" ] && echo "$ALL_APPS" || echo "$APPS_FILTER") +Apps backed up: $APP_COUNT +kubectl: $(kubectl version --client -o yaml 2>/dev/null | grep gitVersion | head -1 || echo 'N/A') +Storage Tiers: + - Local: /root/backups/ + - VM: ${VM_HOST}:${VM_PORT} → ${VM_DEST} + - Cloud: Cloudflare R2 → s3://${R2_BUCKET}/backups/ +EOF + +echo "" >> "$BACKUP_DIR/backup-info.txt" +echo "Per-app checksums:" >> "$BACKUP_DIR/backup-info.txt" +find "$BACKUP_DIR" -type f \( -name "*.tar.gz" -o -name "*.sql.gz" \) -print0 \ + | xargs -0 -r sha256sum >> "$BACKUP_DIR/backup-info.txt" +echo " ✅ Done" + +# -------------------------------------------------- +# Compress (identical logic to backup-myapps.sh) +# -------------------------------------------------- +echo "" +echo "🗜️ Compressing backup..." +cd /root/backups + +tar -czf "${BACKUP_NAME}.tar.gz" "${BACKUP_NAME}/" 2>/dev/null +if [ $? -ne 0 ]; then + echo " ❌ CRITICAL: Compression failed!" + log_status "FAILED" "$BACKUP_NAME" "compression_failed" + exit 1 +fi + +COMPRESSED_SIZE=$(du -h "${BACKUP_NAME}.tar.gz" | cut -f1) +echo " ✅ Compressed size: $COMPRESSED_SIZE → $BACKUP_ARCHIVE" + +sha256sum "${BACKUP_NAME}.tar.gz" > "${BACKUP_NAME}.tar.gz.sha256" +echo " ✅ Checksum written" + +rm -rf "$BACKUP_DIR" +TIER1_STATUS="ok:${COMPRESSED_SIZE}" +echo "" +echo "✅ [TIER 1] Local backup complete: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)" + +# -------------------------------------------------- +# Retention — keep only MAX_BACKUPS locally (own glob, own lineage) +# -------------------------------------------------- +echo "" +echo "🧹 [Retention] Keeping latest ${MAX_BACKUPS} local k8s backups..." +ARCHIVE_LIST=$(ls -t /root/backups/myapps-k8s-backup-*.tar.gz 2>/dev/null || true) +ARCHIVE_COUNT=$(echo "$ARCHIVE_LIST" | grep -c '.tar.gz' 2>/dev/null || echo "0") + +if [ "$ARCHIVE_COUNT" -gt "$MAX_BACKUPS" ]; then + TO_DELETE=$(echo "$ARCHIVE_LIST" | tail -n +$((MAX_BACKUPS + 1))) + while IFS= read -r old_file; do + [ -z "$old_file" ] && continue + rm -f "$old_file" "${old_file}.sha256" + echo " 🗑️ Deleted: $(basename "$old_file")" + done <<< "$TO_DELETE" +else + echo " ✅ ${ARCHIVE_COUNT}/${MAX_BACKUPS} backups — nothing to prune" +fi + +# -------------------------------------------------- +# VM transfer [TIER 2] — non-fatal (identical to backup-myapps.sh) +# -------------------------------------------------- +echo "" +echo "📤 [TIER 2] Sending backup to VM (${VM_HOST}:${VM_PORT})..." + +scp -i "$VM_KEY" -P "$VM_PORT" \ + -o StrictHostKeyChecking=no -o ConnectTimeout=30 \ + "${BACKUP_NAME}.tar.gz" \ + "${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null +VM_SCP_RC=$? + +if [ $VM_SCP_RC -eq 0 ]; then + echo " ✅ Backup sent to VM successfully!" + scp -i "$VM_KEY" -P "$VM_PORT" \ + -o StrictHostKeyChecking=no -o ConnectTimeout=15 \ + "${BACKUP_NAME}.tar.gz.sha256" \ + "${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null || true + TIER2_STATUS="ok" +else + echo " ⚠️ VM transfer failed (rc=$VM_SCP_RC) — local backup is safe, continuing..." + TIER2_STATUS="failed:rc=${VM_SCP_RC}" +fi + +# -------------------------------------------------- +# Cloudflare R2 [TIER 3] — non-fatal (identical to backup-myapps.sh, +# key prefix stays "backups/" — shared with the legacy lineage in the +# same bucket, distinguished by filename prefix) +# -------------------------------------------------- +echo "" +echo "☁️ [TIER 3] Uploading to Cloudflare R2..." + +if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then + echo " ⚠️ R2 credentials not found — skipping cloud upload" + TIER3_STATUS="skipped:no_credentials" +else + echo " Credentials: found (key=${AWS_ACCESS_KEY_ID:0:8}...)" + echo " Bucket: ${R2_BUCKET}" + echo " Endpoint: ${R2_ENDPOINT}" + + PYTHON_BIN="/root/management-platform/venv/bin/python3" + if [ ! -f "$PYTHON_BIN" ]; then + PYTHON_BIN="python3" + fi + + R2_SCRIPT="/tmp/r2_upload_$$.py" + cat > "$R2_SCRIPT" << PYEOF +import sys, os, boto3 +from datetime import datetime, timezone + +account_id = "${R2_ACCOUNT_ID}" +bucket = "${R2_BUCKET}" +endpoint = "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com" +access_key = "${AWS_ACCESS_KEY_ID}" +secret_key = "${AWS_SECRET_ACCESS_KEY}" +archive = "/root/backups/${BACKUP_NAME}.tar.gz" +sha_file = archive + '.sha256' +key = "backups/${BACKUP_NAME}.tar.gz" +max_r2 = ${MAX_R2_BACKUPS} +name_prefix = "backups/myapps-k8s-backup-" + +try: + client = boto3.client( + 's3', + endpoint_url=endpoint, + aws_access_key_id=access_key, + aws_secret_access_key=secret_key, + region_name='auto' + ) + + try: + client.head_bucket(Bucket=bucket) + except Exception: + client.create_bucket(Bucket=bucket) + print(f' Created bucket: {bucket}') + + size_mb = os.path.getsize(archive) / 1024 / 1024 + timestamp = datetime.now(timezone.utc).isoformat() + print(f' Uploading {size_mb:.1f} MB to r2://{bucket}/{key}') + client.upload_file( + archive, bucket, key, + ExtraArgs={"Metadata": { + "uploaded-by": "backup-k8s-apps-script", + "uploaded-at": timestamp, + "original-file": archive.split("/")[-1], + }} + ) + print(' ✅ R2 archive uploaded successfully') + + if os.path.exists(sha_file): + client.upload_file(sha_file, bucket, key + '.sha256') + print(' ✅ R2 SHA256 checksum uploaded') + + # Retention scoped to this script's own lineage (name_prefix) only — + # never touches the legacy myapps-backup-* objects in the same bucket. + print(f' 🧹 Enforcing R2 retention (max {max_r2}, prefix={name_prefix})...') + resp = client.list_objects_v2(Bucket=bucket, Prefix=name_prefix) + objects = sorted( + [o for o in resp.get('Contents', []) if not o['Key'].endswith('.sha256')], + key=lambda x: x['LastModified'], + reverse=True + ) + to_delete = objects[max_r2:] + if not to_delete: + print(f' ✅ {len(objects)}/{max_r2} R2 backups — nothing to prune') + for obj in to_delete: + old_key = obj['Key'] + client.delete_object(Bucket=bucket, Key=old_key) + try: + client.delete_object(Bucket=bucket, Key=old_key + '.sha256') + except Exception: + pass + print(f' 🗑️ Deleted from R2: {old_key.replace("backups/", "")}') + + sys.exit(0) + +except Exception as e: + print(f' ERROR: {e}') + sys.exit(2) +PYEOF + + "$PYTHON_BIN" "$R2_SCRIPT" + R2_RC=$? + rm -f "$R2_SCRIPT" + + if [ $R2_RC -eq 0 ]; then + echo " ✅ R2 upload + retention complete" + TIER3_STATUS="ok" + else + echo " ⚠️ R2 upload failed (rc=$R2_RC) — local backup is safe at: $BACKUP_ARCHIVE" + TIER3_STATUS="failed:rc=${R2_RC}" + fi +fi + +# -------------------------------------------------- +# Final summary + log +# -------------------------------------------------- +echo "" +echo "=========================================" +echo "✅ K8S BACKUP COMPLETE" +echo "" +echo " 📦 Name: $BACKUP_NAME" +echo " 💾 Local: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)" +echo " 🖥️ VM: ${TIER2_STATUS}" +echo " ☁️ R2: ${TIER3_STATUS}" +echo "=========================================" + +log_status "SUCCESS" "$BACKUP_NAME" \ + "apps=${APP_COUNT} | size=${COMPRESSED_SIZE} | tier1=${TIER1_STATUS} | tier2=${TIER2_STATUS} | tier3=${TIER3_STATUS}" diff --git a/backup/restore-k8s-apps.sh b/backup/restore-k8s-apps.sh new file mode 100755 index 0000000..33039a4 --- /dev/null +++ b/backup/restore-k8s-apps.sh @@ -0,0 +1,269 @@ +#!/bin/bash +# ============================================= +# restore-k8s-apps.sh — k8s-native replacement for restore-myapps.sh +# Restores: n8n, Odoo, Mautic, Nextcloud, Frappe/ERPNext (all live in k3s) +# +# Contract mirrors restore-myapps.sh exactly, so wiring into app.py's +# restore_start route is a pure path swap: +# - Expects to be run from inside an already-extracted backup directory +# (SCRIPT_DIR = the script's own directory), i.e. app.py's existing +# "tar -xzf backup --strip-components=1 && cp restore-*.sh && +# cd && bash restore-*.sh --apps X" flow works unchanged — +# only the two script filenames/paths need to change on the Python side. +# - Same --apps flag: comma-separated subset of frappe,odoo,nextcloud,mautic,n8n +# (omit = restore all found in the backup). +# +# Restore order per app (migration-status.md §5 — never populate a PVC +# that's still mounted by a running consumer; always populate-then-deploy): +# 1. kubectl apply the Secret + manifests (idempotent — creates if missing, +# updates in place if present; this is also the DR-onto-fresh-cluster path +# for anything that isn't a PVC, which pre-exists there already this session) +# 2. Scale the app Deployment to 0 and wait for the pod to terminate +# 3. Populate the app-data PVC via a short-lived loader pod + kubectl exec +# (never raw filesystem/PVC-path access) +# 4. Scale the app Deployment back up +# 5. Restore the DB via pg_restore/mysql (SQL-level, never a raw datadir copy) +# against the DB Deployment, which stays running throughout +# +# NOT included this session (explicitly deferred, see backup-migration-plan.md §5): +# - Cross-cluster / disaster-recovery restore. This script assumes the target +# cluster/namespaces/PVCs already exist (same-cluster restore-in-place). +# A fresh-cluster bootstrap + PVC-from-scratch path is future work. +# ============================================= + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +APPS_FILTER="all" +VALID_APPS="frappe odoo nextcloud mautic n8n" + +while [[ $# -gt 0 ]]; do + case "$1" in + --apps) + APPS_FILTER="$2" + shift 2 + ;; + *) + shift + ;; + esac +done + +if [ "$APPS_FILTER" != "all" ]; then + APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') + IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER" + for _a in "${_APP_CHECK[@]}"; do + if [[ " $VALID_APPS " != *" $_a "* ]]; then + echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS" + exit 1 + fi + done +fi + +app_selected() { + local app="$1" + [ "$APPS_FILTER" = "all" ] && return 0 + [[ ",$APPS_FILTER," == *",${app},"* ]] +} + +# -------------------------------------------------- +# Per-app metadata table (same shape as backup-k8s-apps.sh) +# -------------------------------------------------- +declare -A NS=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext ) +declare -A APP_DEPLOY=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web ) +declare -A DB_DEPLOY=( [n8n]=n8n-postgres [odoo]=odoo-postgres [mautic]=mautic-mariadb [nextcloud]=nextcloud-postgres [frappe]=erpnext-mariadb ) +declare -A DB_ENGINE=( [n8n]=postgres [odoo]=postgres [mautic]=mariadb [nextcloud]=postgres [frappe]=mariadb ) +declare -A DB_NAME=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=DYNAMIC ) +declare -A PVC_NAME=( [n8n]=n8n-data [odoo]=odoo-data [mautic]=mautic-app-data [nextcloud]=nextcloud-app-data [frappe]=erpnext-sites ) +declare -A PVC_PATH=( [n8n]="/home/node/.n8n" [odoo]="/var/lib/odoo" [mautic]="/var/www/html" [nextcloud]="/var/www/html" [frappe]="/home/frappe/frappe-bench/sites" ) +declare -A SECRET_NAME=( [n8n]=n8n-secrets [odoo]=odoo-secrets [mautic]=mautic-secrets [nextcloud]=nextcloud-secrets [frappe]=erpnext-secrets ) +declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" ) +declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD ) +declare -A DOMAIN=( [n8n]="n8nwf.nav.ovh" [odoo]="odooo.nav.ovh" [mautic]="mautics.nav.ovh" [nextcloud]="next.cloud.nav.ovh" [frappe]="erpnext.navitrends.ovh" ) + +ALL_APPS="n8n odoo mautic nextcloud frappe" +LOADER_IMAGE="alpine:3.20" + +echo "=========================================" +echo "🔄 k8s Restore — SAME-CLUSTER MODE" +echo " Backup dir: $SCRIPT_DIR" +if [ "$APPS_FILTER" = "all" ]; then + echo " Apps: ALL (found in backup)" +else + echo " Apps: $APPS_FILTER" +fi +echo "=========================================" + +check_resources() { + local avail_mb avail_gb + avail_mb=$(free -m | awk '/^Mem:/{print $7}') + avail_gb=$(df -BG / | awk 'NR==2{gsub("G","",$4); print $4}') + if [ "$avail_mb" -lt 1536 ] || [ "$avail_gb" -lt 10 ]; then + echo " ❌ Resource safety threshold hit (RAM=${avail_mb}MB, Disk=${avail_gb}GB) — aborting." + exit 1 + fi +} + +RESTORED_APPS=() +FAILED_APPS=() + +check_resources + +for app in $ALL_APPS; do + app_selected "$app" || continue + + if [ ! -d "$SCRIPT_DIR/$app" ]; then + echo "" + echo " ⏭️ $app — not present in this backup, skipping" + continue + fi + + ns="${NS[$app]}" + app_deploy="${APP_DEPLOY[$app]}" + db_deploy="${DB_DEPLOY[$app]}" + db_engine="${DB_ENGINE[$app]}" + db_name="${DB_NAME[$app]}" + pvc_name="${PVC_NAME[$app]}" + pvc_path="${PVC_PATH[$app]}" + secret_name="${SECRET_NAME[$app]}" + db_user_key="${DB_USER_KEY[$app]}" + db_pass_key="${DB_PASS_KEY[$app]}" + APP_DIR="$SCRIPT_DIR/$app" + + echo "" + echo "-----------------------------------------" + echo "🔧 [$app] namespace=$ns" + echo "-----------------------------------------" + + app_ok=true + + # ---- 1. Secret + manifests (apply — idempotent) ---- + if [ -f "$APP_DIR/secret.yaml" ]; then + echo -n " 🔑 Applying Secret ... " + kubectl apply -f "$APP_DIR/secret.yaml" &>/dev/null && echo "✅" || { echo "⚠️ FAILED"; app_ok=false; } + fi + if [ -f "$APP_DIR/manifests.yaml" ]; then + echo -n " 📄 Applying manifests ... " + kubectl apply -f "$APP_DIR/manifests.yaml" &>/dev/null && echo "✅" || { echo "⚠️ FAILED"; app_ok=false; } + fi + + # ---- 2. Scale app down ---- + prior_replicas=$(kubectl get deployment "$app_deploy" -n "$ns" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 1) + [ -z "$prior_replicas" ] && prior_replicas=1 + echo -n " ⏸️ Scaling $app_deploy to 0 ... " + kubectl scale deployment "$app_deploy" -n "$ns" --replicas=0 &>/dev/null \ + && kubectl wait --for=delete pod -l app="$app" -n "$ns" --timeout=90s &>/dev/null + echo "✅" + + # ---- 3. Restore PVC data via loader pod ---- + if [ -f "$APP_DIR/pvc-data.tar.gz" ]; then + if kubectl get pvc "$pvc_name" -n "$ns" &>/dev/null; then + loader="${app}-restore-loader" + echo -n " 📦 Populating PVC $pvc_name via loader pod ... " + kubectl run "$loader" -n "$ns" --image="$LOADER_IMAGE" --restart=Never \ + --overrides="{\"spec\":{\"containers\":[{\"name\":\"loader\",\"image\":\"${LOADER_IMAGE}\",\"command\":[\"sleep\",\"3600\"],\"volumeMounts\":[{\"name\":\"data\",\"mountPath\":\"/restore\"}]}],\"volumes\":[{\"name\":\"data\",\"persistentVolumeClaim\":{\"claimName\":\"${pvc_name}\"}}]}}" \ + &>/dev/null + kubectl wait --for=condition=Ready "pod/$loader" -n "$ns" --timeout=60s &>/dev/null + + kubectl exec -n "$ns" "$loader" -- sh -c 'rm -rf /restore/* /restore/.[!.]* /restore/..?* 2>/dev/null; true' &>/dev/null + if kubectl exec -i -n "$ns" "$loader" -- tar xzf - -C /restore < "$APP_DIR/pvc-data.tar.gz" &>"$APP_DIR/restore-pvc.err"; then + echo "✅" + rm -f "$APP_DIR/restore-pvc.err" + else + echo "⚠️ FAILED (see $APP_DIR/restore-pvc.err)" + app_ok=false + fi + kubectl delete pod "$loader" -n "$ns" --wait=false &>/dev/null + else + echo " ⚠️ PVC '$pvc_name' does not exist in this cluster — skipping PVC restore" + echo " (fresh-cluster/DR provisioning of PVCs from manifests is not yet implemented)" + app_ok=false + fi + else + echo " ⏭️ No pvc-data.tar.gz in backup — skipping PVC restore" + fi + + # ---- 4. Scale app back up ---- + echo -n " ▶️ Scaling $app_deploy back to $prior_replicas ... " + kubectl scale deployment "$app_deploy" -n "$ns" --replicas="$prior_replicas" &>/dev/null && echo "✅" || echo "⚠️ FAILED" + + # ---- 5. DB restore (SQL-level, never raw datadir copy) ---- + if [ -f "$APP_DIR/db-dump.sql.gz" ]; then + echo -n " 💾 Restoring DB ($db_engine) ... " + case "$db_engine" in + postgres) + db_user=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_user_key}}" 2>/dev/null | base64 -d) + db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d) + if [ -n "$db_user" ] && [ -n "$db_pass" ]; then + kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \ + psql -U "$db_user" -d postgres -c "DROP DATABASE IF EXISTS \"$db_name\";" &>/dev/null + kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \ + createdb -U "$db_user" "$db_name" &>/dev/null + if gunzip -c "$APP_DIR/db-dump.sql.gz" | \ + kubectl exec -i -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \ + psql -U "$db_user" -d "$db_name" &>"$APP_DIR/restore-db.err"; then + echo "✅" + rm -f "$APP_DIR/restore-db.err" + else + echo "⚠️ FAILED (see $APP_DIR/restore-db.err)" + app_ok=false + fi + else + echo "⚠️ could not read DB credentials from (restored) secret" + app_ok=false + fi + ;; + mariadb) + if [ "$db_name" = "DYNAMIC" ]; then + db_name=$(cat "$APP_DIR/db-name.txt" 2>/dev/null) + fi + db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d) + if [ -n "$db_name" ] && [ -n "$db_pass" ]; then + kubectl exec -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \ + mysql -uroot -e "DROP DATABASE IF EXISTS \`$db_name\`;" &>/dev/null + if gunzip -c "$APP_DIR/db-dump.sql.gz" | \ + kubectl exec -i -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \ + mysql -uroot &>"$APP_DIR/restore-db.err"; then + echo "✅ (db=$db_name)" + rm -f "$APP_DIR/restore-db.err" + else + echo "⚠️ FAILED (see $APP_DIR/restore-db.err)" + app_ok=false + fi + else + echo "⚠️ could not determine DB name/credentials" + app_ok=false + fi + ;; + esac + else + echo " ⏭️ No db-dump.sql.gz in backup — skipping DB restore" + fi + + if [ "$app_ok" = true ]; then + RESTORED_APPS+=("$app") + else + FAILED_APPS+=("$app") + fi + + check_resources +done + +# -------------------------------------------------- +# Summary +# -------------------------------------------------- +echo "" +echo "=========================================" +echo "✅ RESTORE COMPLETE" +echo "=========================================" +for app in "${RESTORED_APPS[@]:-}"; do + [ -z "$app" ] && continue + echo " ${app} → https://${DOMAIN[$app]}" +done +if [ "${#FAILED_APPS[@]}" -gt 0 ]; then + echo "" + echo " ⚠️ Had failures: ${FAILED_APPS[*]} — check the .err files under each app's backup dir" +fi +echo "========================================="