Add k8s-native backup/restore scripts for the 5 live apps
backup-myapps.sh/restore-myapps.sh only ever knew about Docker volumes, which no longer exist for n8n/odoo/mautic/nextcloud/frappe now that they're all running in k3s — nightly backups have been silently hollow for these apps since the migration. These new scripts capture/restore each app's k8s manifests, Secret, DB (pg_dump/mysqldump via kubectl exec), and app-data PVC contents (kubectl exec tar, never raw filesystem access), reusing the existing local+VM+R2 3-2-1 storage/retention logic unchanged. Tested standalone: full backup of all 5 apps produces verified non-empty manifests/secrets/DB dumps/PVC data; restore-in-place tested end-to-end against n8n (DB row counts and PVC data matched exactly pre/post, app verified healthy over HTTPS). Not wired into management-platform yet (app.py stays untouched) — restore script also doesn't yet handle a fresh/DR cluster where the PVCs and their manifests don't already exist (same-cluster restore only for now). Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
This commit is contained in:
root
484
backup/backup-k8s-apps.sh
Executable file
484
backup/backup-k8s-apps.sh
Executable file
@@ -0,0 +1,484 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# backup-k8s-apps.sh — k8s-native replacement for backup-myapps.sh
|
||||
# Backs up: n8n, Odoo, Mautic, Nextcloud, Frappe/ERPNext (all live in k3s)
|
||||
#
|
||||
# Per app, captures:
|
||||
# - k8s manifests (Deployment/Service/ConfigMap/Ingress, via `kubectl get -o yaml`)
|
||||
# - Secret manifest (plaintext for now — encryption-at-rest explicitly deferred)
|
||||
# - DB dump (pg_dump / mysqldump via `kubectl exec`, never a raw PGDATA/datadir copy)
|
||||
# - App-data PVC contents (via `kubectl exec ... tar`, never raw filesystem access —
|
||||
# see migration-status.md §5, sandbox-filesystem-divergence rule)
|
||||
#
|
||||
# Storage tiers (logic reused verbatim from backup-myapps.sh, just repointed at the
|
||||
# new archive name/contents):
|
||||
# 1. Local → /root/backups/ (CRITICAL — always runs)
|
||||
# 2. VM → SSH tunnel → /backups/cloudproject/
|
||||
# 3. Cloud → Cloudflare R2 (S3-compatible, optional)
|
||||
#
|
||||
# NOTE: R2 and VM failures do NOT stop the backup. Local backup always completes first.
|
||||
#
|
||||
# Archive name is deliberately NOT "myapps-backup-*" (the legacy Docker-era pattern
|
||||
# still produced nightly by /root/backup-myapps.sh via cron) — using a distinct
|
||||
# "myapps-k8s-backup-*" prefix keeps the two backup lineages from mixing in the same
|
||||
# retention pool / R2 prefix until the legacy cron job is explicitly retired.
|
||||
# =============================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
# --------------------------------------------------
|
||||
# Parse --apps
|
||||
# --------------------------------------------------
|
||||
APPS_FILTER="all" # "all" or comma-separated: frappe,odoo,nextcloud,mautic,n8n
|
||||
VALID_APPS="frappe odoo nextcloud mautic n8n"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--apps)
|
||||
APPS_FILTER="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$APPS_FILTER" != "all" ]; then
|
||||
APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
|
||||
IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER"
|
||||
for _a in "${_APP_CHECK[@]}"; do
|
||||
if [[ " $VALID_APPS " != *" $_a "* ]]; then
|
||||
echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
app_selected() {
|
||||
local app="$1"
|
||||
[ "$APPS_FILTER" = "all" ] && return 0
|
||||
[[ ",$APPS_FILTER," == *",${app},"* ]]
|
||||
}
|
||||
|
||||
# --------------------------------------------------
|
||||
# Per-app metadata table
|
||||
# app-key : namespace : app-deployment : db-deployment : db-engine : db-name(or DYNAMIC) : pvc-mount-path : container-name(for exec) : secret-name : db-user-key : db-pass-key
|
||||
# --------------------------------------------------
|
||||
declare -A NS=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext )
|
||||
declare -A APP_DEPLOY=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web )
|
||||
declare -A APP_CTR=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web )
|
||||
declare -A DB_DEPLOY=( [n8n]=n8n-postgres [odoo]=odoo-postgres [mautic]=mautic-mariadb [nextcloud]=nextcloud-postgres [frappe]=erpnext-mariadb )
|
||||
declare -A DB_ENGINE=( [n8n]=postgres [odoo]=postgres [mautic]=mariadb [nextcloud]=postgres [frappe]=mariadb )
|
||||
declare -A DB_NAME=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=DYNAMIC )
|
||||
declare -A PVC_PATH=( [n8n]="/home/node/.n8n" [odoo]="/var/lib/odoo" [mautic]="/var/www/html" [nextcloud]="/var/www/html" [frappe]="/home/frappe/frappe-bench/sites" )
|
||||
declare -A SECRET_NAME=( [n8n]=n8n-secrets [odoo]=odoo-secrets [mautic]=mautic-secrets [nextcloud]=nextcloud-secrets [frappe]=erpnext-secrets )
|
||||
declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" )
|
||||
declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD )
|
||||
|
||||
ALL_APPS="n8n odoo mautic nextcloud frappe"
|
||||
|
||||
BACKUP_DATE=$(date +%Y%m%d_%H%M%S)
|
||||
BACKUP_NAME="myapps-k8s-backup-${BACKUP_DATE}"
|
||||
BACKUP_DIR="/root/backups/${BACKUP_NAME}"
|
||||
BACKUP_ARCHIVE="/root/backups/${BACKUP_NAME}.tar.gz"
|
||||
|
||||
VM_USER="root"
|
||||
VM_HOST="178.18.243.51"
|
||||
VM_PORT="22"
|
||||
VM_KEY="/root/.ssh/id_rsa"
|
||||
VM_DEST="/backups/cloudproject/"
|
||||
|
||||
# ── Load R2 credentials (identical to backup-myapps.sh) ────────────────────────
|
||||
CREDENTIALS_FILE="/root/.r2-credentials"
|
||||
if [ -f "$CREDENTIALS_FILE" ]; then
|
||||
set +u
|
||||
source "$CREDENTIALS_FILE"
|
||||
set -u
|
||||
fi
|
||||
|
||||
if [ -z "${AWS_ACCESS_KEY_ID:-}" ]; then
|
||||
CONFIG_PY="/root/management-platform/config.py"
|
||||
if [ -f "$CONFIG_PY" ]; then
|
||||
_KEY=$(grep -oP '(?<=R2_ACCESS_KEY_ID",\s{5}")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
|
||||
_SEC=$(grep -oP '(?<=R2_SECRET_ACCESS_KEY", ")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
|
||||
_BKT=$(grep -oP '(?<=R2_BUCKET_NAME",\s{7}")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
|
||||
[ -n "$_KEY" ] && export AWS_ACCESS_KEY_ID="$_KEY"
|
||||
[ -n "$_SEC" ] && export AWS_SECRET_ACCESS_KEY="$_SEC"
|
||||
[ -n "$_BKT" ] && export R2_BUCKET_NAME="$_BKT"
|
||||
fi
|
||||
fi
|
||||
|
||||
R2_ACCOUNT_ID="${R2_ACCOUNT_ID:-35e00c230cc8066252a2d9890b69aea2}"
|
||||
R2_BUCKET="${R2_BUCKET_NAME:-navitrends-backups}"
|
||||
R2_ENDPOINT="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
|
||||
|
||||
BACKUP_LOG_FILE="/root/backups/k8s-backup-status.log"
|
||||
MAX_BACKUPS=10
|
||||
MAX_R2_BACKUPS=5
|
||||
|
||||
TIER1_STATUS="pending"
|
||||
TIER2_STATUS="pending"
|
||||
TIER3_STATUS="pending"
|
||||
|
||||
log_status() {
|
||||
local status="$1" name="$2" msg="${3:-}"
|
||||
echo "$(date '+%Y-%m-%d %H:%M:%S') | ${status} | ${name} | ${msg}" >> "$BACKUP_LOG_FILE"
|
||||
}
|
||||
|
||||
# --------------------------------------------------
|
||||
# Resource-safety gate (migration-status.md §5): abort before starting
|
||||
# any heavy per-app work if we're already below threshold.
|
||||
# --------------------------------------------------
|
||||
check_resources() {
|
||||
local avail_mb avail_gb
|
||||
avail_mb=$(free -m | awk '/^Mem:/{print $7}')
|
||||
avail_gb=$(df -BG / | awk 'NR==2{gsub("G","",$4); print $4}')
|
||||
if [ "$avail_mb" -lt 1536 ]; then
|
||||
echo " ❌ RAM available ${avail_mb}MB < 1536MB threshold — aborting."
|
||||
log_status "FAILED" "$BACKUP_NAME" "resource_safety_ram_${avail_mb}mb"
|
||||
exit 1
|
||||
fi
|
||||
if [ "$avail_gb" -lt 10 ]; then
|
||||
echo " ❌ Disk available ${avail_gb}GB < 10GB threshold — aborting."
|
||||
log_status "FAILED" "$BACKUP_NAME" "resource_safety_disk_${avail_gb}gb"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
echo "========================================="
|
||||
echo "📦 Starting k8s Backup: $BACKUP_NAME"
|
||||
if [ "$APPS_FILTER" = "all" ]; then
|
||||
echo " Apps: ALL ($ALL_APPS)"
|
||||
else
|
||||
echo " Apps: $APPS_FILTER"
|
||||
fi
|
||||
echo " Tiers: Local → VM → ☁ Cloudflare R2"
|
||||
echo "========================================="
|
||||
|
||||
check_resources
|
||||
mkdir -p "$BACKUP_DIR" "/root/backups"
|
||||
cd "$BACKUP_DIR"
|
||||
|
||||
APP_COUNT=0
|
||||
for app in $ALL_APPS; do
|
||||
app_selected "$app" || continue
|
||||
ns="${NS[$app]}"
|
||||
app_deploy="${APP_DEPLOY[$app]}"
|
||||
app_ctr="${APP_CTR[$app]}"
|
||||
db_deploy="${DB_DEPLOY[$app]}"
|
||||
db_engine="${DB_ENGINE[$app]}"
|
||||
db_name="${DB_NAME[$app]}"
|
||||
pvc_path="${PVC_PATH[$app]}"
|
||||
secret_name="${SECRET_NAME[$app]}"
|
||||
db_user_key="${DB_USER_KEY[$app]}"
|
||||
db_pass_key="${DB_PASS_KEY[$app]}"
|
||||
|
||||
echo ""
|
||||
echo "-----------------------------------------"
|
||||
echo "📁 [$app] namespace=$ns"
|
||||
echo "-----------------------------------------"
|
||||
|
||||
APP_DIR="$BACKUP_DIR/$app"
|
||||
mkdir -p "$APP_DIR"
|
||||
|
||||
# ---- 1. Manifests ----
|
||||
echo -n " 📄 Manifests (deployment/service/configmap/ingress) ... "
|
||||
kubectl get deployment,service,configmap,ingress -n "$ns" -l owner=ameni-boukattaya -o yaml \
|
||||
> "$APP_DIR/manifests.yaml" 2>"$APP_DIR/manifests.err" \
|
||||
&& { echo "✅"; rm -f "$APP_DIR/manifests.err"; } || echo "⚠️ FAILED (see manifests.err)"
|
||||
|
||||
# ---- 2. Secret (plaintext, encryption-at-rest deferred) ----
|
||||
echo -n " 🔑 Secret ($secret_name) ... "
|
||||
kubectl get secret "$secret_name" -n "$ns" -o yaml \
|
||||
> "$APP_DIR/secret.yaml" 2>"$APP_DIR/secret.err" \
|
||||
&& { echo "✅"; rm -f "$APP_DIR/secret.err"; } || echo "⚠️ FAILED (see secret.err)"
|
||||
|
||||
# ---- 3. DB dump via kubectl exec (never raw datadir copy) ----
|
||||
echo -n " 💾 DB dump ($db_engine) ... "
|
||||
case "$db_engine" in
|
||||
postgres)
|
||||
db_user=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_user_key}}" 2>/dev/null | base64 -d)
|
||||
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
||||
if [ -n "$db_user" ] && [ -n "$db_pass" ]; then
|
||||
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
||||
pg_dump -U "$db_user" -d "$db_name" 2>"$APP_DIR/db-dump.err" \
|
||||
| gzip > "$APP_DIR/db-dump.sql.gz"
|
||||
pg_dump_rc=${PIPESTATUS[0]}
|
||||
if [ "$pg_dump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then
|
||||
echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1))"
|
||||
rm -f "$APP_DIR/db-dump.err"
|
||||
else
|
||||
echo "⚠️ FAILED (rc=$pg_dump_rc, see db-dump.err)"
|
||||
fi
|
||||
else
|
||||
echo "⚠️ could not read DB credentials from secret"
|
||||
fi
|
||||
;;
|
||||
mariadb)
|
||||
if [ "$db_name" = "DYNAMIC" ]; then
|
||||
# Frappe: real db_name/db_password live in site_config.json on the
|
||||
# sites PVC, not in the k8s Secret. Dump with root instead, but still
|
||||
# need the site's real db_name to target the right database.
|
||||
site_dir=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
|
||||
sh -c 'for d in sites/*/; do [ -f "$d/site_config.json" ] && [ ! -L "${d%/}" ] && basename "$d" && break; done' 2>/dev/null)
|
||||
db_name=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
|
||||
cat "sites/${site_dir}/site_config.json" 2>/dev/null \
|
||||
| python3 -c 'import json,sys; print(json.load(sys.stdin)["db_name"])' 2>/dev/null)
|
||||
fi
|
||||
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
||||
if [ -n "$db_name" ] && [ -n "$db_pass" ]; then
|
||||
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \
|
||||
mysqldump -uroot --databases "$db_name" 2>"$APP_DIR/db-dump.err" \
|
||||
| gzip > "$APP_DIR/db-dump.sql.gz"
|
||||
mysqldump_rc=${PIPESTATUS[0]}
|
||||
if [ "$mysqldump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then
|
||||
echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1), db=$db_name)"
|
||||
echo "$db_name" > "$APP_DIR/db-name.txt"
|
||||
rm -f "$APP_DIR/db-dump.err"
|
||||
else
|
||||
echo "⚠️ FAILED (rc=$mysqldump_rc, see db-dump.err)"
|
||||
fi
|
||||
else
|
||||
echo "⚠️ could not determine DB name/credentials"
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
# ---- 4. App-data PVC contents via kubectl exec (live tar, app-data only,
|
||||
# never raw filesystem/PVC-path access — §5 sandbox-divergence rule) ----
|
||||
echo -n " 📦 PVC data ($pvc_path) ... "
|
||||
kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
|
||||
tar czf - -C "$pvc_path" . 2>"$APP_DIR/pvc-data.err" \
|
||||
> "$APP_DIR/pvc-data.tar.gz"
|
||||
if [ -s "$APP_DIR/pvc-data.tar.gz" ]; then
|
||||
echo "✅ ($(du -h "$APP_DIR/pvc-data.tar.gz" | cut -f1))"
|
||||
rm -f "$APP_DIR/pvc-data.err"
|
||||
else
|
||||
echo "⚠️ FAILED (see pvc-data.err)"
|
||||
fi
|
||||
|
||||
APP_COUNT=$((APP_COUNT + 1))
|
||||
|
||||
# Resource check between apps, not just at the start (§5: check every
|
||||
# 1-2 min during long-running/heavy ops; PVC tars can be sizeable).
|
||||
check_resources
|
||||
done
|
||||
|
||||
# --------------------------------------------------
|
||||
# Metadata + checksum
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "📝 Writing backup metadata..."
|
||||
cat > "$BACKUP_DIR/backup-info.txt" << EOF
|
||||
Backup Name: $BACKUP_NAME
|
||||
Backup Date: $(date)
|
||||
Hostname: $(hostname)
|
||||
Apps: $([ "$APPS_FILTER" = "all" ] && echo "$ALL_APPS" || echo "$APPS_FILTER")
|
||||
Apps backed up: $APP_COUNT
|
||||
kubectl: $(kubectl version --client -o yaml 2>/dev/null | grep gitVersion | head -1 || echo 'N/A')
|
||||
Storage Tiers:
|
||||
- Local: /root/backups/
|
||||
- VM: ${VM_HOST}:${VM_PORT} → ${VM_DEST}
|
||||
- Cloud: Cloudflare R2 → s3://${R2_BUCKET}/backups/
|
||||
EOF
|
||||
|
||||
echo "" >> "$BACKUP_DIR/backup-info.txt"
|
||||
echo "Per-app checksums:" >> "$BACKUP_DIR/backup-info.txt"
|
||||
find "$BACKUP_DIR" -type f \( -name "*.tar.gz" -o -name "*.sql.gz" \) -print0 \
|
||||
| xargs -0 -r sha256sum >> "$BACKUP_DIR/backup-info.txt"
|
||||
echo " ✅ Done"
|
||||
|
||||
# --------------------------------------------------
|
||||
# Compress (identical logic to backup-myapps.sh)
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "🗜️ Compressing backup..."
|
||||
cd /root/backups
|
||||
|
||||
tar -czf "${BACKUP_NAME}.tar.gz" "${BACKUP_NAME}/" 2>/dev/null
|
||||
if [ $? -ne 0 ]; then
|
||||
echo " ❌ CRITICAL: Compression failed!"
|
||||
log_status "FAILED" "$BACKUP_NAME" "compression_failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
COMPRESSED_SIZE=$(du -h "${BACKUP_NAME}.tar.gz" | cut -f1)
|
||||
echo " ✅ Compressed size: $COMPRESSED_SIZE → $BACKUP_ARCHIVE"
|
||||
|
||||
sha256sum "${BACKUP_NAME}.tar.gz" > "${BACKUP_NAME}.tar.gz.sha256"
|
||||
echo " ✅ Checksum written"
|
||||
|
||||
rm -rf "$BACKUP_DIR"
|
||||
TIER1_STATUS="ok:${COMPRESSED_SIZE}"
|
||||
echo ""
|
||||
echo "✅ [TIER 1] Local backup complete: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"
|
||||
|
||||
# --------------------------------------------------
|
||||
# Retention — keep only MAX_BACKUPS locally (own glob, own lineage)
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "🧹 [Retention] Keeping latest ${MAX_BACKUPS} local k8s backups..."
|
||||
ARCHIVE_LIST=$(ls -t /root/backups/myapps-k8s-backup-*.tar.gz 2>/dev/null || true)
|
||||
ARCHIVE_COUNT=$(echo "$ARCHIVE_LIST" | grep -c '.tar.gz' 2>/dev/null || echo "0")
|
||||
|
||||
if [ "$ARCHIVE_COUNT" -gt "$MAX_BACKUPS" ]; then
|
||||
TO_DELETE=$(echo "$ARCHIVE_LIST" | tail -n +$((MAX_BACKUPS + 1)))
|
||||
while IFS= read -r old_file; do
|
||||
[ -z "$old_file" ] && continue
|
||||
rm -f "$old_file" "${old_file}.sha256"
|
||||
echo " 🗑️ Deleted: $(basename "$old_file")"
|
||||
done <<< "$TO_DELETE"
|
||||
else
|
||||
echo " ✅ ${ARCHIVE_COUNT}/${MAX_BACKUPS} backups — nothing to prune"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# VM transfer [TIER 2] — non-fatal (identical to backup-myapps.sh)
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "📤 [TIER 2] Sending backup to VM (${VM_HOST}:${VM_PORT})..."
|
||||
|
||||
scp -i "$VM_KEY" -P "$VM_PORT" \
|
||||
-o StrictHostKeyChecking=no -o ConnectTimeout=30 \
|
||||
"${BACKUP_NAME}.tar.gz" \
|
||||
"${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null
|
||||
VM_SCP_RC=$?
|
||||
|
||||
if [ $VM_SCP_RC -eq 0 ]; then
|
||||
echo " ✅ Backup sent to VM successfully!"
|
||||
scp -i "$VM_KEY" -P "$VM_PORT" \
|
||||
-o StrictHostKeyChecking=no -o ConnectTimeout=15 \
|
||||
"${BACKUP_NAME}.tar.gz.sha256" \
|
||||
"${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null || true
|
||||
TIER2_STATUS="ok"
|
||||
else
|
||||
echo " ⚠️ VM transfer failed (rc=$VM_SCP_RC) — local backup is safe, continuing..."
|
||||
TIER2_STATUS="failed:rc=${VM_SCP_RC}"
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# Cloudflare R2 [TIER 3] — non-fatal (identical to backup-myapps.sh,
|
||||
# key prefix stays "backups/" — shared with the legacy lineage in the
|
||||
# same bucket, distinguished by filename prefix)
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "☁️ [TIER 3] Uploading to Cloudflare R2..."
|
||||
|
||||
if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then
|
||||
echo " ⚠️ R2 credentials not found — skipping cloud upload"
|
||||
TIER3_STATUS="skipped:no_credentials"
|
||||
else
|
||||
echo " Credentials: found (key=${AWS_ACCESS_KEY_ID:0:8}...)"
|
||||
echo " Bucket: ${R2_BUCKET}"
|
||||
echo " Endpoint: ${R2_ENDPOINT}"
|
||||
|
||||
PYTHON_BIN="/root/management-platform/venv/bin/python3"
|
||||
if [ ! -f "$PYTHON_BIN" ]; then
|
||||
PYTHON_BIN="python3"
|
||||
fi
|
||||
|
||||
R2_SCRIPT="/tmp/r2_upload_$$.py"
|
||||
cat > "$R2_SCRIPT" << PYEOF
|
||||
import sys, os, boto3
|
||||
from datetime import datetime, timezone
|
||||
|
||||
account_id = "${R2_ACCOUNT_ID}"
|
||||
bucket = "${R2_BUCKET}"
|
||||
endpoint = "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
|
||||
access_key = "${AWS_ACCESS_KEY_ID}"
|
||||
secret_key = "${AWS_SECRET_ACCESS_KEY}"
|
||||
archive = "/root/backups/${BACKUP_NAME}.tar.gz"
|
||||
sha_file = archive + '.sha256'
|
||||
key = "backups/${BACKUP_NAME}.tar.gz"
|
||||
max_r2 = ${MAX_R2_BACKUPS}
|
||||
name_prefix = "backups/myapps-k8s-backup-"
|
||||
|
||||
try:
|
||||
client = boto3.client(
|
||||
's3',
|
||||
endpoint_url=endpoint,
|
||||
aws_access_key_id=access_key,
|
||||
aws_secret_access_key=secret_key,
|
||||
region_name='auto'
|
||||
)
|
||||
|
||||
try:
|
||||
client.head_bucket(Bucket=bucket)
|
||||
except Exception:
|
||||
client.create_bucket(Bucket=bucket)
|
||||
print(f' Created bucket: {bucket}')
|
||||
|
||||
size_mb = os.path.getsize(archive) / 1024 / 1024
|
||||
timestamp = datetime.now(timezone.utc).isoformat()
|
||||
print(f' Uploading {size_mb:.1f} MB to r2://{bucket}/{key}')
|
||||
client.upload_file(
|
||||
archive, bucket, key,
|
||||
ExtraArgs={"Metadata": {
|
||||
"uploaded-by": "backup-k8s-apps-script",
|
||||
"uploaded-at": timestamp,
|
||||
"original-file": archive.split("/")[-1],
|
||||
}}
|
||||
)
|
||||
print(' ✅ R2 archive uploaded successfully')
|
||||
|
||||
if os.path.exists(sha_file):
|
||||
client.upload_file(sha_file, bucket, key + '.sha256')
|
||||
print(' ✅ R2 SHA256 checksum uploaded')
|
||||
|
||||
# Retention scoped to this script's own lineage (name_prefix) only —
|
||||
# never touches the legacy myapps-backup-* objects in the same bucket.
|
||||
print(f' 🧹 Enforcing R2 retention (max {max_r2}, prefix={name_prefix})...')
|
||||
resp = client.list_objects_v2(Bucket=bucket, Prefix=name_prefix)
|
||||
objects = sorted(
|
||||
[o for o in resp.get('Contents', []) if not o['Key'].endswith('.sha256')],
|
||||
key=lambda x: x['LastModified'],
|
||||
reverse=True
|
||||
)
|
||||
to_delete = objects[max_r2:]
|
||||
if not to_delete:
|
||||
print(f' ✅ {len(objects)}/{max_r2} R2 backups — nothing to prune')
|
||||
for obj in to_delete:
|
||||
old_key = obj['Key']
|
||||
client.delete_object(Bucket=bucket, Key=old_key)
|
||||
try:
|
||||
client.delete_object(Bucket=bucket, Key=old_key + '.sha256')
|
||||
except Exception:
|
||||
pass
|
||||
print(f' 🗑️ Deleted from R2: {old_key.replace("backups/", "")}')
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
except Exception as e:
|
||||
print(f' ERROR: {e}')
|
||||
sys.exit(2)
|
||||
PYEOF
|
||||
|
||||
"$PYTHON_BIN" "$R2_SCRIPT"
|
||||
R2_RC=$?
|
||||
rm -f "$R2_SCRIPT"
|
||||
|
||||
if [ $R2_RC -eq 0 ]; then
|
||||
echo " ✅ R2 upload + retention complete"
|
||||
TIER3_STATUS="ok"
|
||||
else
|
||||
echo " ⚠️ R2 upload failed (rc=$R2_RC) — local backup is safe at: $BACKUP_ARCHIVE"
|
||||
TIER3_STATUS="failed:rc=${R2_RC}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# Final summary + log
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "✅ K8S BACKUP COMPLETE"
|
||||
echo ""
|
||||
echo " 📦 Name: $BACKUP_NAME"
|
||||
echo " 💾 Local: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"
|
||||
echo " 🖥️ VM: ${TIER2_STATUS}"
|
||||
echo " ☁️ R2: ${TIER3_STATUS}"
|
||||
echo "========================================="
|
||||
|
||||
log_status "SUCCESS" "$BACKUP_NAME" \
|
||||
"apps=${APP_COUNT} | size=${COMPRESSED_SIZE} | tier1=${TIER1_STATUS} | tier2=${TIER2_STATUS} | tier3=${TIER3_STATUS}"
|
||||
269
backup/restore-k8s-apps.sh
Executable file
269
backup/restore-k8s-apps.sh
Executable file
@@ -0,0 +1,269 @@
|
||||
#!/bin/bash
|
||||
# =============================================
|
||||
# restore-k8s-apps.sh — k8s-native replacement for restore-myapps.sh
|
||||
# Restores: n8n, Odoo, Mautic, Nextcloud, Frappe/ERPNext (all live in k3s)
|
||||
#
|
||||
# Contract mirrors restore-myapps.sh exactly, so wiring into app.py's
|
||||
# restore_start route is a pure path swap:
|
||||
# - Expects to be run from inside an already-extracted backup directory
|
||||
# (SCRIPT_DIR = the script's own directory), i.e. app.py's existing
|
||||
# "tar -xzf backup --strip-components=1 && cp restore-*.sh <session> &&
|
||||
# cd <session> && bash restore-*.sh --apps X" flow works unchanged —
|
||||
# only the two script filenames/paths need to change on the Python side.
|
||||
# - Same --apps flag: comma-separated subset of frappe,odoo,nextcloud,mautic,n8n
|
||||
# (omit = restore all found in the backup).
|
||||
#
|
||||
# Restore order per app (migration-status.md §5 — never populate a PVC
|
||||
# that's still mounted by a running consumer; always populate-then-deploy):
|
||||
# 1. kubectl apply the Secret + manifests (idempotent — creates if missing,
|
||||
# updates in place if present; this is also the DR-onto-fresh-cluster path
|
||||
# for anything that isn't a PVC, which pre-exists there already this session)
|
||||
# 2. Scale the app Deployment to 0 and wait for the pod to terminate
|
||||
# 3. Populate the app-data PVC via a short-lived loader pod + kubectl exec
|
||||
# (never raw filesystem/PVC-path access)
|
||||
# 4. Scale the app Deployment back up
|
||||
# 5. Restore the DB via pg_restore/mysql (SQL-level, never a raw datadir copy)
|
||||
# against the DB Deployment, which stays running throughout
|
||||
#
|
||||
# NOT included this session (explicitly deferred, see backup-migration-plan.md §5):
|
||||
# - Cross-cluster / disaster-recovery restore. This script assumes the target
|
||||
# cluster/namespaces/PVCs already exist (same-cluster restore-in-place).
|
||||
# A fresh-cluster bootstrap + PVC-from-scratch path is future work.
|
||||
# =============================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
APPS_FILTER="all"
|
||||
VALID_APPS="frappe odoo nextcloud mautic n8n"
|
||||
|
||||
while [[ $# -gt 0 ]]; do
|
||||
case "$1" in
|
||||
--apps)
|
||||
APPS_FILTER="$2"
|
||||
shift 2
|
||||
;;
|
||||
*)
|
||||
shift
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "$APPS_FILTER" != "all" ]; then
|
||||
APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
|
||||
IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER"
|
||||
for _a in "${_APP_CHECK[@]}"; do
|
||||
if [[ " $VALID_APPS " != *" $_a "* ]]; then
|
||||
echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
app_selected() {
|
||||
local app="$1"
|
||||
[ "$APPS_FILTER" = "all" ] && return 0
|
||||
[[ ",$APPS_FILTER," == *",${app},"* ]]
|
||||
}
|
||||
|
||||
# --------------------------------------------------
|
||||
# Per-app metadata table (same shape as backup-k8s-apps.sh)
|
||||
# --------------------------------------------------
|
||||
declare -A NS=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext )
|
||||
declare -A APP_DEPLOY=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web )
|
||||
declare -A DB_DEPLOY=( [n8n]=n8n-postgres [odoo]=odoo-postgres [mautic]=mautic-mariadb [nextcloud]=nextcloud-postgres [frappe]=erpnext-mariadb )
|
||||
declare -A DB_ENGINE=( [n8n]=postgres [odoo]=postgres [mautic]=mariadb [nextcloud]=postgres [frappe]=mariadb )
|
||||
declare -A DB_NAME=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=DYNAMIC )
|
||||
declare -A PVC_NAME=( [n8n]=n8n-data [odoo]=odoo-data [mautic]=mautic-app-data [nextcloud]=nextcloud-app-data [frappe]=erpnext-sites )
|
||||
declare -A PVC_PATH=( [n8n]="/home/node/.n8n" [odoo]="/var/lib/odoo" [mautic]="/var/www/html" [nextcloud]="/var/www/html" [frappe]="/home/frappe/frappe-bench/sites" )
|
||||
declare -A SECRET_NAME=( [n8n]=n8n-secrets [odoo]=odoo-secrets [mautic]=mautic-secrets [nextcloud]=nextcloud-secrets [frappe]=erpnext-secrets )
|
||||
declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" )
|
||||
declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD )
|
||||
declare -A DOMAIN=( [n8n]="n8nwf.nav.ovh" [odoo]="odooo.nav.ovh" [mautic]="mautics.nav.ovh" [nextcloud]="next.cloud.nav.ovh" [frappe]="erpnext.navitrends.ovh" )
|
||||
|
||||
ALL_APPS="n8n odoo mautic nextcloud frappe"
|
||||
LOADER_IMAGE="alpine:3.20"
|
||||
|
||||
echo "========================================="
|
||||
echo "🔄 k8s Restore — SAME-CLUSTER MODE"
|
||||
echo " Backup dir: $SCRIPT_DIR"
|
||||
if [ "$APPS_FILTER" = "all" ]; then
|
||||
echo " Apps: ALL (found in backup)"
|
||||
else
|
||||
echo " Apps: $APPS_FILTER"
|
||||
fi
|
||||
echo "========================================="
|
||||
|
||||
check_resources() {
|
||||
local avail_mb avail_gb
|
||||
avail_mb=$(free -m | awk '/^Mem:/{print $7}')
|
||||
avail_gb=$(df -BG / | awk 'NR==2{gsub("G","",$4); print $4}')
|
||||
if [ "$avail_mb" -lt 1536 ] || [ "$avail_gb" -lt 10 ]; then
|
||||
echo " ❌ Resource safety threshold hit (RAM=${avail_mb}MB, Disk=${avail_gb}GB) — aborting."
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
RESTORED_APPS=()
|
||||
FAILED_APPS=()
|
||||
|
||||
check_resources
|
||||
|
||||
for app in $ALL_APPS; do
|
||||
app_selected "$app" || continue
|
||||
|
||||
if [ ! -d "$SCRIPT_DIR/$app" ]; then
|
||||
echo ""
|
||||
echo " ⏭️ $app — not present in this backup, skipping"
|
||||
continue
|
||||
fi
|
||||
|
||||
ns="${NS[$app]}"
|
||||
app_deploy="${APP_DEPLOY[$app]}"
|
||||
db_deploy="${DB_DEPLOY[$app]}"
|
||||
db_engine="${DB_ENGINE[$app]}"
|
||||
db_name="${DB_NAME[$app]}"
|
||||
pvc_name="${PVC_NAME[$app]}"
|
||||
pvc_path="${PVC_PATH[$app]}"
|
||||
secret_name="${SECRET_NAME[$app]}"
|
||||
db_user_key="${DB_USER_KEY[$app]}"
|
||||
db_pass_key="${DB_PASS_KEY[$app]}"
|
||||
APP_DIR="$SCRIPT_DIR/$app"
|
||||
|
||||
echo ""
|
||||
echo "-----------------------------------------"
|
||||
echo "🔧 [$app] namespace=$ns"
|
||||
echo "-----------------------------------------"
|
||||
|
||||
app_ok=true
|
||||
|
||||
# ---- 1. Secret + manifests (apply — idempotent) ----
|
||||
if [ -f "$APP_DIR/secret.yaml" ]; then
|
||||
echo -n " 🔑 Applying Secret ... "
|
||||
kubectl apply -f "$APP_DIR/secret.yaml" &>/dev/null && echo "✅" || { echo "⚠️ FAILED"; app_ok=false; }
|
||||
fi
|
||||
if [ -f "$APP_DIR/manifests.yaml" ]; then
|
||||
echo -n " 📄 Applying manifests ... "
|
||||
kubectl apply -f "$APP_DIR/manifests.yaml" &>/dev/null && echo "✅" || { echo "⚠️ FAILED"; app_ok=false; }
|
||||
fi
|
||||
|
||||
# ---- 2. Scale app down ----
|
||||
prior_replicas=$(kubectl get deployment "$app_deploy" -n "$ns" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 1)
|
||||
[ -z "$prior_replicas" ] && prior_replicas=1
|
||||
echo -n " ⏸️ Scaling $app_deploy to 0 ... "
|
||||
kubectl scale deployment "$app_deploy" -n "$ns" --replicas=0 &>/dev/null \
|
||||
&& kubectl wait --for=delete pod -l app="$app" -n "$ns" --timeout=90s &>/dev/null
|
||||
echo "✅"
|
||||
|
||||
# ---- 3. Restore PVC data via loader pod ----
|
||||
if [ -f "$APP_DIR/pvc-data.tar.gz" ]; then
|
||||
if kubectl get pvc "$pvc_name" -n "$ns" &>/dev/null; then
|
||||
loader="${app}-restore-loader"
|
||||
echo -n " 📦 Populating PVC $pvc_name via loader pod ... "
|
||||
kubectl run "$loader" -n "$ns" --image="$LOADER_IMAGE" --restart=Never \
|
||||
--overrides="{\"spec\":{\"containers\":[{\"name\":\"loader\",\"image\":\"${LOADER_IMAGE}\",\"command\":[\"sleep\",\"3600\"],\"volumeMounts\":[{\"name\":\"data\",\"mountPath\":\"/restore\"}]}],\"volumes\":[{\"name\":\"data\",\"persistentVolumeClaim\":{\"claimName\":\"${pvc_name}\"}}]}}" \
|
||||
&>/dev/null
|
||||
kubectl wait --for=condition=Ready "pod/$loader" -n "$ns" --timeout=60s &>/dev/null
|
||||
|
||||
kubectl exec -n "$ns" "$loader" -- sh -c 'rm -rf /restore/* /restore/.[!.]* /restore/..?* 2>/dev/null; true' &>/dev/null
|
||||
if kubectl exec -i -n "$ns" "$loader" -- tar xzf - -C /restore < "$APP_DIR/pvc-data.tar.gz" &>"$APP_DIR/restore-pvc.err"; then
|
||||
echo "✅"
|
||||
rm -f "$APP_DIR/restore-pvc.err"
|
||||
else
|
||||
echo "⚠️ FAILED (see $APP_DIR/restore-pvc.err)"
|
||||
app_ok=false
|
||||
fi
|
||||
kubectl delete pod "$loader" -n "$ns" --wait=false &>/dev/null
|
||||
else
|
||||
echo " ⚠️ PVC '$pvc_name' does not exist in this cluster — skipping PVC restore"
|
||||
echo " (fresh-cluster/DR provisioning of PVCs from manifests is not yet implemented)"
|
||||
app_ok=false
|
||||
fi
|
||||
else
|
||||
echo " ⏭️ No pvc-data.tar.gz in backup — skipping PVC restore"
|
||||
fi
|
||||
|
||||
# ---- 4. Scale app back up ----
|
||||
echo -n " ▶️ Scaling $app_deploy back to $prior_replicas ... "
|
||||
kubectl scale deployment "$app_deploy" -n "$ns" --replicas="$prior_replicas" &>/dev/null && echo "✅" || echo "⚠️ FAILED"
|
||||
|
||||
# ---- 5. DB restore (SQL-level, never raw datadir copy) ----
|
||||
if [ -f "$APP_DIR/db-dump.sql.gz" ]; then
|
||||
echo -n " 💾 Restoring DB ($db_engine) ... "
|
||||
case "$db_engine" in
|
||||
postgres)
|
||||
db_user=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_user_key}}" 2>/dev/null | base64 -d)
|
||||
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
||||
if [ -n "$db_user" ] && [ -n "$db_pass" ]; then
|
||||
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
||||
psql -U "$db_user" -d postgres -c "DROP DATABASE IF EXISTS \"$db_name\";" &>/dev/null
|
||||
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
||||
createdb -U "$db_user" "$db_name" &>/dev/null
|
||||
if gunzip -c "$APP_DIR/db-dump.sql.gz" | \
|
||||
kubectl exec -i -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
||||
psql -U "$db_user" -d "$db_name" &>"$APP_DIR/restore-db.err"; then
|
||||
echo "✅"
|
||||
rm -f "$APP_DIR/restore-db.err"
|
||||
else
|
||||
echo "⚠️ FAILED (see $APP_DIR/restore-db.err)"
|
||||
app_ok=false
|
||||
fi
|
||||
else
|
||||
echo "⚠️ could not read DB credentials from (restored) secret"
|
||||
app_ok=false
|
||||
fi
|
||||
;;
|
||||
mariadb)
|
||||
if [ "$db_name" = "DYNAMIC" ]; then
|
||||
db_name=$(cat "$APP_DIR/db-name.txt" 2>/dev/null)
|
||||
fi
|
||||
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
||||
if [ -n "$db_name" ] && [ -n "$db_pass" ]; then
|
||||
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \
|
||||
mysql -uroot -e "DROP DATABASE IF EXISTS \`$db_name\`;" &>/dev/null
|
||||
if gunzip -c "$APP_DIR/db-dump.sql.gz" | \
|
||||
kubectl exec -i -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \
|
||||
mysql -uroot &>"$APP_DIR/restore-db.err"; then
|
||||
echo "✅ (db=$db_name)"
|
||||
rm -f "$APP_DIR/restore-db.err"
|
||||
else
|
||||
echo "⚠️ FAILED (see $APP_DIR/restore-db.err)"
|
||||
app_ok=false
|
||||
fi
|
||||
else
|
||||
echo "⚠️ could not determine DB name/credentials"
|
||||
app_ok=false
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
else
|
||||
echo " ⏭️ No db-dump.sql.gz in backup — skipping DB restore"
|
||||
fi
|
||||
|
||||
if [ "$app_ok" = true ]; then
|
||||
RESTORED_APPS+=("$app")
|
||||
else
|
||||
FAILED_APPS+=("$app")
|
||||
fi
|
||||
|
||||
check_resources
|
||||
done
|
||||
|
||||
# --------------------------------------------------
|
||||
# Summary
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "========================================="
|
||||
echo "✅ RESTORE COMPLETE"
|
||||
echo "========================================="
|
||||
for app in "${RESTORED_APPS[@]:-}"; do
|
||||
[ -z "$app" ] && continue
|
||||
echo " ${app} → https://${DOMAIN[$app]}"
|
||||
done
|
||||
if [ "${#FAILED_APPS[@]}" -gt 0 ]; then
|
||||
echo ""
|
||||
echo " ⚠️ Had failures: ${FAILED_APPS[*]} — check the .err files under each app's backup dir"
|
||||
fi
|
||||
echo "========================================="
|
||||
Reference in New Issue
Block a user