49b68cf820
Investigated a resource-safety abort during Odoo restore testing: available RAM was hovering ~1.3-1.5GB even under normal conditions on this host (k3s + Jenkins + Docker monitoring/wazuh stack + dev tooling baseline), not a leak from any single process. 1536MB was tripping on ordinary single-app backups; 1024MB still leaves real headroom above what a pg_dump/mysqldump/tar step actually needs. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_017avLHFqkiti3g62Anq9sVA
301 lines
14 KiB
Bash
Executable File
301 lines
14 KiB
Bash
Executable File
#!/bin/bash
|
|
# =============================================
|
|
# restore-k8s-apps.sh — k8s-native replacement for restore-myapps.sh
|
|
# Restores: n8n, Odoo, Mautic, Nextcloud, Frappe/ERPNext (all live in k3s)
|
|
#
|
|
# Contract mirrors restore-myapps.sh exactly, so wiring into app.py's
|
|
# restore_start route is a pure path swap:
|
|
# - Expects to be run from inside an already-extracted backup directory
|
|
# (SCRIPT_DIR = the script's own directory), i.e. app.py's existing
|
|
# "tar -xzf backup --strip-components=1 && cp restore-*.sh <session> &&
|
|
# cd <session> && bash restore-*.sh --apps X" flow works unchanged —
|
|
# only the two script filenames/paths need to change on the Python side.
|
|
# - Same --apps flag: comma-separated subset of frappe,odoo,nextcloud,mautic,n8n
|
|
# (omit = restore all found in the backup).
|
|
#
|
|
# Restore order per app (migration-status.md §5 — never populate a PVC
|
|
# that's still mounted by a running consumer; always populate-then-deploy):
|
|
# 1. kubectl apply the Secret + manifests (idempotent — creates if missing,
|
|
# updates in place if present; this is also the DR-onto-fresh-cluster path
|
|
# for anything that isn't a PVC, which pre-exists there already this session)
|
|
# 2. Scale the app Deployment to 0 and wait for the pod to terminate
|
|
# 3. Populate the app-data PVC via a short-lived loader pod + kubectl exec
|
|
# (never raw filesystem/PVC-path access)
|
|
# 4. Scale the app Deployment back up
|
|
# 5. Restore the DB via pg_restore/mysql (SQL-level, never a raw datadir copy)
|
|
# against the DB Deployment, which stays running throughout
|
|
#
|
|
# NOT included this session (explicitly deferred, see backup-migration-plan.md §5):
|
|
# - Cross-cluster / disaster-recovery restore. This script assumes the target
|
|
# cluster/namespaces/PVCs already exist (same-cluster restore-in-place).
|
|
# A fresh-cluster bootstrap + PVC-from-scratch path is future work.
|
|
# =============================================
|
|
|
|
set -uo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
cd "$SCRIPT_DIR"
|
|
|
|
# When run inside the management-platform pod, only the static k3s binary is
|
|
# hostPath-mounted (no separate kubectl binary) — fall back to `k3s kubectl`.
|
|
# IMPORTANT: this pod also has a pre-existing rw hostPath mount of /root (for
|
|
# unrelated file-browsing features), which means the host's own root-owned
|
|
# ~/.kube/config (full cluster-admin) is silently visible inside the pod too.
|
|
# `--kubeconfig=/dev/null` is REQUIRED here to stop k3s kubectl from picking
|
|
# that up as a base config — without it, every command below silently runs
|
|
# as cluster-admin instead of the scoped management-platform-sa token,
|
|
# defeating the RBAC in management-platform-backup-rbac.yaml entirely.
|
|
if ! command -v kubectl &>/dev/null && command -v k3s &>/dev/null; then
|
|
kubectl() {
|
|
k3s kubectl \
|
|
--kubeconfig=/dev/null \
|
|
--server=https://kubernetes.default.svc \
|
|
--certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \
|
|
--token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
|
|
"$@"
|
|
}
|
|
fi
|
|
|
|
APPS_FILTER="all"
|
|
VALID_APPS="frappe odoo nextcloud mautic n8n"
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--apps)
|
|
APPS_FILTER="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ "$APPS_FILTER" != "all" ]; then
|
|
APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
|
|
IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER"
|
|
for _a in "${_APP_CHECK[@]}"; do
|
|
if [[ " $VALID_APPS " != *" $_a "* ]]; then
|
|
echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS"
|
|
exit 1
|
|
fi
|
|
done
|
|
fi
|
|
|
|
app_selected() {
|
|
local app="$1"
|
|
[ "$APPS_FILTER" = "all" ] && return 0
|
|
[[ ",$APPS_FILTER," == *",${app},"* ]]
|
|
}
|
|
|
|
# --------------------------------------------------
|
|
# Per-app metadata table (same shape as backup-k8s-apps.sh)
|
|
# --------------------------------------------------
|
|
declare -A NS=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext )
|
|
declare -A APP_DEPLOY=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web )
|
|
declare -A DB_DEPLOY=( [n8n]=n8n-postgres [odoo]=odoo-postgres [mautic]=mautic-mariadb [nextcloud]=nextcloud-postgres [frappe]=erpnext-mariadb )
|
|
declare -A DB_ENGINE=( [n8n]=postgres [odoo]=postgres [mautic]=mariadb [nextcloud]=postgres [frappe]=mariadb )
|
|
declare -A DB_NAME=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=DYNAMIC )
|
|
declare -A PVC_NAME=( [n8n]=n8n-data [odoo]=odoo-data [mautic]=mautic-app-data [nextcloud]=nextcloud-app-data [frappe]=erpnext-sites )
|
|
declare -A PVC_PATH=( [n8n]="/home/node/.n8n" [odoo]="/var/lib/odoo" [mautic]="/var/www/html" [nextcloud]="/var/www/html" [frappe]="/home/frappe/frappe-bench/sites" )
|
|
declare -A SECRET_NAME=( [n8n]=n8n-secrets [odoo]=odoo-secrets [mautic]=mautic-secrets [nextcloud]=nextcloud-secrets [frappe]=erpnext-secrets )
|
|
declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" )
|
|
declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD )
|
|
declare -A DOMAIN=( [n8n]="n8nwf.nav.ovh" [odoo]="odooo.nav.ovh" [mautic]="mautics.nav.ovh" [nextcloud]="next.cloud.nav.ovh" [frappe]="erpnext.navitrends.ovh" )
|
|
|
|
ALL_APPS="n8n odoo mautic nextcloud frappe"
|
|
LOADER_IMAGE="alpine:3.20"
|
|
|
|
echo "========================================="
|
|
echo "🔄 k8s Restore — SAME-CLUSTER MODE"
|
|
echo " Backup dir: $SCRIPT_DIR"
|
|
if [ "$APPS_FILTER" = "all" ]; then
|
|
echo " Apps: ALL (found in backup)"
|
|
else
|
|
echo " Apps: $APPS_FILTER"
|
|
fi
|
|
echo "========================================="
|
|
|
|
check_resources() {
|
|
local avail_mb avail_gb
|
|
# /proc/meminfo instead of `free` — `free` isn't present in the
|
|
# management-platform pod's minimal image, /proc/meminfo always is.
|
|
avail_mb=$(awk '/^MemAvailable:/{printf "%d", $2/1024}' /proc/meminfo)
|
|
avail_gb=$(df -BG / | awk 'NR==2{gsub("G","",$4); print $4}')
|
|
# Threshold lowered 1536->1024MB 2026-08-21 — see backup-k8s-apps.sh
|
|
# check_resources() comment for the investigation (baseline host RAM,
|
|
# not a leak).
|
|
if [ "$avail_mb" -lt 1024 ] || [ "$avail_gb" -lt 10 ]; then
|
|
echo " ❌ Resource safety threshold hit (RAM=${avail_mb}MB, Disk=${avail_gb}GB) — aborting."
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
RESTORED_APPS=()
|
|
FAILED_APPS=()
|
|
|
|
check_resources
|
|
|
|
for app in $ALL_APPS; do
|
|
app_selected "$app" || continue
|
|
|
|
if [ ! -d "$SCRIPT_DIR/$app" ]; then
|
|
echo ""
|
|
echo " ⏭️ $app — not present in this backup, skipping"
|
|
continue
|
|
fi
|
|
|
|
ns="${NS[$app]}"
|
|
app_deploy="${APP_DEPLOY[$app]}"
|
|
db_deploy="${DB_DEPLOY[$app]}"
|
|
db_engine="${DB_ENGINE[$app]}"
|
|
db_name="${DB_NAME[$app]}"
|
|
pvc_name="${PVC_NAME[$app]}"
|
|
pvc_path="${PVC_PATH[$app]}"
|
|
secret_name="${SECRET_NAME[$app]}"
|
|
db_user_key="${DB_USER_KEY[$app]}"
|
|
db_pass_key="${DB_PASS_KEY[$app]}"
|
|
APP_DIR="$SCRIPT_DIR/$app"
|
|
|
|
echo ""
|
|
echo "-----------------------------------------"
|
|
echo "🔧 [$app] namespace=$ns"
|
|
echo "-----------------------------------------"
|
|
|
|
app_ok=true
|
|
|
|
# ---- 1. Secret (apply — idempotent update, no create: every Secret
|
|
# restored here already exists in a same-cluster restore) ----
|
|
if [ -f "$APP_DIR/secret.yaml" ]; then
|
|
echo -n " 🔑 Applying Secret ... "
|
|
kubectl apply -f "$APP_DIR/secret.yaml" &>/dev/null && echo "✅" || { echo "⚠️ FAILED"; app_ok=false; }
|
|
fi
|
|
# NOTE: manifests.yaml (Deployment/Service/ConfigMap/Ingress specs) is
|
|
# captured by backup-k8s-apps.sh but deliberately NOT re-applied here.
|
|
# Same-cluster restore-in-place never needs to reconcile those specs —
|
|
# only Secret values, PVC data, and DB content actually change — and
|
|
# the pod's RBAC (management-platform-backup-role) intentionally grants
|
|
# no write on Deployments/Services/ConfigMaps/Ingresses beyond the
|
|
# deployments/scale subresource used in step 2/4 below. manifests.yaml
|
|
# is kept in every backup purely as a captured reference for a future
|
|
# fresh-cluster/DR restore path, which is explicitly out of scope today.
|
|
|
|
# ---- 2. Scale app down ----
|
|
prior_replicas=$(kubectl get deployment "$app_deploy" -n "$ns" -o jsonpath='{.spec.replicas}' 2>/dev/null || echo 1)
|
|
[ -z "$prior_replicas" ] && prior_replicas=1
|
|
echo -n " ⏸️ Scaling $app_deploy to 0 ... "
|
|
kubectl scale deployment "$app_deploy" -n "$ns" --replicas=0 &>/dev/null \
|
|
&& kubectl wait --for=delete pod -l app="$app" -n "$ns" --timeout=90s &>/dev/null
|
|
echo "✅"
|
|
|
|
# ---- 3. Restore PVC data via loader pod ----
|
|
if [ -f "$APP_DIR/pvc-data.tar.gz" ]; then
|
|
if kubectl get pvc "$pvc_name" -n "$ns" &>/dev/null; then
|
|
loader="${app}-restore-loader"
|
|
echo -n " 📦 Populating PVC $pvc_name via loader pod ... "
|
|
kubectl run "$loader" -n "$ns" --image="$LOADER_IMAGE" --restart=Never \
|
|
--overrides="{\"spec\":{\"containers\":[{\"name\":\"loader\",\"image\":\"${LOADER_IMAGE}\",\"command\":[\"sleep\",\"3600\"],\"volumeMounts\":[{\"name\":\"data\",\"mountPath\":\"/restore\"}]}],\"volumes\":[{\"name\":\"data\",\"persistentVolumeClaim\":{\"claimName\":\"${pvc_name}\"}}]}}" \
|
|
&>/dev/null
|
|
kubectl wait --for=condition=Ready "pod/$loader" -n "$ns" --timeout=60s &>/dev/null
|
|
|
|
kubectl exec -n "$ns" "$loader" -- sh -c 'rm -rf /restore/* /restore/.[!.]* /restore/..?* 2>/dev/null; true' &>/dev/null
|
|
if kubectl exec -i -n "$ns" "$loader" -- tar xzf - -C /restore < "$APP_DIR/pvc-data.tar.gz" &>"$APP_DIR/restore-pvc.err"; then
|
|
echo "✅"
|
|
rm -f "$APP_DIR/restore-pvc.err"
|
|
else
|
|
echo "⚠️ FAILED (see $APP_DIR/restore-pvc.err)"
|
|
app_ok=false
|
|
fi
|
|
kubectl delete pod "$loader" -n "$ns" --wait=false &>/dev/null
|
|
else
|
|
echo " ⚠️ PVC '$pvc_name' does not exist in this cluster — skipping PVC restore"
|
|
echo " (fresh-cluster/DR provisioning of PVCs from manifests is not yet implemented)"
|
|
app_ok=false
|
|
fi
|
|
else
|
|
echo " ⏭️ No pvc-data.tar.gz in backup — skipping PVC restore"
|
|
fi
|
|
|
|
# ---- 4. Scale app back up ----
|
|
echo -n " ▶️ Scaling $app_deploy back to $prior_replicas ... "
|
|
kubectl scale deployment "$app_deploy" -n "$ns" --replicas="$prior_replicas" &>/dev/null && echo "✅" || echo "⚠️ FAILED"
|
|
|
|
# ---- 5. DB restore (SQL-level, never raw datadir copy) ----
|
|
if [ -f "$APP_DIR/db-dump.sql.gz" ]; then
|
|
echo -n " 💾 Restoring DB ($db_engine) ... "
|
|
case "$db_engine" in
|
|
postgres)
|
|
db_user=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_user_key}}" 2>/dev/null | base64 -d)
|
|
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
|
if [ -n "$db_user" ] && [ -n "$db_pass" ]; then
|
|
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
|
psql -U "$db_user" -d postgres -c "DROP DATABASE IF EXISTS \"$db_name\";" &>/dev/null
|
|
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
|
createdb -U "$db_user" "$db_name" &>/dev/null
|
|
if gunzip -c "$APP_DIR/db-dump.sql.gz" | \
|
|
kubectl exec -i -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
|
psql -U "$db_user" -d "$db_name" &>"$APP_DIR/restore-db.err"; then
|
|
echo "✅"
|
|
rm -f "$APP_DIR/restore-db.err"
|
|
else
|
|
echo "⚠️ FAILED (see $APP_DIR/restore-db.err)"
|
|
app_ok=false
|
|
fi
|
|
else
|
|
echo "⚠️ could not read DB credentials from (restored) secret"
|
|
app_ok=false
|
|
fi
|
|
;;
|
|
mariadb)
|
|
if [ "$db_name" = "DYNAMIC" ]; then
|
|
db_name=$(cat "$APP_DIR/db-name.txt" 2>/dev/null)
|
|
fi
|
|
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
|
if [ -n "$db_name" ] && [ -n "$db_pass" ]; then
|
|
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \
|
|
mysql -uroot -e "DROP DATABASE IF EXISTS \`$db_name\`;" &>/dev/null
|
|
if gunzip -c "$APP_DIR/db-dump.sql.gz" | \
|
|
kubectl exec -i -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \
|
|
mysql -uroot &>"$APP_DIR/restore-db.err"; then
|
|
echo "✅ (db=$db_name)"
|
|
rm -f "$APP_DIR/restore-db.err"
|
|
else
|
|
echo "⚠️ FAILED (see $APP_DIR/restore-db.err)"
|
|
app_ok=false
|
|
fi
|
|
else
|
|
echo "⚠️ could not determine DB name/credentials"
|
|
app_ok=false
|
|
fi
|
|
;;
|
|
esac
|
|
else
|
|
echo " ⏭️ No db-dump.sql.gz in backup — skipping DB restore"
|
|
fi
|
|
|
|
if [ "$app_ok" = true ]; then
|
|
RESTORED_APPS+=("$app")
|
|
else
|
|
FAILED_APPS+=("$app")
|
|
fi
|
|
|
|
check_resources
|
|
done
|
|
|
|
# --------------------------------------------------
|
|
# Summary
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "========================================="
|
|
echo "✅ RESTORE COMPLETE"
|
|
echo "========================================="
|
|
for app in "${RESTORED_APPS[@]:-}"; do
|
|
[ -z "$app" ] && continue
|
|
echo " ${app} → https://${DOMAIN[$app]}"
|
|
done
|
|
if [ "${#FAILED_APPS[@]}" -gt 0 ]; then
|
|
echo ""
|
|
echo " ⚠️ Had failures: ${FAILED_APPS[*]} — check the .err files under each app's backup dir"
|
|
fi
|
|
echo "========================================="
|