459c2974f0
Checking disk usage on the VM (95% full, blocking the planned k3s install) found /backups/cloudproject had 85 legacy Docker-era myapps-backup-* archives totaling 131GB, dating back to May — the whole tier had never had any retention at all, only local (TIER 1) and R2 (TIER 3) did. Those 85 are from before the k8s migration entirely (the Docker volumes they captured don't exist anymore), so deleted all of them directly on the VM rather than just trimming a few. Added retention to the VM upload step mirroring the existing local policy exactly: same MAX_BACKUPS, own myapps-k8s-backup-* glob only — never touches the other unrelated backup directories that also live on this VM (imcbackups/mpack/garage, other servers' backups, already separately reviewed and left alone except a manual one-off trim of garage's uncapped 31-day history down to 10). Disk usage: 183G/193G (95%) -> 50G/193G (26%) after cleanup.
633 lines
27 KiB
Bash
Executable File
633 lines
27 KiB
Bash
Executable File
#!/bin/bash
|
|
# =============================================
|
|
# backup-k8s-apps.sh — k8s-native replacement for backup-myapps.sh
|
|
# Backs up: n8n, Odoo, Mautic, Nextcloud, Frappe/ERPNext (all live in k3s)
|
|
#
|
|
# Per app, captures:
|
|
# - k8s manifests (Deployment/Service/ConfigMap/Ingress, via `kubectl get -o yaml`)
|
|
# - Secret manifest (plaintext for now — encryption-at-rest explicitly deferred)
|
|
# - DB dump (pg_dump / mysqldump via `kubectl exec`, never a raw PGDATA/datadir copy)
|
|
# - App-data PVC contents (via `kubectl exec ... tar`, never raw filesystem access —
|
|
# see migration-status.md §5, sandbox-filesystem-divergence rule)
|
|
#
|
|
# Storage tiers (logic reused verbatim from backup-myapps.sh, just repointed at the
|
|
# new archive name/contents):
|
|
# 1. Local → /root/backups/ (CRITICAL — always runs)
|
|
# 2. VM → SSH tunnel → /backups/cloudproject/
|
|
# 3. Cloud → Cloudflare R2 (S3-compatible, optional)
|
|
#
|
|
# NOTE: R2 and VM failures do NOT stop the backup. Local backup always completes first.
|
|
#
|
|
# Archive name is deliberately NOT "myapps-backup-*" (the legacy Docker-era pattern
|
|
# still produced nightly by /root/backup-myapps.sh via cron) — using a distinct
|
|
# "myapps-k8s-backup-*" prefix keeps the two backup lineages from mixing in the same
|
|
# retention pool / R2 prefix until the legacy cron job is explicitly retired.
|
|
# =============================================
|
|
|
|
set -uo pipefail
|
|
|
|
# When run inside the management-platform pod, only the static k3s binary is
|
|
# hostPath-mounted (no separate kubectl binary) — fall back to `k3s kubectl`.
|
|
# IMPORTANT: this pod also has a pre-existing rw hostPath mount of /root (for
|
|
# unrelated file-browsing features), which means the host's own root-owned
|
|
# ~/.kube/config (full cluster-admin) is silently visible inside the pod too.
|
|
# `--kubeconfig=/dev/null` is REQUIRED here to stop k3s kubectl from picking
|
|
# that up as a base config — without it, every command below silently runs
|
|
# as cluster-admin instead of the scoped management-platform-sa token,
|
|
# defeating the RBAC in management-platform-backup-rbac.yaml entirely.
|
|
if ! command -v kubectl &>/dev/null && command -v k3s &>/dev/null; then
|
|
kubectl() {
|
|
k3s kubectl \
|
|
--kubeconfig=/dev/null \
|
|
--server=https://kubernetes.default.svc \
|
|
--certificate-authority=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \
|
|
--token="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
|
|
"$@"
|
|
}
|
|
fi
|
|
|
|
# --------------------------------------------------
|
|
# Parse --apps
|
|
# --------------------------------------------------
|
|
APPS_FILTER="all" # "all" or comma-separated: frappe,odoo,nextcloud,mautic,n8n
|
|
VALID_APPS="frappe odoo nextcloud mautic n8n"
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--apps)
|
|
APPS_FILTER="$2"
|
|
shift 2
|
|
;;
|
|
*)
|
|
shift
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if [ "$APPS_FILTER" != "all" ]; then
|
|
APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
|
|
IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER"
|
|
for _a in "${_APP_CHECK[@]}"; do
|
|
if [[ " $VALID_APPS " != *" $_a "* ]]; then
|
|
echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS"
|
|
exit 1
|
|
fi
|
|
done
|
|
fi
|
|
|
|
app_selected() {
|
|
local app="$1"
|
|
[ "$APPS_FILTER" = "all" ] && return 0
|
|
[[ ",$APPS_FILTER," == *",${app},"* ]]
|
|
}
|
|
|
|
# --------------------------------------------------
|
|
# Per-app metadata table
|
|
# app-key : namespace : app-deployment : db-deployment : db-engine : db-name(or DYNAMIC) : pvc-mount-path : container-name(for exec) : secret-name : db-user-key : db-pass-key
|
|
# --------------------------------------------------
|
|
declare -A NS=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext )
|
|
declare -A APP_DEPLOY=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web )
|
|
declare -A APP_CTR=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=erpnext-web )
|
|
declare -A DB_DEPLOY=( [n8n]=n8n-postgres [odoo]=odoo-postgres [mautic]=mautic-mariadb [nextcloud]=nextcloud-postgres [frappe]=erpnext-mariadb )
|
|
declare -A DB_ENGINE=( [n8n]=postgres [odoo]=postgres [mautic]=mariadb [nextcloud]=postgres [frappe]=mariadb )
|
|
declare -A DB_NAME=( [n8n]=n8n [odoo]=odoo [mautic]=mautic [nextcloud]=nextcloud [frappe]=DYNAMIC )
|
|
declare -A PVC_PATH=( [n8n]="/home/node/.n8n" [odoo]="/var/lib/odoo" [mautic]="/var/www/html" [nextcloud]="/var/www/html" [frappe]="/home/frappe/frappe-bench/sites" )
|
|
declare -A SECRET_NAME=( [n8n]=n8n-secrets [odoo]=odoo-secrets [mautic]=mautic-secrets [nextcloud]=nextcloud-secrets [frappe]=erpnext-secrets )
|
|
declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" )
|
|
declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD )
|
|
|
|
# Reproducible framework/vendor code that lives on the same PVC as real user
|
|
# data for these two apps (single shared mount, no separate volume). Both
|
|
# images bake these dirs into their own filesystem layer independent of any
|
|
# volume — Mautic has no logic to reconstruct them, so restore-k8s-apps.sh
|
|
# re-seeds them straight from the app's own image instead of from backup.
|
|
# Never pull request/exclude changes here without updating the matching
|
|
# RECONSTRUCT_DIRS table in restore-k8s-apps.sh.
|
|
declare -A TAR_EXCLUDES=( [mautic]="node_modules vendor" [nextcloud]="apps core dist 3rdparty lib" )
|
|
|
|
ALL_APPS="n8n odoo mautic nextcloud frappe"
|
|
|
|
BACKUP_DATE=$(date +%Y%m%d_%H%M%S)
|
|
BACKUP_NAME="myapps-k8s-backup-${BACKUP_DATE}"
|
|
BACKUP_DIR="/root/backups/${BACKUP_NAME}"
|
|
BACKUP_ARCHIVE="/root/backups/${BACKUP_NAME}.tar.gz"
|
|
|
|
VM_USER="root"
|
|
VM_HOST="178.18.243.51"
|
|
VM_PORT="22"
|
|
VM_KEY="/root/.ssh/id_rsa"
|
|
VM_DEST="/backups/cloudproject/"
|
|
|
|
# ── Load R2 credentials (identical to backup-myapps.sh) ────────────────────────
|
|
CREDENTIALS_FILE="/root/.r2-credentials"
|
|
if [ -f "$CREDENTIALS_FILE" ]; then
|
|
set +u
|
|
source "$CREDENTIALS_FILE"
|
|
set -u
|
|
fi
|
|
|
|
if [ -z "${AWS_ACCESS_KEY_ID:-}" ]; then
|
|
CONFIG_PY="/root/management-platform/config.py"
|
|
if [ -f "$CONFIG_PY" ]; then
|
|
_KEY=$(grep -oP '(?<=R2_ACCESS_KEY_ID",\s{5}")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
|
|
_SEC=$(grep -oP '(?<=R2_SECRET_ACCESS_KEY", ")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
|
|
_BKT=$(grep -oP '(?<=R2_BUCKET_NAME",\s{7}")[^"]+' "$CONFIG_PY" 2>/dev/null || true)
|
|
[ -n "$_KEY" ] && export AWS_ACCESS_KEY_ID="$_KEY"
|
|
[ -n "$_SEC" ] && export AWS_SECRET_ACCESS_KEY="$_SEC"
|
|
[ -n "$_BKT" ] && export R2_BUCKET_NAME="$_BKT"
|
|
fi
|
|
fi
|
|
|
|
R2_ACCOUNT_ID="${R2_ACCOUNT_ID:-35e00c230cc8066252a2d9890b69aea2}"
|
|
R2_BUCKET="${R2_BUCKET_NAME:-navitrends-backups}"
|
|
R2_ENDPOINT="https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
|
|
|
|
BACKUP_LOG_FILE="/root/backups/k8s-backup-status.log"
|
|
MAX_BACKUPS=10
|
|
MAX_R2_BACKUPS=5
|
|
|
|
TIER1_STATUS="pending"
|
|
TIER2_STATUS="pending"
|
|
TIER3_STATUS="pending"
|
|
|
|
log_status() {
|
|
local status="$1" name="$2" msg="${3:-}"
|
|
echo "$(date '+%Y-%m-%d %H:%M:%S') | ${status} | ${name} | ${msg}" >> "$BACKUP_LOG_FILE"
|
|
}
|
|
|
|
# --------------------------------------------------
|
|
# Resource-safety gate (migration-status.md §5): abort before starting
|
|
# any heavy per-app work if we're already below threshold.
|
|
# --------------------------------------------------
|
|
check_resources() {
|
|
local avail_mb avail_gb
|
|
# /proc/meminfo instead of `free` — `free` isn't present in the
|
|
# management-platform pod's minimal image, /proc/meminfo always is.
|
|
avail_mb=$(awk '/^MemAvailable:/{printf "%d", $2/1024}' /proc/meminfo)
|
|
avail_gb=$(df -BG / | awk 'NR==2{gsub("G","",$4); print $4}')
|
|
# Threshold lowered 1536->1024MB 2026-08-21: this host's steady-state
|
|
# available RAM sits ~1.3-1.5GB even when idle (k3s + Jenkins + Docker
|
|
# monitoring/wazuh stack + dev tooling baseline) — investigated, no
|
|
# single runaway process, just genuine multi-service baseline. 1536MB
|
|
# was tripping on ordinary single-app backups. 1024MB still leaves
|
|
# real headroom above what a pg_dump/mysqldump/tar op actually needs.
|
|
if [ "$avail_mb" -lt 1024 ]; then
|
|
echo " ❌ RAM available ${avail_mb}MB < 1024MB threshold — aborting."
|
|
log_status "FAILED" "$BACKUP_NAME" "resource_safety_ram_${avail_mb}mb"
|
|
exit 1
|
|
fi
|
|
if [ "$avail_gb" -lt 10 ]; then
|
|
echo " ❌ Disk available ${avail_gb}GB < 10GB threshold — aborting."
|
|
log_status "FAILED" "$BACKUP_NAME" "resource_safety_disk_${avail_gb}gb"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# --------------------------------------------------
|
|
# Per-backup status metadata sidecar (.meta.json) — powers the UI's
|
|
# green/yellow/red status dot without ever needing to decompress the
|
|
# (possibly 600MB+) archive itself. Written only after all 3 tiers are
|
|
# known, then best-effort mirrored to VM/R2 alongside the archive, same
|
|
# as the .sha256 sidecar already is — so status is readable from
|
|
# whichever tier the UI is listing, not just local.
|
|
# --------------------------------------------------
|
|
write_meta_sidecar() {
|
|
local size_bytes meta_path
|
|
size_bytes=$(stat -c%s "$BACKUP_ARCHIVE" 2>/dev/null || echo 0)
|
|
meta_path="/root/backups/${BACKUP_NAME}.tar.gz.meta.json"
|
|
|
|
python3 - "$BACKUP_NAME" "$size_bytes" "$TIER1_STATUS" "$TIER2_STATUS" "$TIER3_STATUS" \
|
|
"${#INCLUDED_APPS[@]}" "${INCLUDED_APPS[@]}" "${ERRORS[@]}" > "$meta_path" <<'PYEOF'
|
|
import sys, json, datetime
|
|
|
|
backup_name = sys.argv[1]
|
|
size_bytes = int(sys.argv[2])
|
|
tier1, tier2, tier3 = sys.argv[3], sys.argv[4], sys.argv[5]
|
|
n_apps = int(sys.argv[6])
|
|
apps = sys.argv[7:7 + n_apps]
|
|
errors_raw = sys.argv[7 + n_apps:]
|
|
|
|
def norm_tier(s):
|
|
# "ok:572M" -> "ok"; "failed:rc=2" / "skipped:no_credentials" pass through
|
|
# unchanged so the UI can tell a real failure apart from a deliberate skip.
|
|
return 'ok' if s.startswith('ok') else s
|
|
|
|
errors = []
|
|
for e in errors_raw:
|
|
parts = e.split(':', 2)
|
|
errors.append({
|
|
'app': parts[0] if len(parts) > 0 else '',
|
|
'step': parts[1] if len(parts) > 1 else '',
|
|
'detail': parts[2] if len(parts) > 2 else '',
|
|
})
|
|
|
|
meta = {
|
|
'backup_name': backup_name,
|
|
'created_at': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds'),
|
|
'apps': apps,
|
|
'apps_key': ','.join(sorted(apps)),
|
|
'size_bytes': size_bytes,
|
|
'errors': errors,
|
|
'tiers': {'local': norm_tier(tier1), 'vm': norm_tier(tier2), 'r2': norm_tier(tier3)},
|
|
}
|
|
json.dump(meta, sys.stdout, indent=2)
|
|
PYEOF
|
|
|
|
if [ -s "$meta_path" ]; then
|
|
echo " ✅ Metadata sidecar written: $(basename "$meta_path")"
|
|
else
|
|
echo " ⚠️ Metadata sidecar write failed"
|
|
return 1
|
|
fi
|
|
|
|
# Best-effort mirror to whichever tiers actually succeeded — never fatal,
|
|
# the local sidecar (already written above) is what matters most.
|
|
if [ "$TIER2_STATUS" = "ok" ]; then
|
|
scp -i "$VM_KEY" -P "$VM_PORT" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
|
|
"$meta_path" "${VM_USER}@${VM_HOST}:${VM_DEST}" &>/dev/null || true
|
|
fi
|
|
if [ "$TIER3_STATUS" = "ok" ] && [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
|
|
META_SCRIPT="/tmp/r2_meta_$$.py"
|
|
cat > "$META_SCRIPT" << PYEOF2
|
|
import boto3
|
|
client = boto3.client('s3', endpoint_url="${R2_ENDPOINT}",
|
|
aws_access_key_id="${AWS_ACCESS_KEY_ID}", aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}",
|
|
region_name='auto')
|
|
client.upload_file("$meta_path", "${R2_BUCKET}", "backups/${BACKUP_NAME}.tar.gz.meta.json")
|
|
PYEOF2
|
|
"$PYTHON_BIN" "$META_SCRIPT" &>/dev/null || true
|
|
rm -f "$META_SCRIPT"
|
|
fi
|
|
}
|
|
|
|
echo "========================================="
|
|
echo "📦 Starting k8s Backup: $BACKUP_NAME"
|
|
if [ "$APPS_FILTER" = "all" ]; then
|
|
echo " Apps: ALL ($ALL_APPS)"
|
|
else
|
|
echo " Apps: $APPS_FILTER"
|
|
fi
|
|
echo " Tiers: Local → VM → ☁ Cloudflare R2"
|
|
echo "========================================="
|
|
|
|
check_resources
|
|
mkdir -p "$BACKUP_DIR" "/root/backups"
|
|
cd "$BACKUP_DIR"
|
|
|
|
APP_COUNT=0
|
|
INCLUDED_APPS=() # apps actually attempted this run — feeds the .meta.json sidecar's apps_key
|
|
ERRORS=() # "app:step:detail" entries — any entry here makes the sidecar's status red
|
|
for app in $ALL_APPS; do
|
|
app_selected "$app" || continue
|
|
INCLUDED_APPS+=("$app")
|
|
ns="${NS[$app]}"
|
|
app_deploy="${APP_DEPLOY[$app]}"
|
|
app_ctr="${APP_CTR[$app]}"
|
|
db_deploy="${DB_DEPLOY[$app]}"
|
|
db_engine="${DB_ENGINE[$app]}"
|
|
db_name="${DB_NAME[$app]}"
|
|
pvc_path="${PVC_PATH[$app]}"
|
|
secret_name="${SECRET_NAME[$app]}"
|
|
db_user_key="${DB_USER_KEY[$app]}"
|
|
db_pass_key="${DB_PASS_KEY[$app]}"
|
|
|
|
echo ""
|
|
echo "-----------------------------------------"
|
|
echo "📁 [$app] namespace=$ns"
|
|
echo "-----------------------------------------"
|
|
|
|
APP_DIR="$BACKUP_DIR/$app"
|
|
mkdir -p "$APP_DIR"
|
|
|
|
# ---- 1. Manifests ----
|
|
echo -n " 📄 Manifests (deployment/service/configmap/ingress) ... "
|
|
kubectl get deployment,service,configmap,ingress -n "$ns" -l owner=ameni-boukattaya -o yaml \
|
|
> "$APP_DIR/manifests.yaml" 2>"$APP_DIR/manifests.err" \
|
|
&& { echo "✅"; rm -f "$APP_DIR/manifests.err"; } \
|
|
|| { echo "⚠️ FAILED (see manifests.err)"; ERRORS+=("$app:manifests:$(tr '\n' ' ' < "$APP_DIR/manifests.err" | cut -c1-200)"); }
|
|
|
|
# ---- 2. Secret (plaintext, encryption-at-rest deferred) ----
|
|
echo -n " 🔑 Secret ($secret_name) ... "
|
|
kubectl get secret "$secret_name" -n "$ns" -o yaml \
|
|
> "$APP_DIR/secret.yaml" 2>"$APP_DIR/secret.err" \
|
|
&& { echo "✅"; rm -f "$APP_DIR/secret.err"; } \
|
|
|| { echo "⚠️ FAILED (see secret.err)"; ERRORS+=("$app:secret:$(tr '\n' ' ' < "$APP_DIR/secret.err" | cut -c1-200)"); }
|
|
|
|
# ---- 3. DB dump via kubectl exec (never raw datadir copy) ----
|
|
echo -n " 💾 DB dump ($db_engine) ... "
|
|
case "$db_engine" in
|
|
postgres)
|
|
db_user=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_user_key}}" 2>/dev/null | base64 -d)
|
|
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
|
if [ -n "$db_user" ] && [ -n "$db_pass" ]; then
|
|
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env PGPASSWORD="$db_pass" \
|
|
pg_dump -U "$db_user" -d "$db_name" 2>"$APP_DIR/db-dump.err" \
|
|
| gzip > "$APP_DIR/db-dump.sql.gz"
|
|
pg_dump_rc=${PIPESTATUS[0]}
|
|
if [ "$pg_dump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then
|
|
echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1))"
|
|
rm -f "$APP_DIR/db-dump.err"
|
|
else
|
|
echo "⚠️ FAILED (rc=$pg_dump_rc, see db-dump.err)"
|
|
ERRORS+=("$app:db_dump:rc=$pg_dump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)")
|
|
fi
|
|
else
|
|
echo "⚠️ could not read DB credentials from secret"
|
|
ERRORS+=("$app:db_dump:could not read DB credentials from secret")
|
|
fi
|
|
;;
|
|
mariadb)
|
|
if [ "$db_name" = "DYNAMIC" ]; then
|
|
# Frappe: real db_name/db_password live in site_config.json on the
|
|
# sites PVC, not in the k8s Secret. Dump with root instead, but still
|
|
# need the site's real db_name to target the right database.
|
|
site_dir=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
|
|
sh -c 'for d in sites/*/; do [ -f "$d/site_config.json" ] && [ ! -L "${d%/}" ] && basename "$d" && break; done' 2>/dev/null)
|
|
db_name=$(kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
|
|
cat "sites/${site_dir}/site_config.json" 2>/dev/null \
|
|
| python3 -c 'import json,sys; print(json.load(sys.stdin)["db_name"])' 2>/dev/null)
|
|
fi
|
|
db_pass=$(kubectl get secret "$secret_name" -n "$ns" -o jsonpath="{.data.${db_pass_key}}" 2>/dev/null | base64 -d)
|
|
if [ -n "$db_name" ] && [ -n "$db_pass" ]; then
|
|
kubectl exec -n "$ns" "deploy/${db_deploy}" -- env MYSQL_PWD="$db_pass" \
|
|
mysqldump -uroot --databases "$db_name" 2>"$APP_DIR/db-dump.err" \
|
|
| gzip > "$APP_DIR/db-dump.sql.gz"
|
|
mysqldump_rc=${PIPESTATUS[0]}
|
|
if [ "$mysqldump_rc" -eq 0 ] && [ -s "$APP_DIR/db-dump.sql.gz" ]; then
|
|
echo "✅ ($(du -h "$APP_DIR/db-dump.sql.gz" | cut -f1), db=$db_name)"
|
|
echo "$db_name" > "$APP_DIR/db-name.txt"
|
|
rm -f "$APP_DIR/db-dump.err"
|
|
else
|
|
echo "⚠️ FAILED (rc=$mysqldump_rc, see db-dump.err)"
|
|
ERRORS+=("$app:db_dump:rc=$mysqldump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)")
|
|
fi
|
|
else
|
|
echo "⚠️ could not determine DB name/credentials"
|
|
ERRORS+=("$app:db_dump:could not determine DB name/credentials")
|
|
fi
|
|
;;
|
|
esac
|
|
|
|
# ---- 4. App-data PVC contents via kubectl exec (live tar, app-data only,
|
|
# never raw filesystem/PVC-path access — §5 sandbox-divergence rule) ----
|
|
exclude_args=()
|
|
if [ -n "${TAR_EXCLUDES[$app]:-}" ]; then
|
|
for _excl_dir in ${TAR_EXCLUDES[$app]}; do
|
|
exclude_args+=(--exclude="./$_excl_dir")
|
|
done
|
|
fi
|
|
echo -n " 📦 PVC data ($pvc_path${exclude_args:+, excl: ${TAR_EXCLUDES[$app]}}) ... "
|
|
kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \
|
|
tar czf - "${exclude_args[@]}" -C "$pvc_path" . 2>"$APP_DIR/pvc-data.err" \
|
|
> "$APP_DIR/pvc-data.tar.gz"
|
|
if [ -s "$APP_DIR/pvc-data.tar.gz" ]; then
|
|
echo "✅ ($(du -h "$APP_DIR/pvc-data.tar.gz" | cut -f1))"
|
|
rm -f "$APP_DIR/pvc-data.err"
|
|
else
|
|
echo "⚠️ FAILED (see pvc-data.err)"
|
|
ERRORS+=("$app:pvc_data:$(tr '\n' ' ' < "$APP_DIR/pvc-data.err" 2>/dev/null | cut -c1-160)")
|
|
fi
|
|
|
|
APP_COUNT=$((APP_COUNT + 1))
|
|
|
|
# Resource check between apps, not just at the start (§5: check every
|
|
# 1-2 min during long-running/heavy ops; PVC tars can be sizeable).
|
|
check_resources
|
|
done
|
|
|
|
# --------------------------------------------------
|
|
# Metadata + checksum
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "📝 Writing backup metadata..."
|
|
cat > "$BACKUP_DIR/backup-info.txt" << EOF
|
|
Backup Name: $BACKUP_NAME
|
|
Backup Date: $(date)
|
|
Hostname: $(hostname)
|
|
Apps: $([ "$APPS_FILTER" = "all" ] && echo "$ALL_APPS" || echo "$APPS_FILTER")
|
|
Apps backed up: $APP_COUNT
|
|
kubectl: $(kubectl version --client -o yaml 2>/dev/null | grep gitVersion | head -1 || echo 'N/A')
|
|
Storage Tiers:
|
|
- Local: /root/backups/
|
|
- VM: ${VM_HOST}:${VM_PORT} → ${VM_DEST}
|
|
- Cloud: Cloudflare R2 → s3://${R2_BUCKET}/backups/
|
|
EOF
|
|
|
|
echo "" >> "$BACKUP_DIR/backup-info.txt"
|
|
echo "Per-app checksums:" >> "$BACKUP_DIR/backup-info.txt"
|
|
find "$BACKUP_DIR" -type f \( -name "*.tar.gz" -o -name "*.sql.gz" \) -print0 \
|
|
| xargs -0 -r sha256sum >> "$BACKUP_DIR/backup-info.txt"
|
|
echo " ✅ Done"
|
|
|
|
# --------------------------------------------------
|
|
# Compress (identical logic to backup-myapps.sh)
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "🗜️ Compressing backup..."
|
|
cd /root/backups
|
|
|
|
tar -czf "${BACKUP_NAME}.tar.gz" "${BACKUP_NAME}/" 2>/dev/null
|
|
if [ $? -ne 0 ]; then
|
|
echo " ❌ CRITICAL: Compression failed!"
|
|
log_status "FAILED" "$BACKUP_NAME" "compression_failed"
|
|
exit 1
|
|
fi
|
|
|
|
COMPRESSED_SIZE=$(du -h "${BACKUP_NAME}.tar.gz" | cut -f1)
|
|
echo " ✅ Compressed size: $COMPRESSED_SIZE → $BACKUP_ARCHIVE"
|
|
|
|
sha256sum "${BACKUP_NAME}.tar.gz" > "${BACKUP_NAME}.tar.gz.sha256"
|
|
echo " ✅ Checksum written"
|
|
|
|
rm -rf "$BACKUP_DIR"
|
|
TIER1_STATUS="ok:${COMPRESSED_SIZE}"
|
|
echo ""
|
|
echo "✅ [TIER 1] Local backup complete: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"
|
|
|
|
# --------------------------------------------------
|
|
# Retention — keep only MAX_BACKUPS locally (own glob, own lineage)
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "🧹 [Retention] Keeping latest ${MAX_BACKUPS} local k8s backups..."
|
|
ARCHIVE_LIST=$(ls -t /root/backups/myapps-k8s-backup-*.tar.gz 2>/dev/null || true)
|
|
ARCHIVE_COUNT=$(echo "$ARCHIVE_LIST" | grep -c '.tar.gz' 2>/dev/null || echo "0")
|
|
|
|
if [ "$ARCHIVE_COUNT" -gt "$MAX_BACKUPS" ]; then
|
|
TO_DELETE=$(echo "$ARCHIVE_LIST" | tail -n +$((MAX_BACKUPS + 1)))
|
|
while IFS= read -r old_file; do
|
|
[ -z "$old_file" ] && continue
|
|
rm -f "$old_file" "${old_file}.sha256" "${old_file}.meta.json"
|
|
echo " 🗑️ Deleted: $(basename "$old_file")"
|
|
done <<< "$TO_DELETE"
|
|
else
|
|
echo " ✅ ${ARCHIVE_COUNT}/${MAX_BACKUPS} backups — nothing to prune"
|
|
fi
|
|
|
|
# --------------------------------------------------
|
|
# VM transfer [TIER 2] — non-fatal (identical to backup-myapps.sh)
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "📤 [TIER 2] Sending backup to VM (${VM_HOST}:${VM_PORT})..."
|
|
|
|
scp -i "$VM_KEY" -P "$VM_PORT" \
|
|
-o StrictHostKeyChecking=no -o ConnectTimeout=30 \
|
|
"${BACKUP_NAME}.tar.gz" \
|
|
"${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null
|
|
VM_SCP_RC=$?
|
|
|
|
if [ $VM_SCP_RC -eq 0 ]; then
|
|
echo " ✅ Backup sent to VM successfully!"
|
|
scp -i "$VM_KEY" -P "$VM_PORT" \
|
|
-o StrictHostKeyChecking=no -o ConnectTimeout=15 \
|
|
"${BACKUP_NAME}.tar.gz.sha256" \
|
|
"${VM_USER}@${VM_HOST}:${VM_DEST}" 2>/dev/null || true
|
|
TIER2_STATUS="ok"
|
|
|
|
# VM-side retention — this tier never had any pruning at all until
|
|
# found accumulating 85 legacy pre-k8s archives (131GB) untouched
|
|
# since May. Mirrors the local [TIER 1] policy above: same
|
|
# MAX_BACKUPS, own glob/lineage only (never touches other
|
|
# unrelated backup dirs also living on this VM).
|
|
ssh -i "$VM_KEY" -p "$VM_PORT" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
|
|
"${VM_USER}@${VM_HOST}" \
|
|
"cd ${VM_DEST} && ls -t myapps-k8s-backup-*.tar.gz 2>/dev/null | tail -n +$((MAX_BACKUPS + 1)) | xargs -r -I{} rm -f {} {}.sha256 {}.meta.json" \
|
|
&>/dev/null || true
|
|
else
|
|
echo " ⚠️ VM transfer failed (rc=$VM_SCP_RC) — local backup is safe, continuing..."
|
|
TIER2_STATUS="failed:rc=${VM_SCP_RC}"
|
|
fi
|
|
|
|
# --------------------------------------------------
|
|
# Cloudflare R2 [TIER 3] — non-fatal (identical to backup-myapps.sh,
|
|
# key prefix stays "backups/" — shared with the legacy lineage in the
|
|
# same bucket, distinguished by filename prefix)
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "☁️ [TIER 3] Uploading to Cloudflare R2..."
|
|
|
|
if [ -z "${AWS_ACCESS_KEY_ID:-}" ] || [ -z "${AWS_SECRET_ACCESS_KEY:-}" ]; then
|
|
echo " ⚠️ R2 credentials not found — skipping cloud upload"
|
|
TIER3_STATUS="skipped:no_credentials"
|
|
else
|
|
echo " Credentials: found (key=${AWS_ACCESS_KEY_ID:0:8}...)"
|
|
echo " Bucket: ${R2_BUCKET}"
|
|
echo " Endpoint: ${R2_ENDPOINT}"
|
|
|
|
PYTHON_BIN="/root/management-platform/venv/bin/python3"
|
|
if [ ! -f "$PYTHON_BIN" ]; then
|
|
PYTHON_BIN="python3"
|
|
fi
|
|
|
|
R2_SCRIPT="/tmp/r2_upload_$$.py"
|
|
cat > "$R2_SCRIPT" << PYEOF
|
|
import sys, os, boto3
|
|
from datetime import datetime, timezone
|
|
|
|
account_id = "${R2_ACCOUNT_ID}"
|
|
bucket = "${R2_BUCKET}"
|
|
endpoint = "https://${R2_ACCOUNT_ID}.r2.cloudflarestorage.com"
|
|
access_key = "${AWS_ACCESS_KEY_ID}"
|
|
secret_key = "${AWS_SECRET_ACCESS_KEY}"
|
|
archive = "/root/backups/${BACKUP_NAME}.tar.gz"
|
|
sha_file = archive + '.sha256'
|
|
key = "backups/${BACKUP_NAME}.tar.gz"
|
|
max_r2 = ${MAX_R2_BACKUPS}
|
|
name_prefix = "backups/myapps-k8s-backup-"
|
|
|
|
try:
|
|
client = boto3.client(
|
|
's3',
|
|
endpoint_url=endpoint,
|
|
aws_access_key_id=access_key,
|
|
aws_secret_access_key=secret_key,
|
|
region_name='auto'
|
|
)
|
|
|
|
try:
|
|
client.head_bucket(Bucket=bucket)
|
|
except Exception:
|
|
client.create_bucket(Bucket=bucket)
|
|
print(f' Created bucket: {bucket}')
|
|
|
|
size_mb = os.path.getsize(archive) / 1024 / 1024
|
|
timestamp = datetime.now(timezone.utc).isoformat()
|
|
print(f' Uploading {size_mb:.1f} MB to r2://{bucket}/{key}')
|
|
client.upload_file(
|
|
archive, bucket, key,
|
|
ExtraArgs={"Metadata": {
|
|
"uploaded-by": "backup-k8s-apps-script",
|
|
"uploaded-at": timestamp,
|
|
"original-file": archive.split("/")[-1],
|
|
}}
|
|
)
|
|
print(' ✅ R2 archive uploaded successfully')
|
|
|
|
if os.path.exists(sha_file):
|
|
client.upload_file(sha_file, bucket, key + '.sha256')
|
|
print(' ✅ R2 SHA256 checksum uploaded')
|
|
|
|
# Retention scoped to this script's own lineage (name_prefix) only —
|
|
# never touches the legacy myapps-backup-* objects in the same bucket.
|
|
print(f' 🧹 Enforcing R2 retention (max {max_r2}, prefix={name_prefix})...')
|
|
resp = client.list_objects_v2(Bucket=bucket, Prefix=name_prefix)
|
|
objects = sorted(
|
|
[o for o in resp.get('Contents', []) if not o['Key'].endswith('.sha256')],
|
|
key=lambda x: x['LastModified'],
|
|
reverse=True
|
|
)
|
|
to_delete = objects[max_r2:]
|
|
if not to_delete:
|
|
print(f' ✅ {len(objects)}/{max_r2} R2 backups — nothing to prune')
|
|
for obj in to_delete:
|
|
old_key = obj['Key']
|
|
client.delete_object(Bucket=bucket, Key=old_key)
|
|
for suffix in ('.sha256', '.meta.json'):
|
|
try:
|
|
client.delete_object(Bucket=bucket, Key=old_key + suffix)
|
|
except Exception:
|
|
pass
|
|
print(f' 🗑️ Deleted from R2: {old_key.replace("backups/", "")}')
|
|
|
|
sys.exit(0)
|
|
|
|
except Exception as e:
|
|
print(f' ERROR: {e}')
|
|
sys.exit(2)
|
|
PYEOF
|
|
|
|
"$PYTHON_BIN" "$R2_SCRIPT"
|
|
R2_RC=$?
|
|
rm -f "$R2_SCRIPT"
|
|
|
|
if [ $R2_RC -eq 0 ]; then
|
|
echo " ✅ R2 upload + retention complete"
|
|
TIER3_STATUS="ok"
|
|
else
|
|
echo " ⚠️ R2 upload failed (rc=$R2_RC) — local backup is safe at: $BACKUP_ARCHIVE"
|
|
TIER3_STATUS="failed:rc=${R2_RC}"
|
|
fi
|
|
fi
|
|
|
|
# --------------------------------------------------
|
|
# Status metadata sidecar — after all 3 tiers are known
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "📝 Writing status metadata sidecar..."
|
|
write_meta_sidecar
|
|
|
|
# --------------------------------------------------
|
|
# Final summary + log
|
|
# --------------------------------------------------
|
|
echo ""
|
|
echo "========================================="
|
|
echo "✅ K8S BACKUP COMPLETE"
|
|
echo ""
|
|
echo " 📦 Name: $BACKUP_NAME"
|
|
echo " 💾 Local: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"
|
|
echo " 🖥️ VM: ${TIER2_STATUS}"
|
|
echo " ☁️ R2: ${TIER3_STATUS}"
|
|
echo "========================================="
|
|
|
|
log_status "SUCCESS" "$BACKUP_NAME" \
|
|
"apps=${APP_COUNT} | size=${COMPRESSED_SIZE} | tier1=${TIER1_STATUS} | tier2=${TIER2_STATUS} | tier3=${TIER3_STATUS}"
|