diff --git a/backup/backup-k8s-apps.sh b/backup/backup-k8s-apps.sh index 9b13fef..dc0550f 100755 --- a/backup/backup-k8s-apps.sh +++ b/backup/backup-k8s-apps.sh @@ -96,6 +96,15 @@ declare -A SECRET_NAME=( [n8n]=n8n-secrets [odoo]=odoo-secrets [mautic]=ma declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]=MYSQL_USER [nextcloud]=POSTGRES_USER [frappe]="" ) declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD ) +# Reproducible framework/vendor code that lives on the same PVC as real user +# data for these two apps (single shared mount, no separate volume). Both +# images bake these dirs into their own filesystem layer independent of any +# volume โ€” Mautic has no logic to reconstruct them, so restore-k8s-apps.sh +# re-seeds them straight from the app's own image instead of from backup. +# Never pull request/exclude changes here without updating the matching +# RECONSTRUCT_DIRS table in restore-k8s-apps.sh. +declare -A TAR_EXCLUDES=( [mautic]="node_modules vendor" [nextcloud]="apps core dist 3rdparty lib" ) + ALL_APPS="n8n odoo mautic nextcloud frappe" BACKUP_DATE=$(date +%Y%m%d_%H%M%S) @@ -361,9 +370,15 @@ for app in $ALL_APPS; do # ---- 4. App-data PVC contents via kubectl exec (live tar, app-data only, # never raw filesystem/PVC-path access โ€” ยง5 sandbox-divergence rule) ---- - echo -n " ๐Ÿ“ฆ PVC data ($pvc_path) ... " + exclude_args=() + if [ -n "${TAR_EXCLUDES[$app]:-}" ]; then + for _excl_dir in ${TAR_EXCLUDES[$app]}; do + exclude_args+=(--exclude="./$_excl_dir") + done + fi + echo -n " ๐Ÿ“ฆ PVC data ($pvc_path${exclude_args:+, excl: ${TAR_EXCLUDES[$app]}}) ... " kubectl exec -n "$ns" "deploy/${app_deploy}" -c "$app_ctr" -- \ - tar czf - -C "$pvc_path" . 2>"$APP_DIR/pvc-data.err" \ + tar czf - "${exclude_args[@]}" -C "$pvc_path" . 2>"$APP_DIR/pvc-data.err" \ > "$APP_DIR/pvc-data.tar.gz" if [ -s "$APP_DIR/pvc-data.tar.gz" ]; then echo "โœ… ($(du -h "$APP_DIR/pvc-data.tar.gz" | cut -f1))" diff --git a/backup/restore-k8s-apps.sh b/backup/restore-k8s-apps.sh index 33d8535..5eea5ed 100755 --- a/backup/restore-k8s-apps.sh +++ b/backup/restore-k8s-apps.sh @@ -103,6 +103,24 @@ declare -A DB_USER_KEY=( [n8n]=DB_POSTGRESDB_USER [odoo]=POSTGRES_USER [mautic]= declare -A DB_PASS_KEY=( [n8n]=DB_POSTGRESDB_PASSWORD [odoo]=POSTGRES_PASSWORD [mautic]=MYSQL_ROOT_PASSWORD [nextcloud]=POSTGRES_PASSWORD [frappe]=MARIADB_ROOT_PASSWORD ) declare -A DOMAIN=( [n8n]="n8nwf.nav.ovh" [odoo]="odooo.nav.ovh" [mautic]="mautics.nav.ovh" [nextcloud]="next.cloud.nav.ovh" [frappe]="erpnext.navitrends.ovh" ) +# Dirs backup-k8s-apps.sh excludes from pvc-data.tar.gz for these two apps +# (reproducible framework/vendor code, not user data โ€” see matching +# TAR_EXCLUDES table + comment there). They must be re-seeded here from the +# app's own live image, NOT reconstructed by the container entrypoint: +# - Mautic's image has no logic at all to repopulate node_modules/vendor +# if missing โ€” confirmed empty by reading /entrypoint.sh. +# - Nextcloud's entrypoint only rsyncs from /usr/src/nextcloud when it +# detects image_version > installed_version (via version.php). Since +# version.php itself is NOT excluded (kept, "everything else"), a +# restored version.php already matches the running image's version, so +# that rsync path never fires โ€” relying on it would silently leave +# apps/core/dist/3rdparty/lib missing after restore. +# Both images bake these dirs into their own filesystem layer independent +# of any volume mount, so cp'ing from a loader pod running the same image +# (PVC mounted elsewhere, so it doesn't shadow the image's own copy) is +# reliable regardless of either app's startup logic. +declare -A RECONSTRUCT_DIRS=( [mautic]="node_modules vendor" [nextcloud]="apps core dist 3rdparty lib" ) + ALL_APPS="n8n odoo mautic nextcloud frappe" LOADER_IMAGE="alpine:3.20" @@ -216,9 +234,67 @@ for app in $ALL_APPS; do echo " โญ๏ธ No pvc-data.tar.gz in backup โ€” skipping PVC restore" fi + # ---- 3b. Re-seed reproducible dirs excluded from backup (see + # RECONSTRUCT_DIRS comment above) from the app's own live image, + # never from the backup archive itself ---- + reseed_failed=false + if [ -n "${RECONSTRUCT_DIRS[$app]:-}" ] && kubectl get pvc "$pvc_name" -n "$ns" &>/dev/null; then + app_image=$(kubectl get deployment "$app_deploy" -n "$ns" -o jsonpath='{.spec.template.spec.containers[0].image}' 2>/dev/null) + if [ -n "$app_image" ]; then + seed_loader="${app}-reseed-loader" + echo -n " ๐Ÿงฉ Re-seeding [${RECONSTRUCT_DIRS[$app]}] from $app_image ... " + # imagePullPolicy IfNotPresent: $app_image is read straight off the + # currently-running Deployment, so it's already on this node โ€” + # forcing a re-pull (the ":latest"/floating-tag default) just adds + # a slow, needless network fetch during every restore. + kubectl run "$seed_loader" -n "$ns" --image="$app_image" --restart=Never \ + --overrides="{\"spec\":{\"containers\":[{\"name\":\"loader\",\"image\":\"${app_image}\",\"imagePullPolicy\":\"IfNotPresent\",\"command\":[\"sleep\",\"3600\"],\"volumeMounts\":[{\"name\":\"data\",\"mountPath\":\"/restore\"}]}],\"volumes\":[{\"name\":\"data\",\"persistentVolumeClaim\":{\"claimName\":\"${pvc_name}\"}}]}}" \ + &>/dev/null + # 240s, not 60/90s like the alpine loader above: this pulls the + # full app image (600MB+ for mautic/nextcloud) if it isn't cached + # on this node yet โ€” a 90s timeout was observed to fire mid-pull + # in testing, silently leaving the app pod without its + # reconstructed dirs after being scaled back up. + if kubectl wait --for=condition=Ready "pod/$seed_loader" -n "$ns" --timeout=240s &>/dev/null; then + seed_ok=true + : > "$APP_DIR/reseed.err" + for _seed_dir in ${RECONSTRUCT_DIRS[$app]}; do + kubectl exec -n "$ns" "$seed_loader" -- sh -c "rm -rf '/restore/$_seed_dir' && cp -a '${pvc_path}/${_seed_dir}' '/restore/$_seed_dir'" \ + >>"$APP_DIR/reseed.err" 2>&1 || seed_ok=false + done + if $seed_ok; then + echo "โœ…" + rm -f "$APP_DIR/reseed.err" + else + echo "โš ๏ธ FAILED (see $APP_DIR/reseed.err)" + app_ok=false + reseed_failed=true + fi + else + echo "โš ๏ธ loader pod not ready" + app_ok=false + reseed_failed=true + fi + kubectl delete pod "$seed_loader" -n "$ns" --wait=false &>/dev/null + else + echo " โš ๏ธ Could not determine $app image โ€” skipping re-seed of [${RECONSTRUCT_DIRS[$app]}]" + app_ok=false + reseed_failed=true + fi + fi + # ---- 4. Scale app back up ---- - echo -n " โ–ถ๏ธ Scaling $app_deploy back to $prior_replicas ... " - kubectl scale deployment "$app_deploy" -n "$ns" --replicas="$prior_replicas" &>/dev/null && echo "โœ…" || echo "โš ๏ธ FAILED" + # Never bring the app back with replicas>0 if a required re-seed (3b) + # failed โ€” for Mautic/Nextcloud that means booting with vendor/ or + # core/lib missing, which is a hard crash/broken-app state, not a + # degraded one. Left at 0 so the operator fixes it and scales up + # manually rather than the restore silently serving a broken app. + if $reseed_failed; then + echo " โ›” Skipping scale-up โ€” required re-seed failed (app left at 0 replicas, see above)" + else + echo -n " โ–ถ๏ธ Scaling $app_deploy back to $prior_replicas ... " + kubectl scale deployment "$app_deploy" -n "$ns" --replicas="$prior_replicas" &>/dev/null && echo "โœ…" || echo "โš ๏ธ FAILED" + fi # ---- 5. DB restore (SQL-level, never raw datadir copy) ---- if [ -f "$APP_DIR/db-dump.sql.gz" ]; then