From 995e07c8947c6028ea84c4a31fdda34d176070f0 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 21 Aug 2026 12:37:16 +0200 Subject: [PATCH] Fix two restore-k8s-apps.sh bugs found verifying Nextcloud restore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Secret apply conflict: secret.yaml is a raw `kubectl get -o yaml` dump that includes the live resourceVersion/uid/creationTimestamp at capture time. Applying a stale resourceVersion trips optimistic-concurrency control ("the object has been modified") on effectively every restore, since normal cluster activity keeps bumping it. Strip those three volatile fields before `kubectl apply`. 2. Wrong re-seed source for Nextcloud: the re-seed step added for the PVC exclusion fix assumed both Mautic and Nextcloud bake their excluded dirs in at their own PVC mount path (/var/www/html), copying from pvc_path/$dir. True for Mautic, wrong for Nextcloud — its image keeps the shipped source tree at /usr/src/nextcloud, completely separate from /var/www/html (empty in the raw image at that path). Caught in testing ("cp: cannot stat '/var/www/html/apps'") because the reseed_failed gate from the previous fix did its job and left the app at 0 replicas instead of booting broken — but still required a live manual fix to bring Nextcloud back after the failed test. New RECONSTRUCT_SRC table makes the copy source explicit and independently verified per app instead of assumed by analogy: confirmed /usr/src/nextcloud/{apps,core,dist, 3rdparty,lib} exist in a bare nextcloud:32 pod with no volume mounted before relying on it. Verified end-to-end after both fixes: fresh Nextcloud backup -> restore -> data/ intact (184M, 89 files, unchanged) -> occ status installed/healthy -> 200 on /status.php, via the actual script run (not just the manual recovery), confirming the automated path works, not just my live fix. --- backup/restore-k8s-apps.sh | 30 ++++++++++++++++++++++++++++-- 1 file changed, 28 insertions(+), 2 deletions(-) diff --git a/backup/restore-k8s-apps.sh b/backup/restore-k8s-apps.sh index 5eea5ed..18c7e5f 100755 --- a/backup/restore-k8s-apps.sh +++ b/backup/restore-k8s-apps.sh @@ -119,7 +119,21 @@ declare -A DOMAIN=( [n8n]="n8nwf.nav.ovh" [odoo]="odooo.nav.ovh" [mautic]=" # of any volume mount, so cp'ing from a loader pod running the same image # (PVC mounted elsewhere, so it doesn't shadow the image's own copy) is # reliable regardless of either app's startup logic. +# +# RECONSTRUCT_SRC is the path *inside the image* to copy from — NOT +# necessarily pvc_path. Verified per app by running the actual image with no +# volume mounted and checking where it bakes the dirs in: +# - Mautic bakes node_modules/vendor directly at /var/www/html (its own +# PVC mount path) — same path, source == destination parent. +# - Nextcloud's image keeps its shipped source tree at /usr/src/nextcloud +# (confirmed via `du -sh` in a bare nextcloud:32 pod), completely +# separate from /var/www/html (the PVC mount, empty at that path in the +# image). Assuming pvc_path here (mirroring Mautic's layout) was wrong +# and failed loudly in testing — cp: cannot stat '/var/www/html/apps' — +# caught by the reseed_failed gate above rather than silently booting +# nextcloud broken. declare -A RECONSTRUCT_DIRS=( [mautic]="node_modules vendor" [nextcloud]="apps core dist 3rdparty lib" ) +declare -A RECONSTRUCT_SRC=( [mautic]="/var/www/html" [nextcloud]="/usr/src/nextcloud" ) ALL_APPS="n8n odoo mautic nextcloud frappe" LOADER_IMAGE="alpine:3.20" @@ -186,7 +200,18 @@ for app in $ALL_APPS; do # restored here already exists in a same-cluster restore) ---- if [ -f "$APP_DIR/secret.yaml" ]; then echo -n " 🔑 Applying Secret ... " - kubectl apply -f "$APP_DIR/secret.yaml" &>/dev/null && echo "✅" || { echo "⚠️ FAILED"; app_ok=false; } + # secret.yaml is a raw `kubectl get -o yaml` dump, which includes the + # live resourceVersion/uid/creationTimestamp at capture time. Passing + # a stale resourceVersion to `apply` trips optimistic-concurrency + # control ("the object has been modified") on any restore where the + # secret's real resourceVersion has since moved on — which is every + # restore, since normal cluster activity (this backup pipeline's own + # earlier applies included) bumps it constantly. Strip the volatile + # fields before applying; they're server-assigned and never meant to + # round-trip through a backup. + sed -e '/^ resourceVersion:/d' -e '/^ uid:/d' -e '/^ creationTimestamp:/d' \ + "$APP_DIR/secret.yaml" > "$APP_DIR/secret.apply.yaml" + kubectl apply -f "$APP_DIR/secret.apply.yaml" &>/dev/null && echo "✅" || { echo "⚠️ FAILED"; app_ok=false; } fi # NOTE: manifests.yaml (Deployment/Service/ConfigMap/Ingress specs) is # captured by backup-k8s-apps.sh but deliberately NOT re-applied here. @@ -258,8 +283,9 @@ for app in $ALL_APPS; do if kubectl wait --for=condition=Ready "pod/$seed_loader" -n "$ns" --timeout=240s &>/dev/null; then seed_ok=true : > "$APP_DIR/reseed.err" + seed_src="${RECONSTRUCT_SRC[$app]}" for _seed_dir in ${RECONSTRUCT_DIRS[$app]}; do - kubectl exec -n "$ns" "$seed_loader" -- sh -c "rm -rf '/restore/$_seed_dir' && cp -a '${pvc_path}/${_seed_dir}' '/restore/$_seed_dir'" \ + kubectl exec -n "$ns" "$seed_loader" -- sh -c "rm -rf '/restore/$_seed_dir' && cp -a '${seed_src}/${_seed_dir}' '/restore/$_seed_dir'" \ >>"$APP_DIR/reseed.err" 2>&1 || seed_ok=false done if $seed_ok; then