Add per-backup status dot (green/yellow/red/unknown) to the UI
backup-k8s-apps.sh now tracks step-level failures (manifests, secret,
db_dump, pvc_data) into an ERRORS array during the per-app loop, and
writes a .meta.json sidecar alongside each archive once all 3 storage
tiers are known - apps included, final size, the error list, and
per-tier ok/failed/skipped status. Sidecar is mirrored to VM/R2 the
same best-effort way the .sha256 sidecar already is, and cleaned up
by both local and R2 retention pruning plus manual delete.
modules/backups.py reads these sidecars (never decompresses the
archive) to compute a status per backup at list-render time:
- red: any recorded error, or any tier status starting with "failed"
- yellow: no errors, but size deviates >40% from the rolling average
of the last 5 backups sharing the same app-combination (apps_key)
- green: no errors, size within range (or first backup of its
app-combination - nothing to compare against yet)
- unknown: no sidecar at all (legacy myapps-backup-* archives, or
any k8s backup made before this shipped) - no backfill attempted,
old runs never recorded step-level failures to reconstruct from
/backups route now passes get_local_backups_with_status()/
get_vm_backups_with_status() instead of the plain filename lists
(get_local_backups()/get_vm_backups() themselves are untouched -
/restore and /api/backups still use the plain versions, they don't
need the dot). Template renders a colored dot next to each entry with
a tooltip showing apps/size.
Verified: real n8n backup produces a correct sidecar synced to all 3
tiers; JSON-writer argv parsing and the red/yellow/green/unknown
decision logic each checked against synthetic cases; full Jinja render
checked against real local + VM data pulled from the live pod.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017avLHFqkiti3g62Anq9sVA
This commit is contained in:
root
@@ -174,6 +174,83 @@ check_resources() {
|
||||
fi
|
||||
}
|
||||
|
||||
# --------------------------------------------------
|
||||
# Per-backup status metadata sidecar (.meta.json) — powers the UI's
|
||||
# green/yellow/red status dot without ever needing to decompress the
|
||||
# (possibly 600MB+) archive itself. Written only after all 3 tiers are
|
||||
# known, then best-effort mirrored to VM/R2 alongside the archive, same
|
||||
# as the .sha256 sidecar already is — so status is readable from
|
||||
# whichever tier the UI is listing, not just local.
|
||||
# --------------------------------------------------
|
||||
write_meta_sidecar() {
|
||||
local size_bytes meta_path
|
||||
size_bytes=$(stat -c%s "$BACKUP_ARCHIVE" 2>/dev/null || echo 0)
|
||||
meta_path="/root/backups/${BACKUP_NAME}.tar.gz.meta.json"
|
||||
|
||||
python3 - "$BACKUP_NAME" "$size_bytes" "$TIER1_STATUS" "$TIER2_STATUS" "$TIER3_STATUS" \
|
||||
"${#INCLUDED_APPS[@]}" "${INCLUDED_APPS[@]}" "${ERRORS[@]}" > "$meta_path" <<'PYEOF'
|
||||
import sys, json, datetime
|
||||
|
||||
backup_name = sys.argv[1]
|
||||
size_bytes = int(sys.argv[2])
|
||||
tier1, tier2, tier3 = sys.argv[3], sys.argv[4], sys.argv[5]
|
||||
n_apps = int(sys.argv[6])
|
||||
apps = sys.argv[7:7 + n_apps]
|
||||
errors_raw = sys.argv[7 + n_apps:]
|
||||
|
||||
def norm_tier(s):
|
||||
# "ok:572M" -> "ok"; "failed:rc=2" / "skipped:no_credentials" pass through
|
||||
# unchanged so the UI can tell a real failure apart from a deliberate skip.
|
||||
return 'ok' if s.startswith('ok') else s
|
||||
|
||||
errors = []
|
||||
for e in errors_raw:
|
||||
parts = e.split(':', 2)
|
||||
errors.append({
|
||||
'app': parts[0] if len(parts) > 0 else '',
|
||||
'step': parts[1] if len(parts) > 1 else '',
|
||||
'detail': parts[2] if len(parts) > 2 else '',
|
||||
})
|
||||
|
||||
meta = {
|
||||
'backup_name': backup_name,
|
||||
'created_at': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds'),
|
||||
'apps': apps,
|
||||
'apps_key': ','.join(sorted(apps)),
|
||||
'size_bytes': size_bytes,
|
||||
'errors': errors,
|
||||
'tiers': {'local': norm_tier(tier1), 'vm': norm_tier(tier2), 'r2': norm_tier(tier3)},
|
||||
}
|
||||
json.dump(meta, sys.stdout, indent=2)
|
||||
PYEOF
|
||||
|
||||
if [ -s "$meta_path" ]; then
|
||||
echo " ✅ Metadata sidecar written: $(basename "$meta_path")"
|
||||
else
|
||||
echo " ⚠️ Metadata sidecar write failed"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# Best-effort mirror to whichever tiers actually succeeded — never fatal,
|
||||
# the local sidecar (already written above) is what matters most.
|
||||
if [ "$TIER2_STATUS" = "ok" ]; then
|
||||
scp -i "$VM_KEY" -P "$VM_PORT" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \
|
||||
"$meta_path" "${VM_USER}@${VM_HOST}:${VM_DEST}" &>/dev/null || true
|
||||
fi
|
||||
if [ "$TIER3_STATUS" = "ok" ] && [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
|
||||
META_SCRIPT="/tmp/r2_meta_$$.py"
|
||||
cat > "$META_SCRIPT" << PYEOF2
|
||||
import boto3
|
||||
client = boto3.client('s3', endpoint_url="${R2_ENDPOINT}",
|
||||
aws_access_key_id="${AWS_ACCESS_KEY_ID}", aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}",
|
||||
region_name='auto')
|
||||
client.upload_file("$meta_path", "${R2_BUCKET}", "backups/${BACKUP_NAME}.tar.gz.meta.json")
|
||||
PYEOF2
|
||||
"$PYTHON_BIN" "$META_SCRIPT" &>/dev/null || true
|
||||
rm -f "$META_SCRIPT"
|
||||
fi
|
||||
}
|
||||
|
||||
echo "========================================="
|
||||
echo "📦 Starting k8s Backup: $BACKUP_NAME"
|
||||
if [ "$APPS_FILTER" = "all" ]; then
|
||||
@@ -189,8 +266,11 @@ mkdir -p "$BACKUP_DIR" "/root/backups"
|
||||
cd "$BACKUP_DIR"
|
||||
|
||||
APP_COUNT=0
|
||||
INCLUDED_APPS=() # apps actually attempted this run — feeds the .meta.json sidecar's apps_key
|
||||
ERRORS=() # "app:step:detail" entries — any entry here makes the sidecar's status red
|
||||
for app in $ALL_APPS; do
|
||||
app_selected "$app" || continue
|
||||
INCLUDED_APPS+=("$app")
|
||||
ns="${NS[$app]}"
|
||||
app_deploy="${APP_DEPLOY[$app]}"
|
||||
app_ctr="${APP_CTR[$app]}"
|
||||
@@ -214,13 +294,15 @@ for app in $ALL_APPS; do
|
||||
echo -n " 📄 Manifests (deployment/service/configmap/ingress) ... "
|
||||
kubectl get deployment,service,configmap,ingress -n "$ns" -l owner=ameni-boukattaya -o yaml \
|
||||
> "$APP_DIR/manifests.yaml" 2>"$APP_DIR/manifests.err" \
|
||||
&& { echo "✅"; rm -f "$APP_DIR/manifests.err"; } || echo "⚠️ FAILED (see manifests.err)"
|
||||
&& { echo "✅"; rm -f "$APP_DIR/manifests.err"; } \
|
||||
|| { echo "⚠️ FAILED (see manifests.err)"; ERRORS+=("$app:manifests:$(tr '\n' ' ' < "$APP_DIR/manifests.err" | cut -c1-200)"); }
|
||||
|
||||
# ---- 2. Secret (plaintext, encryption-at-rest deferred) ----
|
||||
echo -n " 🔑 Secret ($secret_name) ... "
|
||||
kubectl get secret "$secret_name" -n "$ns" -o yaml \
|
||||
> "$APP_DIR/secret.yaml" 2>"$APP_DIR/secret.err" \
|
||||
&& { echo "✅"; rm -f "$APP_DIR/secret.err"; } || echo "⚠️ FAILED (see secret.err)"
|
||||
&& { echo "✅"; rm -f "$APP_DIR/secret.err"; } \
|
||||
|| { echo "⚠️ FAILED (see secret.err)"; ERRORS+=("$app:secret:$(tr '\n' ' ' < "$APP_DIR/secret.err" | cut -c1-200)"); }
|
||||
|
||||
# ---- 3. DB dump via kubectl exec (never raw datadir copy) ----
|
||||
echo -n " 💾 DB dump ($db_engine) ... "
|
||||
@@ -238,9 +320,11 @@ for app in $ALL_APPS; do
|
||||
rm -f "$APP_DIR/db-dump.err"
|
||||
else
|
||||
echo "⚠️ FAILED (rc=$pg_dump_rc, see db-dump.err)"
|
||||
ERRORS+=("$app:db_dump:rc=$pg_dump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)")
|
||||
fi
|
||||
else
|
||||
echo "⚠️ could not read DB credentials from secret"
|
||||
ERRORS+=("$app:db_dump:could not read DB credentials from secret")
|
||||
fi
|
||||
;;
|
||||
mariadb)
|
||||
@@ -266,9 +350,11 @@ for app in $ALL_APPS; do
|
||||
rm -f "$APP_DIR/db-dump.err"
|
||||
else
|
||||
echo "⚠️ FAILED (rc=$mysqldump_rc, see db-dump.err)"
|
||||
ERRORS+=("$app:db_dump:rc=$mysqldump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)")
|
||||
fi
|
||||
else
|
||||
echo "⚠️ could not determine DB name/credentials"
|
||||
ERRORS+=("$app:db_dump:could not determine DB name/credentials")
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
@@ -284,6 +370,7 @@ for app in $ALL_APPS; do
|
||||
rm -f "$APP_DIR/pvc-data.err"
|
||||
else
|
||||
echo "⚠️ FAILED (see pvc-data.err)"
|
||||
ERRORS+=("$app:pvc_data:$(tr '\n' ' ' < "$APP_DIR/pvc-data.err" 2>/dev/null | cut -c1-160)")
|
||||
fi
|
||||
|
||||
APP_COUNT=$((APP_COUNT + 1))
|
||||
@@ -354,7 +441,7 @@ if [ "$ARCHIVE_COUNT" -gt "$MAX_BACKUPS" ]; then
|
||||
TO_DELETE=$(echo "$ARCHIVE_LIST" | tail -n +$((MAX_BACKUPS + 1)))
|
||||
while IFS= read -r old_file; do
|
||||
[ -z "$old_file" ] && continue
|
||||
rm -f "$old_file" "${old_file}.sha256"
|
||||
rm -f "$old_file" "${old_file}.sha256" "${old_file}.meta.json"
|
||||
echo " 🗑️ Deleted: $(basename "$old_file")"
|
||||
done <<< "$TO_DELETE"
|
||||
else
|
||||
@@ -469,10 +556,11 @@ try:
|
||||
for obj in to_delete:
|
||||
old_key = obj['Key']
|
||||
client.delete_object(Bucket=bucket, Key=old_key)
|
||||
try:
|
||||
client.delete_object(Bucket=bucket, Key=old_key + '.sha256')
|
||||
except Exception:
|
||||
pass
|
||||
for suffix in ('.sha256', '.meta.json'):
|
||||
try:
|
||||
client.delete_object(Bucket=bucket, Key=old_key + suffix)
|
||||
except Exception:
|
||||
pass
|
||||
print(f' 🗑️ Deleted from R2: {old_key.replace("backups/", "")}')
|
||||
|
||||
sys.exit(0)
|
||||
@@ -495,6 +583,13 @@ PYEOF
|
||||
fi
|
||||
fi
|
||||
|
||||
# --------------------------------------------------
|
||||
# Status metadata sidecar — after all 3 tiers are known
|
||||
# --------------------------------------------------
|
||||
echo ""
|
||||
echo "📝 Writing status metadata sidecar..."
|
||||
write_meta_sidecar
|
||||
|
||||
# --------------------------------------------------
|
||||
# Final summary + log
|
||||
# --------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user