diff --git a/backup/backup-k8s-apps.sh b/backup/backup-k8s-apps.sh index b029442..9b13fef 100755 --- a/backup/backup-k8s-apps.sh +++ b/backup/backup-k8s-apps.sh @@ -174,6 +174,83 @@ check_resources() { fi } +# -------------------------------------------------- +# Per-backup status metadata sidecar (.meta.json) — powers the UI's +# green/yellow/red status dot without ever needing to decompress the +# (possibly 600MB+) archive itself. Written only after all 3 tiers are +# known, then best-effort mirrored to VM/R2 alongside the archive, same +# as the .sha256 sidecar already is — so status is readable from +# whichever tier the UI is listing, not just local. +# -------------------------------------------------- +write_meta_sidecar() { + local size_bytes meta_path + size_bytes=$(stat -c%s "$BACKUP_ARCHIVE" 2>/dev/null || echo 0) + meta_path="/root/backups/${BACKUP_NAME}.tar.gz.meta.json" + + python3 - "$BACKUP_NAME" "$size_bytes" "$TIER1_STATUS" "$TIER2_STATUS" "$TIER3_STATUS" \ + "${#INCLUDED_APPS[@]}" "${INCLUDED_APPS[@]}" "${ERRORS[@]}" > "$meta_path" <<'PYEOF' +import sys, json, datetime + +backup_name = sys.argv[1] +size_bytes = int(sys.argv[2]) +tier1, tier2, tier3 = sys.argv[3], sys.argv[4], sys.argv[5] +n_apps = int(sys.argv[6]) +apps = sys.argv[7:7 + n_apps] +errors_raw = sys.argv[7 + n_apps:] + +def norm_tier(s): + # "ok:572M" -> "ok"; "failed:rc=2" / "skipped:no_credentials" pass through + # unchanged so the UI can tell a real failure apart from a deliberate skip. + return 'ok' if s.startswith('ok') else s + +errors = [] +for e in errors_raw: + parts = e.split(':', 2) + errors.append({ + 'app': parts[0] if len(parts) > 0 else '', + 'step': parts[1] if len(parts) > 1 else '', + 'detail': parts[2] if len(parts) > 2 else '', + }) + +meta = { + 'backup_name': backup_name, + 'created_at': datetime.datetime.now(datetime.timezone.utc).isoformat(timespec='seconds'), + 'apps': apps, + 'apps_key': ','.join(sorted(apps)), + 'size_bytes': size_bytes, + 'errors': errors, + 'tiers': {'local': norm_tier(tier1), 'vm': norm_tier(tier2), 'r2': norm_tier(tier3)}, +} +json.dump(meta, sys.stdout, indent=2) +PYEOF + + if [ -s "$meta_path" ]; then + echo " ✅ Metadata sidecar written: $(basename "$meta_path")" + else + echo " ⚠️ Metadata sidecar write failed" + return 1 + fi + + # Best-effort mirror to whichever tiers actually succeeded — never fatal, + # the local sidecar (already written above) is what matters most. + if [ "$TIER2_STATUS" = "ok" ]; then + scp -i "$VM_KEY" -P "$VM_PORT" -o StrictHostKeyChecking=no -o ConnectTimeout=15 \ + "$meta_path" "${VM_USER}@${VM_HOST}:${VM_DEST}" &>/dev/null || true + fi + if [ "$TIER3_STATUS" = "ok" ] && [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then + META_SCRIPT="/tmp/r2_meta_$$.py" + cat > "$META_SCRIPT" << PYEOF2 +import boto3 +client = boto3.client('s3', endpoint_url="${R2_ENDPOINT}", + aws_access_key_id="${AWS_ACCESS_KEY_ID}", aws_secret_access_key="${AWS_SECRET_ACCESS_KEY}", + region_name='auto') +client.upload_file("$meta_path", "${R2_BUCKET}", "backups/${BACKUP_NAME}.tar.gz.meta.json") +PYEOF2 + "$PYTHON_BIN" "$META_SCRIPT" &>/dev/null || true + rm -f "$META_SCRIPT" + fi +} + echo "=========================================" echo "📦 Starting k8s Backup: $BACKUP_NAME" if [ "$APPS_FILTER" = "all" ]; then @@ -189,8 +266,11 @@ mkdir -p "$BACKUP_DIR" "/root/backups" cd "$BACKUP_DIR" APP_COUNT=0 +INCLUDED_APPS=() # apps actually attempted this run — feeds the .meta.json sidecar's apps_key +ERRORS=() # "app:step:detail" entries — any entry here makes the sidecar's status red for app in $ALL_APPS; do app_selected "$app" || continue + INCLUDED_APPS+=("$app") ns="${NS[$app]}" app_deploy="${APP_DEPLOY[$app]}" app_ctr="${APP_CTR[$app]}" @@ -214,13 +294,15 @@ for app in $ALL_APPS; do echo -n " 📄 Manifests (deployment/service/configmap/ingress) ... " kubectl get deployment,service,configmap,ingress -n "$ns" -l owner=ameni-boukattaya -o yaml \ > "$APP_DIR/manifests.yaml" 2>"$APP_DIR/manifests.err" \ - && { echo "✅"; rm -f "$APP_DIR/manifests.err"; } || echo "⚠️ FAILED (see manifests.err)" + && { echo "✅"; rm -f "$APP_DIR/manifests.err"; } \ + || { echo "⚠️ FAILED (see manifests.err)"; ERRORS+=("$app:manifests:$(tr '\n' ' ' < "$APP_DIR/manifests.err" | cut -c1-200)"); } # ---- 2. Secret (plaintext, encryption-at-rest deferred) ---- echo -n " 🔑 Secret ($secret_name) ... " kubectl get secret "$secret_name" -n "$ns" -o yaml \ > "$APP_DIR/secret.yaml" 2>"$APP_DIR/secret.err" \ - && { echo "✅"; rm -f "$APP_DIR/secret.err"; } || echo "⚠️ FAILED (see secret.err)" + && { echo "✅"; rm -f "$APP_DIR/secret.err"; } \ + || { echo "⚠️ FAILED (see secret.err)"; ERRORS+=("$app:secret:$(tr '\n' ' ' < "$APP_DIR/secret.err" | cut -c1-200)"); } # ---- 3. DB dump via kubectl exec (never raw datadir copy) ---- echo -n " 💾 DB dump ($db_engine) ... " @@ -238,9 +320,11 @@ for app in $ALL_APPS; do rm -f "$APP_DIR/db-dump.err" else echo "⚠️ FAILED (rc=$pg_dump_rc, see db-dump.err)" + ERRORS+=("$app:db_dump:rc=$pg_dump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)") fi else echo "⚠️ could not read DB credentials from secret" + ERRORS+=("$app:db_dump:could not read DB credentials from secret") fi ;; mariadb) @@ -266,9 +350,11 @@ for app in $ALL_APPS; do rm -f "$APP_DIR/db-dump.err" else echo "⚠️ FAILED (rc=$mysqldump_rc, see db-dump.err)" + ERRORS+=("$app:db_dump:rc=$mysqldump_rc $(tr '\n' ' ' < "$APP_DIR/db-dump.err" 2>/dev/null | cut -c1-160)") fi else echo "⚠️ could not determine DB name/credentials" + ERRORS+=("$app:db_dump:could not determine DB name/credentials") fi ;; esac @@ -284,6 +370,7 @@ for app in $ALL_APPS; do rm -f "$APP_DIR/pvc-data.err" else echo "⚠️ FAILED (see pvc-data.err)" + ERRORS+=("$app:pvc_data:$(tr '\n' ' ' < "$APP_DIR/pvc-data.err" 2>/dev/null | cut -c1-160)") fi APP_COUNT=$((APP_COUNT + 1)) @@ -354,7 +441,7 @@ if [ "$ARCHIVE_COUNT" -gt "$MAX_BACKUPS" ]; then TO_DELETE=$(echo "$ARCHIVE_LIST" | tail -n +$((MAX_BACKUPS + 1))) while IFS= read -r old_file; do [ -z "$old_file" ] && continue - rm -f "$old_file" "${old_file}.sha256" + rm -f "$old_file" "${old_file}.sha256" "${old_file}.meta.json" echo " 🗑️ Deleted: $(basename "$old_file")" done <<< "$TO_DELETE" else @@ -469,10 +556,11 @@ try: for obj in to_delete: old_key = obj['Key'] client.delete_object(Bucket=bucket, Key=old_key) - try: - client.delete_object(Bucket=bucket, Key=old_key + '.sha256') - except Exception: - pass + for suffix in ('.sha256', '.meta.json'): + try: + client.delete_object(Bucket=bucket, Key=old_key + suffix) + except Exception: + pass print(f' 🗑️ Deleted from R2: {old_key.replace("backups/", "")}') sys.exit(0) @@ -495,6 +583,13 @@ PYEOF fi fi +# -------------------------------------------------- +# Status metadata sidecar — after all 3 tiers are known +# -------------------------------------------------- +echo "" +echo "📝 Writing status metadata sidecar..." +write_meta_sidecar + # -------------------------------------------------- # Final summary + log # -------------------------------------------------- diff --git a/platform/app.py b/platform/app.py index efba6ec..6dd5964 100644 --- a/platform/app.py +++ b/platform/app.py @@ -17,6 +17,7 @@ from modules.auth import login_required from modules.backups import ( get_containers, get_all_root_containers, get_local_backups, get_vm_backups, + get_local_backups_with_status, get_vm_backups_with_status, get_all_stats, get_system_info, get_rootless_user_containers_remote, container_action, get_container_status, @@ -169,8 +170,8 @@ def cluster_page(): def backups_page(): return render_template( 'pages/backups.html', - backups=get_local_backups(), - vm_backups=get_vm_backups(), + backups=get_local_backups_with_status(), + vm_backups=get_vm_backups_with_status(), main_server=MAIN_SERVER_IP, active_page='backups', page_title='Backup Management', diff --git a/platform/modules/backups.py b/platform/modules/backups.py index 33c48d0..966ac0f 100644 --- a/platform/modules/backups.py +++ b/platform/modules/backups.py @@ -127,6 +127,127 @@ def get_vm_backups(): return vm_backups +# ──────────────────────────────────────────────────────────────── +# BACKUP STATUS DOTS (green/yellow/red/unknown) +# +# Reads the .meta.json sidecar backup-k8s-apps.sh writes alongside each +# archive (never decompresses the archive itself). Legacy myapps-backup-* +# archives and any k8s backup made before this feature shipped have no +# sidecar and show as 'unknown' — there's no reliable way to reconstruct +# per-step error info after the fact, and no backfill is attempted. +# ──────────────────────────────────────────────────────────────── + +ROLLING_WINDOW = 5 +SIZE_DEVIATION_THRESHOLD = 0.40 + + +def _load_local_sidecars(local_dir='/root/backups'): + """All .meta.json sidecars in a directory, newest first. Used both to + look up one backup's own metadata and as the history pool for its + rolling-average comparison.""" + sidecars = {} + for path in glob.glob(f'{local_dir}/*.tar.gz.meta.json'): + try: + with open(path) as f: + m = json.load(f) + except Exception: + continue + backup_file = os.path.basename(path)[:-len('.meta.json')] + m['_backup_file'] = backup_file + sidecars[backup_file] = m + return sidecars + + +def _parse_vm_sidecars(cat_output): + """Parse the batched `echo marker + cat` SSH output from _fetch_vm_sidecars + into the same {backup_file: meta} shape as _load_local_sidecars.""" + sidecars = {} + if not cat_output: + return sidecars + for block in cat_output.split('===META:')[1:]: + try: + fname_line, _, rest = block.partition('===\n') + backup_file = fname_line.strip()[:-len('.meta.json')] + m = json.loads(rest) + m['_backup_file'] = backup_file + sidecars[backup_file] = m + except Exception: + continue + return sidecars + + +def _fetch_vm_sidecars(): + local_dir = '/backups/cloudproject' + if os.path.isdir(local_dir): + return _load_local_sidecars(local_dir) + cmd = ( + f"ssh -i {VM_KEY} -p {VM_PORT} -o StrictHostKeyChecking=no " + f"-o ConnectTimeout=10 -o BatchMode=yes {VM_USER}@{VM_HOST} " + f"'for f in /backups/cloudproject/*.tar.gz.meta.json; do " + f"[ -f \"$f\" ] && echo \"===META:$(basename \"$f\")===\" && cat \"$f\"; " + f"done 2>/dev/null'" + ) + stdout, _ = _run(cmd, timeout=25) + return _parse_vm_sidecars(stdout) + + +def _compute_backup_status(meta, all_sidecars): + """all_sidecars: dict of {backup_file: meta} — the pool this entry's + rolling average is computed from (same directory/tier it came from).""" + tiers = meta.get('tiers', {}) + if meta.get('errors') or any(str(v).startswith('failed') for v in tiers.values()): + return 'red' + + apps_key = meta.get('apps_key', '') + size_bytes = meta.get('size_bytes', 0) + created_at = meta.get('created_at', '') + + # Same app-combination, strictly older than this backup, most recent + # first — a fresh, self-calibrating baseline per app-set rather than + # one hardcoded global threshold (a 1-app and a 5-app backup have very + # different normal sizes). + history = sorted( + (m for m in all_sidecars.values() + if m.get('apps_key') == apps_key and m.get('created_at', '') < created_at), + key=lambda m: m.get('created_at', ''), + reverse=True, + )[:ROLLING_WINDOW] + + if not history: + return 'green' # first backup of this app-combination — nothing to compare against + + avg = sum(m.get('size_bytes', 0) for m in history) / len(history) + if avg <= 0: + return 'green' + + deviation = abs(size_bytes - avg) / avg + return 'yellow' if deviation > SIZE_DEVIATION_THRESHOLD else 'green' + + +def _with_status(names, sidecars): + result = [] + for name in names: + meta = sidecars.get(name) + if meta is None: + result.append({'name': name, 'status': 'unknown', 'size_human': None, 'apps': None}) + else: + result.append({ + 'name': name, + 'status': _compute_backup_status(meta, sidecars), + 'size_human': _human_bytes(meta.get('size_bytes', 0)), + 'apps': meta.get('apps'), + }) + return result + + +def get_local_backups_with_status(): + return _with_status(get_local_backups(), _load_local_sidecars()) + + +def get_vm_backups_with_status(): + return _with_status(get_vm_backups(), _fetch_vm_sidecars()) + + # ──────────────────────────────────────────────────────────────── # BACKUP HEALTH AUDIT # ──────────────────────────────────────────────────────────────── @@ -524,11 +645,16 @@ def delete_backup(backup_file, source='local'): if not os.path.exists(archive_path): return False, f'File not found: {archive_path}' os.remove(archive_path) - sha = archive_path + '.sha256' - if os.path.exists(sha): os.remove(sha) + for suffix in ('.sha256', '.meta.json'): + sidecar = archive_path + suffix + if os.path.exists(sidecar): os.remove(sidecar) return True, f'Deleted {backup_file} from main server' else: - cmd = f"rm -f /root/backups/{backup_file} /root/backups/{backup_file}.sha256" + cmd = ( + f"rm -f /root/backups/{backup_file} " + f"/root/backups/{backup_file}.sha256 " + f"/root/backups/{backup_file}.meta.json" + ) out, err = _ssh_main(cmd) if err and 'No such file' not in err: return False, f'Remote delete error: {err}' @@ -540,8 +666,9 @@ def delete_backup(backup_file, source='local'): if not os.path.exists(archive_path): return False, f'File not found: {archive_path}' os.remove(archive_path) - sha = archive_path + '.sha256' - if os.path.exists(sha): os.remove(sha) + for suffix in ('.sha256', '.meta.json'): + sidecar = archive_path + suffix + if os.path.exists(sidecar): os.remove(sidecar) return True, f'Deleted {backup_file} from VM' else: cmd = ( @@ -550,7 +677,8 @@ def delete_backup(backup_file, source='local'): f"-o BatchMode=yes " f"{VM_USER}@{VM_HOST} " f"'rm -f /backups/cloudproject/{backup_file} " - f"/backups/cloudproject/{backup_file}.sha256'" + f"/backups/cloudproject/{backup_file}.sha256 " + f"/backups/cloudproject/{backup_file}.meta.json'" ) out, err = _run(cmd, timeout=30) if err and 'No such file' not in err: diff --git a/platform/static/css/style.css b/platform/static/css/style.css index 6c3581b..0f88dd8 100644 --- a/platform/static/css/style.css +++ b/platform/static/css/style.css @@ -881,7 +881,7 @@ a.sidebar-brand-link { box-shadow: var(--glow-accent); } -.backup-item-main { min-width: 0; padding-bottom: 12px; } +.backup-item-main { min-width: 0; padding-bottom: 12px; display: flex; align-items: center; gap: 8px; } .backup-item:not(:has(.backup-actions)) .backup-item-main { padding-bottom: 0; } .backup-name { @@ -894,6 +894,18 @@ a.sidebar-brand-link { } .backup-name i { margin-right: 6px; opacity: 0.85; } +/* Backup status dot — green/yellow/red from the .meta.json sidecar + backup-k8s-apps.sh writes; gray for backups with no sidecar (legacy + archives, or anything made before this feature shipped). */ +.status-dot { + width: 8px; height: 8px; border-radius: 50%; flex-shrink: 0; + border: 1px solid rgba(0,0,0,0.15); +} +.status-dot-green { background: var(--green); box-shadow: 0 0 6px rgba(45,212,122,0.5); } +.status-dot-yellow { background: var(--yellow); box-shadow: 0 0 6px rgba(245,185,66,0.5); } +.status-dot-red { background: var(--red); box-shadow: 0 0 6px rgba(240,82,82,0.5); } +.status-dot-unknown { background: var(--text3); box-shadow: none; } + .backup-meta { display: block; margin-top: 6px; diff --git a/platform/templates/pages/backups.html b/platform/templates/pages/backups.html index 2d0737f..7664284 100644 --- a/platform/templates/pages/backups.html +++ b/platform/templates/pages/backups.html @@ -286,14 +286,15 @@ {% for b in backups %}
- {{ b }} + + {{ b.name }}
- - - - - + + + + +
{% else %}
No backups
{% endfor %} @@ -310,13 +311,14 @@ {% for b in vm_backups %}
- {{ b }} + + {{ b.name }}
- - - - + + + +
{% else %}
No VM backups
{% endfor %}