ameniboukattaya/CloudOps
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add per-backup status dot (green/yellow/red/unknown) to the UI

Browse Source 
backup-k8s-apps.sh now tracks step-level failures (manifests, secret,
db_dump, pvc_data) into an ERRORS array during the per-app loop, and
writes a .meta.json sidecar alongside each archive once all 3 storage
tiers are known - apps included, final size, the error list, and
per-tier ok/failed/skipped status. Sidecar is mirrored to VM/R2 the
same best-effort way the .sha256 sidecar already is, and cleaned up
by both local and R2 retention pruning plus manual delete.

modules/backups.py reads these sidecars (never decompresses the
archive) to compute a status per backup at list-render time:
  - red: any recorded error, or any tier status starting with "failed"
  - yellow: no errors, but size deviates >40% from the rolling average
    of the last 5 backups sharing the same app-combination (apps_key)
  - green: no errors, size within range (or first backup of its
    app-combination - nothing to compare against yet)
  - unknown: no sidecar at all (legacy myapps-backup-* archives, or
    any k8s backup made before this shipped) - no backfill attempted,
    old runs never recorded step-level failures to reconstruct from

/backups route now passes get_local_backups_with_status()/
get_vm_backups_with_status() instead of the plain filename lists
(get_local_backups()/get_vm_backups() themselves are untouched -
/restore and /api/backups still use the plain versions, they don't
need the dot). Template renders a colored dot next to each entry with
a tooltip showing apps/size.

Verified: real n8n backup produces a correct sidecar synced to all 3
tiers; JSON-writer argv parsing and the red/yellow/green/unknown
decision logic each checked against synthetic cases; full Jinja render
checked against real local + VM data pulled from the live pod.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_017avLHFqkiti3g62Anq9sVA
This commit is contained in:
root
2026-08-21 02:46:50 +02:00
parent de0b1ee4db
commit d498c81d5b
5 changed files with 265 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

140
platform/modules/backups.py
View File

@@ -127,6 +127,127 @@ def get_vm_backups():
return vm_backups
# ────────────────────────────────────────────────────────────────
# BACKUP STATUS DOTS (green/yellow/red/unknown)
#
# Reads the .meta.json sidecar backup-k8s-apps.sh writes alongside each
# archive (never decompresses the archive itself). Legacy myapps-backup-*
# archives and any k8s backup made before this feature shipped have no
# sidecar and show as 'unknown' — there's no reliable way to reconstruct
# per-step error info after the fact, and no backfill is attempted.
# ────────────────────────────────────────────────────────────────
ROLLING_WINDOW = 5
SIZE_DEVIATION_THRESHOLD = 0.40
def _load_local_sidecars(local_dir='/root/backups'):
"""All .meta.json sidecars in a directory, newest first. Used both to
look up one backup's own metadata and as the history pool for its
rolling-average comparison."""
sidecars = {}
for path in glob.glob(f'{local_dir}/*.tar.gz.meta.json'):
try:
with open(path) as f:
m = json.load(f)
except Exception:
continue
backup_file = os.path.basename(path)[:-len('.meta.json')]
m['_backup_file'] = backup_file
sidecars[backup_file] = m
return sidecars
def _parse_vm_sidecars(cat_output):
"""Parse the batched `echo marker + cat` SSH output from _fetch_vm_sidecars
into the same {backup_file: meta} shape as _load_local_sidecars."""
sidecars = {}
if not cat_output:
return sidecars
for block in cat_output.split('===META:')[1:]:
try:
fname_line, _, rest = block.partition('===\n')
backup_file = fname_line.strip()[:-len('.meta.json')]
m = json.loads(rest)
m['_backup_file'] = backup_file
sidecars[backup_file] = m
except Exception:
continue
return sidecars
def _fetch_vm_sidecars():
local_dir = '/backups/cloudproject'
if os.path.isdir(local_dir):
return _load_local_sidecars(local_dir)
cmd = (
f"ssh -i {VM_KEY} -p {VM_PORT} -o StrictHostKeyChecking=no "
f"-o ConnectTimeout=10 -o BatchMode=yes {VM_USER}@{VM_HOST} "
f"'for f in /backups/cloudproject/*.tar.gz.meta.json; do "
f"[ -f \"$f\" ] && echo \"===META:$(basename \"$f\")===\" && cat \"$f\"; "
f"done 2>/dev/null'"
)
stdout, _ = _run(cmd, timeout=25)
return _parse_vm_sidecars(stdout)
def _compute_backup_status(meta, all_sidecars):
"""all_sidecars: dict of {backup_file: meta} — the pool this entry's
rolling average is computed from (same directory/tier it came from)."""
tiers = meta.get('tiers', {})
if meta.get('errors') or any(str(v).startswith('failed') for v in tiers.values()):
return 'red'
apps_key = meta.get('apps_key', '')
size_bytes = meta.get('size_bytes', 0)
created_at = meta.get('created_at', '')
# Same app-combination, strictly older than this backup, most recent
# first — a fresh, self-calibrating baseline per app-set rather than
# one hardcoded global threshold (a 1-app and a 5-app backup have very
# different normal sizes).
history = sorted(
(m for m in all_sidecars.values()
if m.get('apps_key') == apps_key and m.get('created_at', '') < created_at),
key=lambda m: m.get('created_at', ''),
reverse=True,
)[:ROLLING_WINDOW]
if not history:
return 'green' # first backup of this app-combination — nothing to compare against
avg = sum(m.get('size_bytes', 0) for m in history) / len(history)
if avg <= 0:
return 'green'
deviation = abs(size_bytes - avg) / avg
return 'yellow' if deviation > SIZE_DEVIATION_THRESHOLD else 'green'
def _with_status(names, sidecars):
result = []
for name in names:
meta = sidecars.get(name)
if meta is None:
result.append({'name': name, 'status': 'unknown', 'size_human': None, 'apps': None})
else:
result.append({
'name': name,
'status': _compute_backup_status(meta, sidecars),
'size_human': _human_bytes(meta.get('size_bytes', 0)),
'apps': meta.get('apps'),
})
return result
def get_local_backups_with_status():
return _with_status(get_local_backups(), _load_local_sidecars())
def get_vm_backups_with_status():
return _with_status(get_vm_backups(), _fetch_vm_sidecars())
# ────────────────────────────────────────────────────────────────
# BACKUP HEALTH AUDIT
# ────────────────────────────────────────────────────────────────
@@ -524,11 +645,16 @@ def delete_backup(backup_file, source='local'):
if not os.path.exists(archive_path):
return False, f'File not found: {archive_path}'
os.remove(archive_path)
sha = archive_path + '.sha256'
if os.path.exists(sha): os.remove(sha)
for suffix in ('.sha256', '.meta.json'):
sidecar = archive_path + suffix
if os.path.exists(sidecar): os.remove(sidecar)
return True, f'Deleted {backup_file} from main server'
else:
cmd = f"rm -f /root/backups/{backup_file} /root/backups/{backup_file}.sha256"
cmd = (
f"rm -f /root/backups/{backup_file} "
f"/root/backups/{backup_file}.sha256 "
f"/root/backups/{backup_file}.meta.json"
)
out, err = _ssh_main(cmd)
if err and 'No such file' not in err:
return False, f'Remote delete error: {err}'
@@ -540,8 +666,9 @@ def delete_backup(backup_file, source='local'):
if not os.path.exists(archive_path):
return False, f'File not found: {archive_path}'
os.remove(archive_path)
sha = archive_path + '.sha256'
if os.path.exists(sha): os.remove(sha)
for suffix in ('.sha256', '.meta.json'):
sidecar = archive_path + suffix
if os.path.exists(sidecar): os.remove(sidecar)
return True, f'Deleted {backup_file} from VM'
else:
cmd = (
@@ -550,7 +677,8 @@ def delete_backup(backup_file, source='local'):
f"-o BatchMode=yes "
f"{VM_USER}@{VM_HOST} "
f"'rm -f /backups/cloudproject/{backup_file} "
f"/backups/cloudproject/{backup_file}.sha256'"
f"/backups/cloudproject/{backup_file}.sha256 "
f"/backups/cloudproject/{backup_file}.meta.json'"
)
out, err = _run(cmd, timeout=30)
if err and 'No such file' not in err: