11b9f6f0c7
Two real gaps found after testing the live standby platform: 1. modules/kubernetes.py's in-cluster client (load_incluster_config()) can only ever work inside the real k8s pod. On the standby it silently returned K8S_AVAILABLE=False with zero fallback — Cluster, Containers- via-sites, and any cluster/app health data was just empty on that page, not erroring, so it was easy to miss. This holds even if the standby later runs as a pod on a *local* k3s on that server (per plan for the monitoring migration) — in-cluster config there would only ever see that local, unrelated cluster, never the main one. Fixed by adding an SSH+kubectl remote path to list_pods(), list_ deployments(), get_ingress_info(), list_deployments_for_namespace(), and get_pod_metrics(): when not RUNNING_ON_MAIN_SERVER, each runs `kubectl <verb> -o json` on the main server over the tunnel (main-to-vm-tunnel.service) instead of using the python client, parses the raw (camelCase) JSON directly rather than trying to force it through the client-library's snake_case object model, and feeds the same _pod_health/_deploy_health/_age_str helpers either way. The in-cluster path for the real main-server pod is untouched. 2. get_system_info() called psutil.* unconditionally with no RUNNING_ON_MAIN_SERVER awareness at all — on the standby it was silently reporting the VM's OWN cpu/mem/disk/hostname mislabeled as "system info" (hostname field literally showed the VM's hostname). Added _get_main_server_system_info_remote(): SSHes to the main server and gathers the same stats via vmstat/free/df/procfs (the main server's bare host has no psutil — that's only in this app's own container image — so this avoids depending on it being present remotely). Verified end-to-end on the live standby: /api/system now reports the real main server's hostname/cpu/mem/disk/uptime; /api/cluster reports real counts (19 pods, 16/16 deployments healthy, 3 degraded, plus live per-pod metrics via the metrics-server raw API); /api/sites shows real per-app container/role status. All previously silently empty.
1110 lines
42 KiB
Python
1110 lines
42 KiB
Python
# modules/backups.py
|
|
import os
|
|
import glob
|
|
import subprocess
|
|
import json
|
|
import hashlib
|
|
import tarfile
|
|
import re
|
|
import time
|
|
import socket
|
|
import psutil
|
|
from config import (
|
|
RUNNING_ON_MAIN_SERVER,
|
|
MAIN_SERVER_USER, MAIN_SERVER_KEY,
|
|
MAIN_SERVER_SSH_HOST, MAIN_SERVER_SSH_PORT,
|
|
VM_HOST, VM_PORT, VM_KEY, VM_USER,
|
|
)
|
|
|
|
|
|
def _run(cmd, timeout=30):
|
|
try:
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
|
return r.stdout.strip(), r.stderr.strip()
|
|
except Exception as e:
|
|
return '', str(e)
|
|
|
|
|
|
def _human_bytes(n):
|
|
n = int(n)
|
|
if n < 1024:
|
|
return f'{n} B'
|
|
if n < 1024 ** 2:
|
|
return f'{n / 1024:.1f} KB'
|
|
if n < 1024 ** 3:
|
|
return f'{n / (1024 ** 2):.1f} MB'
|
|
if n < 1024 ** 4:
|
|
return f'{n / (1024 ** 3):.2f} GB'
|
|
return f'{n / (1024 ** 4):.2f} TB'
|
|
|
|
|
|
def _ssh_main(remote_cmd, timeout=30):
|
|
if RUNNING_ON_MAIN_SERVER:
|
|
return _run(remote_cmd, timeout=timeout)
|
|
else:
|
|
escaped = remote_cmd.replace("'", "'\\''")
|
|
ssh = (
|
|
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"-o BatchMode=yes "
|
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST}"
|
|
)
|
|
return _run(f"{ssh} '{escaped}'", timeout=timeout)
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUPS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def _list_local_dir(local_dir):
|
|
"""Direct filesystem listing, newest first, both lineages."""
|
|
files = (
|
|
glob.glob(f'{local_dir}/myapps-backup-*.tar.gz')
|
|
+ glob.glob(f'{local_dir}/myapps-k8s-backup-*.tar.gz')
|
|
)
|
|
files.sort(key=os.path.getmtime, reverse=True)
|
|
return [os.path.basename(f) for f in files[:20]]
|
|
|
|
|
|
def get_local_backups():
|
|
# myapps-backup-* is the legacy Docker-era lineage; myapps-k8s-backup-*
|
|
# is the k8s-native one (backup-k8s-apps.sh). Listed together, sorted
|
|
# by mtime, so history from both stays visible in the UI.
|
|
#
|
|
# Try direct filesystem access first: the management-platform pod has
|
|
# /root hostPath-mounted from the main server (see k8s/management-
|
|
# platform-deployment.yaml), so /root/backups is already right here.
|
|
# RUNNING_ON_MAIN_SERVER's hostname check never matches inside a pod
|
|
# (its hostname is the pod name, not the host's), so this used to
|
|
# always fall through to the SSH-to-self branch below — which also
|
|
# doesn't actually work in this topology (wrong key for this deploy
|
|
# model) — silently returning an empty list. Falling back to SSH only
|
|
# when there's genuinely no local mount keeps this working for any
|
|
# future non-mounted deployment too.
|
|
if os.path.isdir('/root/backups'):
|
|
return _list_local_dir('/root/backups')
|
|
|
|
stdout, _ = _ssh_main(
|
|
"ls -t /root/backups/myapps-backup-*.tar.gz "
|
|
"/root/backups/myapps-k8s-backup-*.tar.gz 2>/dev/null | head -20"
|
|
)
|
|
files = []
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
line = line.strip()
|
|
if line:
|
|
files.append(os.path.basename(line))
|
|
return files
|
|
|
|
|
|
def get_vm_backups():
|
|
# Same fix as get_local_backups(): try direct access first (covers the
|
|
# case where this process's filesystem actually is the VM's), fall
|
|
# back to SSH to the VM host otherwise — which is the real case for
|
|
# the management-platform pod, since only /root is mounted in, not
|
|
# anything from the separate VM backup host.
|
|
local_dir = '/backups/cloudproject'
|
|
if os.path.isdir(local_dir):
|
|
return _list_local_dir(local_dir)
|
|
|
|
vm_backups = []
|
|
try:
|
|
cmd = (
|
|
f"ssh -i {VM_KEY} -p {VM_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"-o BatchMode=yes "
|
|
f"{VM_USER}@{VM_HOST} "
|
|
f"'ls -t /backups/cloudproject/myapps-backup-*.tar.gz "
|
|
f"/backups/cloudproject/myapps-k8s-backup-*.tar.gz 2>/dev/null | head -20'"
|
|
)
|
|
stdout, _ = _run(cmd, timeout=25)
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
line = line.strip()
|
|
if line and '.tar.gz' in line:
|
|
vm_backups.append(os.path.basename(line))
|
|
except Exception as e:
|
|
print(f"[backups] VM backup fetch error: {e}")
|
|
return vm_backups
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUP STATUS DOTS (green/yellow/red/unknown)
|
|
#
|
|
# Reads the .meta.json sidecar backup-k8s-apps.sh writes alongside each
|
|
# archive (never decompresses the archive itself). Legacy myapps-backup-*
|
|
# archives and any k8s backup made before this feature shipped have no
|
|
# sidecar and show as 'unknown' — there's no reliable way to reconstruct
|
|
# per-step error info after the fact, and no backfill is attempted.
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
ROLLING_WINDOW = 5
|
|
SIZE_DEVIATION_THRESHOLD = 0.40
|
|
|
|
|
|
def _load_local_sidecars(local_dir='/root/backups'):
|
|
"""All .meta.json sidecars in a directory, newest first. Used both to
|
|
look up one backup's own metadata and as the history pool for its
|
|
rolling-average comparison."""
|
|
sidecars = {}
|
|
for path in glob.glob(f'{local_dir}/*.tar.gz.meta.json'):
|
|
try:
|
|
with open(path) as f:
|
|
m = json.load(f)
|
|
except Exception:
|
|
continue
|
|
backup_file = os.path.basename(path)[:-len('.meta.json')]
|
|
m['_backup_file'] = backup_file
|
|
sidecars[backup_file] = m
|
|
return sidecars
|
|
|
|
|
|
def _parse_vm_sidecars(cat_output):
|
|
"""Parse the batched `echo marker + cat` SSH output from _fetch_vm_sidecars
|
|
into the same {backup_file: meta} shape as _load_local_sidecars."""
|
|
sidecars = {}
|
|
if not cat_output:
|
|
return sidecars
|
|
for block in cat_output.split('===META:')[1:]:
|
|
try:
|
|
fname_line, _, rest = block.partition('===\n')
|
|
backup_file = fname_line.strip()[:-len('.meta.json')]
|
|
m = json.loads(rest)
|
|
m['_backup_file'] = backup_file
|
|
sidecars[backup_file] = m
|
|
except Exception:
|
|
continue
|
|
return sidecars
|
|
|
|
|
|
def _fetch_vm_sidecars():
|
|
local_dir = '/backups/cloudproject'
|
|
if os.path.isdir(local_dir):
|
|
return _load_local_sidecars(local_dir)
|
|
cmd = (
|
|
f"ssh -i {VM_KEY} -p {VM_PORT} -o StrictHostKeyChecking=no "
|
|
f"-o ConnectTimeout=10 -o BatchMode=yes {VM_USER}@{VM_HOST} "
|
|
f"'for f in /backups/cloudproject/*.tar.gz.meta.json; do "
|
|
f"[ -f \"$f\" ] && echo \"===META:$(basename \"$f\")===\" && cat \"$f\"; "
|
|
f"done 2>/dev/null'"
|
|
)
|
|
stdout, _ = _run(cmd, timeout=25)
|
|
return _parse_vm_sidecars(stdout)
|
|
|
|
|
|
def _compute_backup_status(meta, all_sidecars):
|
|
"""all_sidecars: dict of {backup_file: meta} — the pool this entry's
|
|
rolling average is computed from (same directory/tier it came from)."""
|
|
tiers = meta.get('tiers', {})
|
|
if meta.get('errors') or any(str(v).startswith('failed') for v in tiers.values()):
|
|
return 'red'
|
|
|
|
apps_key = meta.get('apps_key', '')
|
|
size_bytes = meta.get('size_bytes', 0)
|
|
created_at = meta.get('created_at', '')
|
|
|
|
# Same app-combination, strictly older than this backup, most recent
|
|
# first — a fresh, self-calibrating baseline per app-set rather than
|
|
# one hardcoded global threshold (a 1-app and a 5-app backup have very
|
|
# different normal sizes).
|
|
history = sorted(
|
|
(m for m in all_sidecars.values()
|
|
if m.get('apps_key') == apps_key and m.get('created_at', '') < created_at),
|
|
key=lambda m: m.get('created_at', ''),
|
|
reverse=True,
|
|
)[:ROLLING_WINDOW]
|
|
|
|
if not history:
|
|
return 'green' # first backup of this app-combination — nothing to compare against
|
|
|
|
avg = sum(m.get('size_bytes', 0) for m in history) / len(history)
|
|
if avg <= 0:
|
|
return 'green'
|
|
|
|
deviation = abs(size_bytes - avg) / avg
|
|
return 'yellow' if deviation > SIZE_DEVIATION_THRESHOLD else 'green'
|
|
|
|
|
|
def _with_status(names, sidecars):
|
|
result = []
|
|
for name in names:
|
|
meta = sidecars.get(name)
|
|
if meta is None:
|
|
result.append({'name': name, 'status': 'unknown', 'size_human': None, 'apps': None})
|
|
else:
|
|
result.append({
|
|
'name': name,
|
|
'status': _compute_backup_status(meta, sidecars),
|
|
'size_human': _human_bytes(meta.get('size_bytes', 0)),
|
|
'apps': meta.get('apps'),
|
|
})
|
|
return result
|
|
|
|
|
|
def get_local_backups_with_status():
|
|
return _with_status(get_local_backups(), _load_local_sidecars())
|
|
|
|
|
|
def get_vm_backups_with_status():
|
|
return _with_status(get_vm_backups(), _fetch_vm_sidecars())
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUP HEALTH AUDIT
|
|
# ────────────────────────────────────────────────────────────────
|
|
#
|
|
# audit_backup() used to be one large function (cognitive complexity 18).
|
|
# It's now an orchestrator that calls one small, single-purpose helper per
|
|
# check. Each helper takes the shared `add()` callback and any state it
|
|
# needs, and is responsible for exactly one check — easy to read, easy to
|
|
# test, easy to extend without pushing complexity back up.
|
|
|
|
def _resolve_archive_path(backup_file, source, add):
|
|
"""Figure out the local path to the archive, pulling it over SSH only
|
|
if it's genuinely not reachable on this filesystem. Returns the path,
|
|
or None if it could not be resolved (in which case an error result
|
|
dict has already been added via `add` and the caller should bail out).
|
|
|
|
Tries direct access first regardless of RUNNING_ON_MAIN_SERVER: the
|
|
management-platform pod has /root hostPath-mounted from the main
|
|
server, so /root/backups is already right here even though the
|
|
hostname check that flag relies on never matches inside a pod."""
|
|
if source == 'local':
|
|
archive_path = f"/root/backups/{backup_file}"
|
|
if os.path.exists(archive_path):
|
|
return archive_path
|
|
tmp_path = f"/tmp/audit_{backup_file}"
|
|
if not os.path.exists(tmp_path):
|
|
pull_cmd = (
|
|
f"scp -i {MAIN_SERVER_KEY} -P {MAIN_SERVER_SSH_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=15 "
|
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST}:/root/backups/{backup_file} "
|
|
f"{tmp_path}"
|
|
)
|
|
_out, err = _run(pull_cmd, timeout=120)
|
|
if not os.path.exists(tmp_path):
|
|
add('File Access', 'fail', f'Could not pull from main server: {err}')
|
|
return None
|
|
return tmp_path
|
|
else:
|
|
archive_path = f"/backups/cloudproject/{backup_file}"
|
|
if os.path.exists(archive_path):
|
|
return archive_path
|
|
tmp_path = f"/tmp/audit_vm_{backup_file}"
|
|
if not os.path.exists(tmp_path):
|
|
pull_cmd = (
|
|
f"scp -i {VM_KEY} -P {VM_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=15 "
|
|
f"{VM_USER}@{VM_HOST}:/backups/cloudproject/{backup_file} "
|
|
f"{tmp_path}"
|
|
)
|
|
_out, err = _run(pull_cmd, timeout=120)
|
|
if not os.path.exists(tmp_path):
|
|
add('File Access', 'fail', f'Could not pull from VM: {err}')
|
|
return None
|
|
return tmp_path
|
|
|
|
|
|
def _check_file_size(archive_path, add):
|
|
size_bytes = os.path.getsize(archive_path)
|
|
size_mb = size_bytes / (1024 * 1024)
|
|
size_human = _human_bytes(size_bytes)
|
|
size_more = [
|
|
f'Exact size: {size_bytes:,} bytes ({size_human})',
|
|
'We flag archives under 1 MB as corrupt and under ~50 MB as unusually small for a full stack backup.',
|
|
]
|
|
if size_bytes < 1024 * 1024:
|
|
add('File Size', 'fail', f'{size_human} — suspiciously tiny, likely corrupt', more=size_more)
|
|
elif size_mb < 50:
|
|
add('File Size', 'warn', f'{size_human} — smaller than expected (typical full backup > 50 MB)', more=size_more)
|
|
else:
|
|
add('File Size', 'pass', f'{size_human} — within expected range', more=size_more)
|
|
return size_bytes, size_human
|
|
|
|
|
|
def _check_checksum(archive_path, add):
|
|
sha_file = archive_path + '.sha256'
|
|
if not os.path.exists(sha_file):
|
|
add('Checksum (SHA256)', 'warn', 'No .sha256 sidecar found — run a new backup to get checksums')
|
|
return
|
|
try:
|
|
with open(sha_file, 'r') as f:
|
|
expected_hash = f.read().split()[0].strip()
|
|
actual_hash = _sha256_file(archive_path)
|
|
if actual_hash == expected_hash:
|
|
add('Checksum (SHA256)', 'pass', f'Hash verified — {actual_hash[:20]}…')
|
|
else:
|
|
add('Checksum (SHA256)', 'fail',
|
|
f'MISMATCH — expected {expected_hash[:20]}… got {actual_hash[:20]}…')
|
|
except Exception as e:
|
|
add('Checksum (SHA256)', 'warn', f'Could not verify: {e}')
|
|
|
|
|
|
def _check_archive_integrity(archive_path, add):
|
|
try:
|
|
result = subprocess.run(
|
|
['gzip', '--test', archive_path],
|
|
capture_output=True, text=True, timeout=120
|
|
)
|
|
if result.returncode == 0:
|
|
add('Archive Integrity', 'pass', 'gzip test passed — archive is not corrupted', more=[
|
|
'Runs gzip --test on the .tar.gz so the compressed stream is readable end-to-end.',
|
|
])
|
|
else:
|
|
add('Archive Integrity', 'fail',
|
|
f'gzip test failed: {(result.stderr or result.stdout)[:200]}')
|
|
except FileNotFoundError:
|
|
_check_archive_integrity_fallback(archive_path, add)
|
|
except subprocess.TimeoutExpired:
|
|
add('Archive Integrity', 'warn', 'Integrity check timed out — file is large, probably OK')
|
|
except Exception as e:
|
|
add('Archive Integrity', 'warn', f'Could not test: {e}')
|
|
|
|
|
|
def _check_archive_integrity_fallback(archive_path, add):
|
|
try:
|
|
import gzip
|
|
with gzip.open(archive_path, 'rb') as f:
|
|
f.read(1024 * 1024)
|
|
add('Archive Integrity', 'pass', 'gzip header valid')
|
|
except Exception as e:
|
|
add('Archive Integrity', 'fail', f'Archive appears corrupt: {e}')
|
|
|
|
|
|
def _list_archive_members(archive_path):
|
|
try:
|
|
with tarfile.open(archive_path, 'r:gz') as tf:
|
|
return tf.getnames()
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
# The two backup lineages have completely different internal layouts:
|
|
# - legacy (myapps-backup-*, from backup-myapps.sh): volumes/*.tar.gz,
|
|
# compose-files/, backup-info.txt at the archive root.
|
|
# - k8s-native (myapps-k8s-backup-*, from backup-k8s-apps.sh): per-app
|
|
# subdirs (n8n/, odoo/, mautic/, nextcloud/, frappe/) each with
|
|
# manifests.yaml, secret.yaml, db-dump.sql.gz, pvc-data.tar.gz, plus
|
|
# backup-info.txt at the archive root.
|
|
# Auditing a k8s-format archive against the legacy layout would always
|
|
# fail it ("volumes/ missing") even when it's a perfectly good backup —
|
|
# same root cause as the myapps-backup-*/myapps-k8s-backup-* prefix bug
|
|
# already fixed elsewhere in this file, just in the audit checks instead
|
|
# of the listing/delete glob.
|
|
_K8S_APP_NAMES = ('n8n', 'odoo', 'mautic', 'nextcloud', 'frappe')
|
|
|
|
|
|
def _is_k8s_format(backup_file):
|
|
return os.path.basename(backup_file).startswith('myapps-k8s-backup-')
|
|
|
|
|
|
def _check_internal_structure(members, backup_file, add):
|
|
if not members:
|
|
add('Internal Structure', 'warn', 'Could not inspect archive members')
|
|
return
|
|
|
|
has_info = any('backup-info.txt' in m for m in members)
|
|
|
|
if _is_k8s_format(backup_file):
|
|
app_dirs = sorted({
|
|
parts[1] for m in members
|
|
if len(parts := m.split('/')) > 1 and parts[1] in _K8S_APP_NAMES
|
|
})
|
|
has_app_data = any(
|
|
m.endswith(('manifests.yaml', 'db-dump.sql.gz', 'pvc-data.tar.gz'))
|
|
for m in members
|
|
)
|
|
issues = []
|
|
if not has_app_data:
|
|
issues.append('no per-app manifests/db-dump/pvc-data found')
|
|
if not has_info:
|
|
issues.append('backup-info.txt missing')
|
|
if issues:
|
|
add('Internal Structure', 'fail', ' · '.join(issues))
|
|
return
|
|
add('Internal Structure', 'pass',
|
|
f'backup-info.txt ✓ apps: {", ".join(app_dirs) or "?"}')
|
|
return
|
|
|
|
has_volumes = any('volumes/' in m for m in members)
|
|
has_compose = any('compose-files/' in m for m in members)
|
|
vol_count = len([m for m in members if '/volumes/' in m and m.endswith('.tar.gz')])
|
|
|
|
issues = []
|
|
if not has_volumes:
|
|
issues.append('volumes/ missing')
|
|
if not has_info:
|
|
issues.append('backup-info.txt missing')
|
|
|
|
if issues:
|
|
add('Internal Structure', 'fail', ' · '.join(issues))
|
|
return
|
|
|
|
detail = 'volumes/ ✓ backup-info.txt ✓'
|
|
if has_compose:
|
|
detail += ' compose-files/ ✓'
|
|
detail += f' ({vol_count} volume archives)'
|
|
add('Internal Structure', 'pass', detail)
|
|
|
|
|
|
_SUSPICIOUS_PATTERNS = [
|
|
(r'\.\./', 'path traversal (..)'),
|
|
(r'^/', 'absolute path in archive'),
|
|
(r'/etc/passwd', '/etc/passwd reference'),
|
|
(r'/etc/shadow', '/etc/shadow reference'),
|
|
(r'\.ssh/', '.ssh directory reference'),
|
|
(r'id_rsa(?!\.pub)', 'private SSH key reference'),
|
|
(r'authorized_keys', 'authorized_keys reference'),
|
|
]
|
|
|
|
|
|
def _check_security_scan(members, add):
|
|
found_suspicious = []
|
|
for m in members:
|
|
for pat, label in _SUSPICIOUS_PATTERNS:
|
|
if re.search(pat, m):
|
|
found_suspicious.append(f'{m} ({label})')
|
|
break
|
|
|
|
if found_suspicious:
|
|
add('Security Scan', 'fail', f'Suspicious entries found: {found_suspicious[:3]}')
|
|
else:
|
|
add('Security Scan', 'pass', 'No path traversal or dangerous entries detected', more=[
|
|
'Member paths are checked for .. segments, absolute roots, and sensitive paths '
|
|
'(e.g. .ssh, /etc/shadow).',
|
|
])
|
|
|
|
|
|
_SCRIPT_EXTENSIONS = ('.sh', '.py', '.pl', '.rb', '.bash', '.zsh')
|
|
_SAFE_PREFIXES = ('compose-files/', 'volumes/', 'container-configs/', 'configs/')
|
|
|
|
|
|
def _is_unexpected_executable_script(member):
|
|
name = member.name
|
|
if any(name.startswith(p) or f'/{p}' in name for p in _SAFE_PREFIXES):
|
|
return False
|
|
name_lower = name.lower()
|
|
has_script_ext = any(name_lower.endswith(ext) for ext in _SCRIPT_EXTENSIONS)
|
|
has_exec_bit = bool(member.mode & 0o111)
|
|
return has_script_ext and has_exec_bit
|
|
|
|
|
|
def _check_executable_scripts(archive_path, add):
|
|
suspicious_scripts = []
|
|
try:
|
|
with tarfile.open(archive_path, 'r:gz') as tf:
|
|
for member in tf.getmembers():
|
|
if member.isfile() and _is_unexpected_executable_script(member):
|
|
suspicious_scripts.append(os.path.basename(member.name))
|
|
except Exception:
|
|
pass
|
|
|
|
if suspicious_scripts:
|
|
add('Executable Scripts', 'warn',
|
|
f'Scripts with execute bit outside expected dirs: {suspicious_scripts[:3]}')
|
|
else:
|
|
add('Executable Scripts', 'pass', 'No unexpected executable scripts found')
|
|
|
|
|
|
def _check_volume_count(members, backup_file, add):
|
|
if _is_k8s_format(backup_file):
|
|
pvc_archives = [m for m in members if m.endswith('/pvc-data.tar.gz')]
|
|
v = len(pvc_archives)
|
|
if v == 0:
|
|
add('App Data Archives', 'fail', 'No pvc-data.tar.gz found in backup')
|
|
elif v < 5:
|
|
add('App Data Archives', 'warn',
|
|
f'{v} app(s) backed up — fine if this was a --apps filtered run, '
|
|
f'otherwise expected 5 for a full backup')
|
|
else:
|
|
add('App Data Archives', 'pass', f'{v} app data archives present')
|
|
return
|
|
|
|
vol_archives = [m for m in members if 'volumes/' in m and m.endswith('.tar.gz')]
|
|
v = len(vol_archives)
|
|
if v == 0:
|
|
add('Volume Count', 'fail', 'No volume archives found in backup')
|
|
elif v < 5:
|
|
add('Volume Count', 'warn', f'Only {v} volumes (expected ≥5 for a full backup)')
|
|
else:
|
|
add('Volume Count', 'pass', f'{v} volume archives present')
|
|
|
|
|
|
def _score_checks(checks):
|
|
weights = {'pass': 10, 'warn': 5, 'fail': 0}
|
|
total = len(checks) * 10
|
|
earned = sum(weights.get(c['status'], 0) for c in checks)
|
|
return int((earned / total) * 100) if total > 0 else 0
|
|
|
|
|
|
def _summary_for_score(score):
|
|
if score >= 90:
|
|
return 'Backup looks healthy and is safe to restore.'
|
|
if score >= 70:
|
|
return 'Minor warnings — likely safe, but review before restoring.'
|
|
if score >= 40:
|
|
return 'Significant issues detected — restore with caution.'
|
|
return 'Multiple checks failed — do NOT restore without manual inspection.'
|
|
|
|
|
|
def _health_tier_for(score, has_fails, has_warns):
|
|
if has_fails:
|
|
return 'critical', 'Unhealthy'
|
|
if score == 100:
|
|
return 'excellent', '100% healthy'
|
|
if score >= 90:
|
|
return 'good', ('Healthy' if not has_warns else 'Healthy (with notes)')
|
|
if score >= 70:
|
|
return 'fair', 'Mostly healthy'
|
|
if score >= 40:
|
|
return 'poor', 'At risk'
|
|
return 'critical', 'Unhealthy'
|
|
|
|
|
|
def _missing_result(backup_file, checks, detail=None):
|
|
return {
|
|
'ok': False, 'score': 0, 'checks': checks,
|
|
'backup_file': backup_file,
|
|
'file_size_bytes': None,
|
|
'file_size_display': None,
|
|
'health_tier': 'critical',
|
|
'health_label': 'Unhealthy',
|
|
'summary': detail or 'Cannot access backup file from this host.',
|
|
}
|
|
|
|
|
|
def audit_backup(backup_file, source='local'):
|
|
checks = []
|
|
|
|
def add(name, status, detail='', more=None):
|
|
entry = {'name': name, 'status': status, 'detail': detail}
|
|
if more:
|
|
entry['more'] = more
|
|
checks.append(entry)
|
|
|
|
archive_path = _resolve_archive_path(backup_file, source, add)
|
|
if archive_path is None:
|
|
return _missing_result(backup_file, checks, 'Cannot access backup file from this host.')
|
|
|
|
if not os.path.exists(archive_path):
|
|
add('File Exists', 'fail', f'Not found: {archive_path}')
|
|
return _missing_result(backup_file, checks, 'Backup file does not exist on disk.')
|
|
add('File Exists', 'pass', archive_path)
|
|
|
|
size_bytes, size_human = _check_file_size(archive_path, add)
|
|
_check_checksum(archive_path, add)
|
|
_check_archive_integrity(archive_path, add)
|
|
|
|
members = _list_archive_members(archive_path)
|
|
_check_internal_structure(members, backup_file, add)
|
|
_check_security_scan(members, add)
|
|
_check_executable_scripts(archive_path, add)
|
|
_check_volume_count(members, backup_file, add)
|
|
|
|
score = _score_checks(checks)
|
|
has_fails = any(c['status'] == 'fail' for c in checks)
|
|
has_warns = any(c['status'] == 'warn' for c in checks)
|
|
ok = not has_fails and score >= 60
|
|
health_tier, health_label = _health_tier_for(score, has_fails, has_warns)
|
|
|
|
return {
|
|
'ok': ok,
|
|
'score': score,
|
|
'checks': checks,
|
|
'summary': _summary_for_score(score),
|
|
'backup_file': backup_file,
|
|
'file_size_bytes': size_bytes,
|
|
'file_size_display': size_human,
|
|
'health_tier': health_tier,
|
|
'health_label': health_label,
|
|
}
|
|
|
|
|
|
def _sha256_file(path):
|
|
h = hashlib.sha256()
|
|
with open(path, 'rb') as f:
|
|
for chunk in iter(lambda: f.read(65536), b''):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# DELETE BACKUP
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def delete_backup(backup_file, source='local'):
|
|
# Accepts both the legacy Docker-era lineage (myapps-backup-*) and the
|
|
# k8s-native one (myapps-k8s-backup-*, from backup-k8s-apps.sh). Still a
|
|
# tight anchor match — this guards against path traversal in a filename
|
|
# that ends up interpolated into a shell/filesystem path below.
|
|
if not re.match(r'^myapps-(k8s-)?backup-\d{8}_\d{6}\.tar\.gz$', backup_file):
|
|
return False, f'Invalid backup filename: {backup_file}'
|
|
|
|
if source == 'local':
|
|
if RUNNING_ON_MAIN_SERVER:
|
|
archive_path = f"/root/backups/{backup_file}"
|
|
if not os.path.exists(archive_path):
|
|
return False, f'File not found: {archive_path}'
|
|
os.remove(archive_path)
|
|
for suffix in ('.sha256', '.meta.json'):
|
|
sidecar = archive_path + suffix
|
|
if os.path.exists(sidecar): os.remove(sidecar)
|
|
return True, f'Deleted {backup_file} from main server'
|
|
else:
|
|
cmd = (
|
|
f"rm -f /root/backups/{backup_file} "
|
|
f"/root/backups/{backup_file}.sha256 "
|
|
f"/root/backups/{backup_file}.meta.json"
|
|
)
|
|
out, err = _ssh_main(cmd)
|
|
if err and 'No such file' not in err:
|
|
return False, f'Remote delete error: {err}'
|
|
return True, f'Deleted {backup_file} from main server'
|
|
|
|
elif source == 'vm':
|
|
archive_path = f"/backups/cloudproject/{backup_file}"
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
if not os.path.exists(archive_path):
|
|
return False, f'File not found: {archive_path}'
|
|
os.remove(archive_path)
|
|
for suffix in ('.sha256', '.meta.json'):
|
|
sidecar = archive_path + suffix
|
|
if os.path.exists(sidecar): os.remove(sidecar)
|
|
return True, f'Deleted {backup_file} from VM'
|
|
else:
|
|
cmd = (
|
|
f"ssh -i {VM_KEY} -p {VM_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"-o BatchMode=yes "
|
|
f"{VM_USER}@{VM_HOST} "
|
|
f"'rm -f /backups/cloudproject/{backup_file} "
|
|
f"/backups/cloudproject/{backup_file}.sha256 "
|
|
f"/backups/cloudproject/{backup_file}.meta.json'"
|
|
)
|
|
out, err = _run(cmd, timeout=30)
|
|
if err and 'No such file' not in err:
|
|
return False, f'VM delete error: {err}'
|
|
return True, f'Deleted {backup_file} from VM'
|
|
|
|
return False, 'Unknown source'
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUP STATUS LOG
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_backup_log_entries(limit=20):
|
|
stdout, _ = _ssh_main(
|
|
f"tail -n {limit} /root/backups/backup-status.log 2>/dev/null || echo ''"
|
|
)
|
|
entries = []
|
|
if not stdout:
|
|
return entries
|
|
for line in stdout.strip().split('\n'):
|
|
if not line.strip():
|
|
continue
|
|
parts = line.split('|')
|
|
entries.append({
|
|
'timestamp': parts[0].strip() if len(parts) > 0 else '',
|
|
'status': parts[1].strip() if len(parts) > 1 else '',
|
|
'name': parts[2].strip() if len(parts) > 2 else '',
|
|
'message': parts[3].strip() if len(parts) > 3 else '',
|
|
})
|
|
return list(reversed(entries))
|
|
|
|
|
|
def get_backup_script_path():
|
|
candidates = ['/root/backup-myapps.sh']
|
|
for p in candidates:
|
|
out, _ = _ssh_main(f"[ -f {p} ] && echo yes || echo no")
|
|
if out.strip() == 'yes':
|
|
return p
|
|
return None
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# CONTAINERS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def _parse_uptime(status):
|
|
"""Human uptime snippet from docker ps Status field."""
|
|
if not status:
|
|
return '—'
|
|
if status.startswith('Up'):
|
|
m = re.match(r'Up\s+([^(]+)', status)
|
|
return ('Up ' + m.group(1).strip()) if m else status.split('(')[0].strip()
|
|
if status.startswith('Exited'):
|
|
return status.split('(')[0].strip() or status
|
|
return status
|
|
|
|
|
|
def _parse_containers(raw, owner='root'):
|
|
containers = []
|
|
if raw:
|
|
for line in raw.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
status = parts[1].strip() if len(parts) > 1 else ''
|
|
containers.append({
|
|
'name': parts[0].strip(),
|
|
'status': status,
|
|
'image': parts[2].strip() if len(parts) > 2 else '',
|
|
'ports': parts[3].strip() if len(parts) > 3 else '',
|
|
'created_at': parts[4].strip() if len(parts) > 4 else '',
|
|
'uptime': _parse_uptime(status),
|
|
'running': status.startswith('Up'),
|
|
'owner': owner,
|
|
})
|
|
return containers
|
|
|
|
|
|
_DOCKER_PS_FMT = (
|
|
"{{.Names}}|{{.Status}}|{{.Image}}|{{.Ports}}|{{.CreatedAt}}"
|
|
)
|
|
|
|
|
|
def get_containers():
|
|
stdout, _ = _ssh_main(
|
|
f"docker ps -a --format '{_DOCKER_PS_FMT}' 2>/dev/null | "
|
|
"grep -E 'frappe|nextcloud|mautic|n8n|odoo'"
|
|
)
|
|
return _parse_containers(stdout)
|
|
|
|
|
|
def get_all_root_containers():
|
|
stdout, _ = _ssh_main(
|
|
f"docker ps -a --format '{_DOCKER_PS_FMT}' 2>/dev/null"
|
|
)
|
|
return _parse_containers(stdout)
|
|
|
|
|
|
def get_rootless_user_containers_remote():
|
|
cmd = "ls /run/user/*/docker.sock 2>/dev/null"
|
|
stdout, _ = _ssh_main(cmd)
|
|
containers = []
|
|
if not stdout:
|
|
return containers
|
|
for sock_path in stdout.split('\n'):
|
|
sock_path = sock_path.strip()
|
|
if not sock_path:
|
|
continue
|
|
try:
|
|
uid = sock_path.split('/run/user/')[1].split('/')[0]
|
|
except (IndexError, ValueError):
|
|
continue
|
|
name_out, _ = _ssh_main(f"getent passwd {uid} | cut -d: -f1")
|
|
username = name_out.strip() or f"uid{uid}"
|
|
ctr_out, _ = _ssh_main(
|
|
f"DOCKER_HOST=unix://{sock_path} "
|
|
f"docker ps -a --format '{_DOCKER_PS_FMT}' 2>/dev/null"
|
|
)
|
|
containers.extend(_parse_containers(ctr_out, owner=username))
|
|
return containers
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# CONTAINER ACTIONS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def container_action(container_name, action):
|
|
if action not in ('start', 'stop', 'restart'):
|
|
return False, "Invalid action"
|
|
safe_name = container_name.replace('"', '').replace(';', '').replace('|', '')
|
|
stdout, stderr = _ssh_main(f"docker {action} {safe_name} 2>&1", timeout=30)
|
|
output = (stdout + stderr).strip()
|
|
return True, output
|
|
|
|
|
|
def get_container_status(container_name):
|
|
safe_name = container_name.replace('"', '').replace(';', '').replace('|', '')
|
|
stdout, _ = _ssh_main(
|
|
f"docker inspect --format='{{{{.State.Status}}}}' {safe_name} 2>/dev/null"
|
|
)
|
|
raw = stdout.strip().lower()
|
|
if raw in ('running', 'restarting'):
|
|
status = 'running'
|
|
elif raw in ('exited', 'stopped', 'dead', 'paused'):
|
|
status = 'stopped'
|
|
else:
|
|
status = 'unknown'
|
|
return {'name': container_name, 'status': status, 'raw': raw}
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# STATS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_container_stats_remote():
|
|
stdout, _ = _ssh_main(
|
|
"docker stats --no-stream --format "
|
|
"'{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}|{{.NetIO}}|{{.BlockIO}}' 2>/dev/null",
|
|
timeout=35
|
|
)
|
|
stats = {}
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
if len(parts) < 6:
|
|
continue
|
|
name = parts[0].strip()
|
|
stats[name] = {
|
|
'cpu': parts[1].strip(),
|
|
'mem': parts[2].strip(),
|
|
'mem_pct': parts[3].strip(),
|
|
'net': parts[4].strip(),
|
|
'block': parts[5].strip(),
|
|
}
|
|
return stats
|
|
|
|
|
|
def get_all_stats():
|
|
all_stats = get_container_stats_remote()
|
|
socks_out, _ = _ssh_main("ls /run/user/*/docker.sock 2>/dev/null")
|
|
if socks_out:
|
|
for sock in socks_out.split('\n'):
|
|
sock = sock.strip()
|
|
if not sock:
|
|
continue
|
|
stdout, _ = _ssh_main(
|
|
f"DOCKER_HOST=unix://{sock} "
|
|
f"docker stats --no-stream --format "
|
|
f"'{{{{.Name}}}}|{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.NetIO}}}}|{{{{.BlockIO}}}}' 2>/dev/null",
|
|
timeout=35
|
|
)
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
if len(parts) < 6:
|
|
continue
|
|
all_stats[parts[0].strip()] = {
|
|
'cpu': parts[1].strip(),
|
|
'mem': parts[2].strip(),
|
|
'mem_pct': parts[3].strip(),
|
|
'net': parts[4].strip(),
|
|
'block': parts[5].strip(),
|
|
}
|
|
return all_stats
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# SYSTEM INFO (FIXED WITH PSUTIL)
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def _get_main_server_system_info_remote():
|
|
"""get_system_info()'s equivalent for the standby: the main server's bare
|
|
host has no psutil (system python3, not this app's container image), so
|
|
this uses plain coreutils/procps instead — reusing _ssh_main means this
|
|
genuinely queries the real main server over the tunnel, not the VM
|
|
itself (the previous unconditional psutil.* calls silently reported the
|
|
VM's own stats mislabeled as "system info" whenever running as standby)."""
|
|
fallback = {
|
|
'cpu_pct': '0', 'memory': 'N/A', 'mem_pct': '0', 'disk': 'N/A',
|
|
'disk_pct': '0', 'load': 'N/A', 'uptime': 'N/A', 'docker_v': 'N/A',
|
|
'hostname': 'unreachable',
|
|
}
|
|
remote_cmd = (
|
|
"hostname; "
|
|
"vmstat 1 2 | tail -1 | awk '{print 100-$15}'; "
|
|
"free -m | awk 'NR==2{printf \"%d %d %d\\n\", $3, $2, $3*100/$2}'; "
|
|
"df -BM --output=used,size,pcent / | tail -1 | tr -d 'M%'; "
|
|
"awk '{printf \"%.2f %.2f %.2f\\n\", $1, $2, $3}' /proc/loadavg; "
|
|
"awk '{print $1}' /proc/uptime; "
|
|
"docker --version 2>/dev/null | awk '{print $3}' | tr -d ','"
|
|
)
|
|
out, err = _ssh_main(remote_cmd, timeout=15)
|
|
lines = out.split('\n') if out else []
|
|
if len(lines) < 7:
|
|
return fallback
|
|
try:
|
|
hostname = lines[0].strip()
|
|
cpu_pct = lines[1].strip()
|
|
mem_used_mb, mem_total_mb, mem_pct = lines[2].split()
|
|
disk_used_mb, disk_total_mb, disk_pct = lines[3].split()
|
|
load = lines[4].strip()
|
|
uptime_s = float(lines[5].strip())
|
|
docker_v = lines[6].strip() or 'N/A'
|
|
|
|
days = int(uptime_s // 86400)
|
|
hours = int((uptime_s % 86400) // 3600)
|
|
minutes = int((uptime_s % 3600) // 60)
|
|
uptime = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
|
|
|
|
return {
|
|
'cpu_pct': cpu_pct,
|
|
'memory': f"{int(mem_used_mb) / 1024:.1f}G/{int(mem_total_mb) / 1024:.1f}G",
|
|
'mem_pct': mem_pct,
|
|
'disk': f"{int(disk_used_mb) / 1024:.0f}G/{int(disk_total_mb) / 1024:.0f}G",
|
|
'disk_pct': disk_pct,
|
|
'load': load,
|
|
'uptime': uptime,
|
|
'docker_v': docker_v,
|
|
'hostname': hostname,
|
|
}
|
|
except Exception as e:
|
|
print(f"[backups] remote system info parse error: {e} (raw: {out!r}, err: {err!r})")
|
|
return fallback
|
|
|
|
|
|
def get_system_info():
|
|
"""Get system information using psutil for reliable metrics inside container"""
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
return _get_main_server_system_info_remote()
|
|
|
|
info = {
|
|
'cpu_pct': '0',
|
|
'memory': 'N/A',
|
|
'mem_pct': '0',
|
|
'disk': 'N/A',
|
|
'disk_pct': '0',
|
|
'load': 'N/A',
|
|
'uptime': 'N/A',
|
|
'docker_v': 'N/A',
|
|
'hostname': socket.gethostname()
|
|
}
|
|
|
|
try:
|
|
# CPU usage - psutil handles /proc access properly
|
|
info['cpu_pct'] = str(psutil.cpu_percent(interval=0.5))
|
|
|
|
# Memory usage
|
|
mem = psutil.virtual_memory()
|
|
mem_used_gb = mem.used / (1024**3)
|
|
mem_total_gb = mem.total / (1024**3)
|
|
info['memory'] = f"{mem_used_gb:.1f}G/{mem_total_gb:.1f}G"
|
|
info['mem_pct'] = str(int(mem.percent))
|
|
|
|
# Load average
|
|
try:
|
|
load = psutil.getloadavg()
|
|
info['load'] = f"{load[0]:.2f} {load[1]:.2f} {load[2]:.2f}"
|
|
except AttributeError:
|
|
# Fallback for systems without getloadavg
|
|
with open('/proc/loadavg', 'r') as f:
|
|
load = f.read().split()[:3]
|
|
info['load'] = ' '.join(load)
|
|
|
|
# Disk usage
|
|
try:
|
|
disk = psutil.disk_usage('/')
|
|
disk_used_gb = disk.used / (1024**3)
|
|
disk_total_gb = disk.total / (1024**3)
|
|
info['disk'] = f"{disk_used_gb:.0f}G/{disk_total_gb:.0f}G"
|
|
info['disk_pct'] = str(int(disk.percent))
|
|
except Exception as e:
|
|
print(f"Disk read error: {e}")
|
|
|
|
# Uptime
|
|
try:
|
|
boot_time = psutil.boot_time()
|
|
uptime_seconds = time.time() - boot_time
|
|
days = int(uptime_seconds // 86400)
|
|
hours = int((uptime_seconds % 86400) // 3600)
|
|
minutes = int((uptime_seconds % 3600) // 60)
|
|
info['uptime'] = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
|
|
except Exception as e:
|
|
print(f"Uptime read error: {e}")
|
|
|
|
# Docker version
|
|
try:
|
|
docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True)
|
|
info['docker_v'] = docker_v.split()[-1].strip(',')
|
|
except Exception as e:
|
|
print(f"Docker version error: {e}")
|
|
|
|
except Exception as e:
|
|
print(f"System info error: {e}")
|
|
# Fallback to /proc method if psutil fails
|
|
info = _get_system_info_fallback()
|
|
|
|
return info
|
|
|
|
|
|
def _get_system_info_fallback():
|
|
"""Fallback method using /proc directly when psutil is not available"""
|
|
info = {
|
|
'cpu_pct': '0',
|
|
'memory': 'N/A',
|
|
'mem_pct': '0',
|
|
'disk': 'N/A',
|
|
'disk_pct': '0',
|
|
'load': 'N/A',
|
|
'uptime': 'N/A',
|
|
'docker_v': 'N/A',
|
|
'hostname': socket.gethostname()
|
|
}
|
|
|
|
try:
|
|
# CPU calculation with delta
|
|
def _read_cpu():
|
|
with open('/proc/stat', 'r') as f:
|
|
fields = f.readline().split()
|
|
idle = int(fields[4])
|
|
total = sum(int(x) for x in fields[1:])
|
|
return idle, total
|
|
|
|
idle1, total1 = _read_cpu()
|
|
time.sleep(0.5)
|
|
idle2, total2 = _read_cpu()
|
|
|
|
idle_delta = idle2 - idle1
|
|
total_delta = total2 - total1
|
|
if total_delta > 0:
|
|
cpu_pct = round(100 * (1 - idle_delta / total_delta), 1)
|
|
info['cpu_pct'] = str(cpu_pct)
|
|
|
|
# Memory
|
|
mem = {}
|
|
with open('/proc/meminfo', 'r') as f:
|
|
for line in f:
|
|
parts = line.split(':')
|
|
if len(parts) == 2:
|
|
mem[parts[0].strip()] = int(parts[1].strip().split()[0])
|
|
|
|
if 'MemTotal' in mem and 'MemAvailable' in mem:
|
|
mem_total_mb = mem['MemTotal'] // 1024
|
|
mem_used_mb = (mem['MemTotal'] - mem['MemAvailable']) // 1024
|
|
info['memory'] = f"{mem_used_mb}M/{mem_total_mb}M"
|
|
info['mem_pct'] = str(int(100 * mem_used_mb / mem_total_mb))
|
|
|
|
# Load
|
|
with open('/proc/loadavg', 'r') as f:
|
|
load = f.read().split()[:3]
|
|
info['load'] = ' '.join(load)
|
|
|
|
# Uptime
|
|
with open('/proc/uptime', 'r') as f:
|
|
secs = int(float(f.read().split()[0]))
|
|
days = secs // 86400
|
|
hours = (secs % 86400) // 3600
|
|
minutes = (secs % 3600) // 60
|
|
info['uptime'] = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
|
|
|
|
# Disk
|
|
try:
|
|
disk = psutil.disk_usage('/')
|
|
disk_used_gb = disk.used / (1024**3)
|
|
disk_total_gb = disk.total / (1024**3)
|
|
info['disk'] = f"{disk_used_gb:.0f}G/{disk_total_gb:.0f}G"
|
|
info['disk_pct'] = str(int(disk.percent))
|
|
except Exception:
|
|
import shutil
|
|
disk = shutil.disk_usage('/')
|
|
info['disk'] = f"{disk.used // (1024**3)}G/{disk.total // (1024**3)}G"
|
|
info['disk_pct'] = str(int(100 * disk.used / disk.total))
|
|
|
|
# Docker version
|
|
try:
|
|
docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True)
|
|
info['docker_v'] = docker_v.split()[-1].strip(',')
|
|
except Exception:
|
|
pass
|
|
|
|
except Exception as e:
|
|
print(f"Fallback system info error: {e}")
|
|
|
|
return info |