Add SSH+kubectl remote fallback so the standby actually shows cluster/app health
Two real gaps found after testing the live standby platform: 1. modules/kubernetes.py's in-cluster client (load_incluster_config()) can only ever work inside the real k8s pod. On the standby it silently returned K8S_AVAILABLE=False with zero fallback — Cluster, Containers- via-sites, and any cluster/app health data was just empty on that page, not erroring, so it was easy to miss. This holds even if the standby later runs as a pod on a *local* k3s on that server (per plan for the monitoring migration) — in-cluster config there would only ever see that local, unrelated cluster, never the main one. Fixed by adding an SSH+kubectl remote path to list_pods(), list_ deployments(), get_ingress_info(), list_deployments_for_namespace(), and get_pod_metrics(): when not RUNNING_ON_MAIN_SERVER, each runs `kubectl <verb> -o json` on the main server over the tunnel (main-to-vm-tunnel.service) instead of using the python client, parses the raw (camelCase) JSON directly rather than trying to force it through the client-library's snake_case object model, and feeds the same _pod_health/_deploy_health/_age_str helpers either way. The in-cluster path for the real main-server pod is untouched. 2. get_system_info() called psutil.* unconditionally with no RUNNING_ON_MAIN_SERVER awareness at all — on the standby it was silently reporting the VM's OWN cpu/mem/disk/hostname mislabeled as "system info" (hostname field literally showed the VM's hostname). Added _get_main_server_system_info_remote(): SSHes to the main server and gathers the same stats via vmstat/free/df/procfs (the main server's bare host has no psutil — that's only in this app's own container image — so this avoids depending on it being present remotely). Verified end-to-end on the live standby: /api/system now reports the real main server's hostname/cpu/mem/disk/uptime; /api/cluster reports real counts (19 pods, 16/16 deployments healthy, 3 degraded, plus live per-pod metrics via the metrics-server raw API); /api/sites shows real per-app container/role status. All previously silently empty.
This commit is contained in:
@@ -895,8 +895,65 @@ def get_all_stats():
|
||||
# SYSTEM INFO (FIXED WITH PSUTIL)
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _get_main_server_system_info_remote():
|
||||
"""get_system_info()'s equivalent for the standby: the main server's bare
|
||||
host has no psutil (system python3, not this app's container image), so
|
||||
this uses plain coreutils/procps instead — reusing _ssh_main means this
|
||||
genuinely queries the real main server over the tunnel, not the VM
|
||||
itself (the previous unconditional psutil.* calls silently reported the
|
||||
VM's own stats mislabeled as "system info" whenever running as standby)."""
|
||||
fallback = {
|
||||
'cpu_pct': '0', 'memory': 'N/A', 'mem_pct': '0', 'disk': 'N/A',
|
||||
'disk_pct': '0', 'load': 'N/A', 'uptime': 'N/A', 'docker_v': 'N/A',
|
||||
'hostname': 'unreachable',
|
||||
}
|
||||
remote_cmd = (
|
||||
"hostname; "
|
||||
"vmstat 1 2 | tail -1 | awk '{print 100-$15}'; "
|
||||
"free -m | awk 'NR==2{printf \"%d %d %d\\n\", $3, $2, $3*100/$2}'; "
|
||||
"df -BM --output=used,size,pcent / | tail -1 | tr -d 'M%'; "
|
||||
"awk '{printf \"%.2f %.2f %.2f\\n\", $1, $2, $3}' /proc/loadavg; "
|
||||
"awk '{print $1}' /proc/uptime; "
|
||||
"docker --version 2>/dev/null | awk '{print $3}' | tr -d ','"
|
||||
)
|
||||
out, err = _ssh_main(remote_cmd, timeout=15)
|
||||
lines = out.split('\n') if out else []
|
||||
if len(lines) < 7:
|
||||
return fallback
|
||||
try:
|
||||
hostname = lines[0].strip()
|
||||
cpu_pct = lines[1].strip()
|
||||
mem_used_mb, mem_total_mb, mem_pct = lines[2].split()
|
||||
disk_used_mb, disk_total_mb, disk_pct = lines[3].split()
|
||||
load = lines[4].strip()
|
||||
uptime_s = float(lines[5].strip())
|
||||
docker_v = lines[6].strip() or 'N/A'
|
||||
|
||||
days = int(uptime_s // 86400)
|
||||
hours = int((uptime_s % 86400) // 3600)
|
||||
minutes = int((uptime_s % 3600) // 60)
|
||||
uptime = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
|
||||
|
||||
return {
|
||||
'cpu_pct': cpu_pct,
|
||||
'memory': f"{int(mem_used_mb) / 1024:.1f}G/{int(mem_total_mb) / 1024:.1f}G",
|
||||
'mem_pct': mem_pct,
|
||||
'disk': f"{int(disk_used_mb) / 1024:.0f}G/{int(disk_total_mb) / 1024:.0f}G",
|
||||
'disk_pct': disk_pct,
|
||||
'load': load,
|
||||
'uptime': uptime,
|
||||
'docker_v': docker_v,
|
||||
'hostname': hostname,
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"[backups] remote system info parse error: {e} (raw: {out!r}, err: {err!r})")
|
||||
return fallback
|
||||
|
||||
|
||||
def get_system_info():
|
||||
"""Get system information using psutil for reliable metrics inside container"""
|
||||
if not RUNNING_ON_MAIN_SERVER:
|
||||
return _get_main_server_system_info_remote()
|
||||
|
||||
info = {
|
||||
'cpu_pct': '0',
|
||||
|
||||
Reference in New Issue
Block a user