Add SSH+kubectl remote fallback so the standby actually shows cluster/app health

Two real gaps found after testing the live standby platform:

1. modules/kubernetes.py's in-cluster client (load_incluster_config())
   can only ever work inside the real k8s pod. On the standby it silently
   returned K8S_AVAILABLE=False with zero fallback — Cluster, Containers-
   via-sites, and any cluster/app health data was just empty on that page,
   not erroring, so it was easy to miss. This holds even if the standby
   later runs as a pod on a *local* k3s on that server (per plan for the
   monitoring migration) — in-cluster config there would only ever see
   that local, unrelated cluster, never the main one.

   Fixed by adding an SSH+kubectl remote path to list_pods(), list_
   deployments(), get_ingress_info(), list_deployments_for_namespace(),
   and get_pod_metrics(): when not RUNNING_ON_MAIN_SERVER, each runs
   `kubectl <verb> -o json` on the main server over the tunnel
   (main-to-vm-tunnel.service) instead of using the python client, parses
   the raw (camelCase) JSON directly rather than trying to force it
   through the client-library's snake_case object model, and feeds the
   same _pod_health/_deploy_health/_age_str helpers either way. The
   in-cluster path for the real main-server pod is untouched.

2. get_system_info() called psutil.* unconditionally with no
   RUNNING_ON_MAIN_SERVER awareness at all — on the standby it was
   silently reporting the VM's OWN cpu/mem/disk/hostname mislabeled as
   "system info" (hostname field literally showed the VM's hostname).
   Added _get_main_server_system_info_remote(): SSHes to the main server
   and gathers the same stats via vmstat/free/df/procfs (the main
   server's bare host has no psutil — that's only in this app's own
   container image — so this avoids depending on it being present
   remotely).

Verified end-to-end on the live standby: /api/system now reports the
real main server's hostname/cpu/mem/disk/uptime; /api/cluster reports
real counts (19 pods, 16/16 deployments healthy, 3 degraded, plus live
per-pod metrics via the metrics-server raw API); /api/sites shows real
per-app container/role status. All previously silently empty.

platform/modules/backups.py

@@ -895,8 +895,65 @@ def get_all_stats():
 # SYSTEM INFO (FIXED WITH PSUTIL)
 # ────────────────────────────────────────────────────────────────
 
+def _get_main_server_system_info_remote():
+    """get_system_info()'s equivalent for the standby: the main server's bare
+    host has no psutil (system python3, not this app's container image), so
+    this uses plain coreutils/procps instead — reusing _ssh_main means this
+    genuinely queries the real main server over the tunnel, not the VM
+    itself (the previous unconditional psutil.* calls silently reported the
+    VM's own stats mislabeled as "system info" whenever running as standby)."""
+    fallback = {
+        'cpu_pct': '0', 'memory': 'N/A', 'mem_pct': '0', 'disk': 'N/A',
+        'disk_pct': '0', 'load': 'N/A', 'uptime': 'N/A', 'docker_v': 'N/A',
+        'hostname': 'unreachable',
+    }
+    remote_cmd = (
+        "hostname; "
+        "vmstat 1 2 | tail -1 | awk '{print 100-$15}'; "
+        "free -m | awk 'NR==2{printf \"%d %d %d\\n\", $3, $2, $3*100/$2}'; "
+        "df -BM --output=used,size,pcent / | tail -1 | tr -d 'M%'; "
+        "awk '{printf \"%.2f %.2f %.2f\\n\", $1, $2, $3}' /proc/loadavg; "
+        "awk '{print $1}' /proc/uptime; "
+        "docker --version 2>/dev/null | awk '{print $3}' | tr -d ','"
+    )
+    out, err = _ssh_main(remote_cmd, timeout=15)
+    lines = out.split('\n') if out else []
+    if len(lines) < 7:
+        return fallback
+    try:
+        hostname = lines[0].strip()
+        cpu_pct = lines[1].strip()
+        mem_used_mb, mem_total_mb, mem_pct = lines[2].split()
+        disk_used_mb, disk_total_mb, disk_pct = lines[3].split()
+        load = lines[4].strip()
+        uptime_s = float(lines[5].strip())
+        docker_v = lines[6].strip() or 'N/A'
+
+        days = int(uptime_s // 86400)
+        hours = int((uptime_s % 86400) // 3600)
+        minutes = int((uptime_s % 3600) // 60)
+        uptime = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
+
+        return {
+            'cpu_pct':  cpu_pct,
+            'memory':   f"{int(mem_used_mb) / 1024:.1f}G/{int(mem_total_mb) / 1024:.1f}G",
+            'mem_pct':  mem_pct,
+            'disk':     f"{int(disk_used_mb) / 1024:.0f}G/{int(disk_total_mb) / 1024:.0f}G",
+            'disk_pct': disk_pct,
+            'load':     load,
+            'uptime':   uptime,
+            'docker_v': docker_v,
+            'hostname': hostname,
+        }
+    except Exception as e:
+        print(f"[backups] remote system info parse error: {e} (raw: {out!r}, err: {err!r})")
+        return fallback
+
+
 def get_system_info():
     """Get system information using psutil for reliable metrics inside container"""
+    if not RUNNING_ON_MAIN_SERVER:
+        return _get_main_server_system_info_remote()
 
     info = {
         'cpu_pct': '0',

platform/modules/kubernetes.py

@@ -4,6 +4,18 @@
 # deployments.apps — no secrets, no write, no exec/log), bound per-namespace
 # across the 8 app namespaces below. There is no cluster-wide RoleBinding,
 # so every list call below is scoped to one namespace at a time.
+#
+# On the standby (RUNNING_ON_MAIN_SERVER False), there is no in-cluster
+# ServiceAccount to load at all — load_incluster_config() always fails
+# there, by design, even if the standby itself later runs as a pod on a
+# *local* k3s on that VM (that cluster isn't this one; in-cluster config
+# would only ever see the wrong, essentially-empty cluster). Every function
+# below instead SSHes to the main server (over main-to-vm-tunnel.service)
+# and runs `kubectl` there directly, where real cluster-admin access
+# already exists — same real-vs-display separation as MAIN_SERVER_SSH_HOST
+# in config.py.
+import json
+import subprocess
 from datetime import datetime, timezone
 
 try:
@@ -13,6 +25,11 @@ try:
 except ImportError:
     _K8S_IMPORT_OK = False
 
+from config import (
+    RUNNING_ON_MAIN_SERVER, MAIN_SERVER_USER, MAIN_SERVER_KEY,
+    MAIN_SERVER_SSH_HOST, MAIN_SERVER_SSH_PORT,
+)
+
 NAMESPACES = [
     'n8n', 'odoo', 'mautic', 'erpnext', 'nextcloud',
     'jenkins-agents', 'jenkins', 'management-platform',
@@ -25,6 +42,78 @@ _networking_v1 = None
 K8S_AVAILABLE = False
 
 
+def _ssh_kubectl(args):
+    """Run `kubectl <args> -o json` on the main server over the reverse
+    tunnel. Returns the parsed JSON (dict) or None on any failure —
+    callers must treat None/empty the same as the in-cluster client
+    returning nothing, never as a hard error."""
+    remote_cmd = "kubectl " + " ".join(args) + " -o json"
+    ssh_cmd = (
+        f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} "
+        f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes "
+        f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} '{remote_cmd}'"
+    )
+    try:
+        r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20)
+        if r.returncode != 0:
+            return None
+        return json.loads(r.stdout)
+    except Exception:
+        return None
+
+
+def _ssh_kubectl_raw(path):
+    """Same as _ssh_kubectl but for `kubectl get --raw <path>` (used for
+    the metrics-server API, which isn't a normal `get <resource>` call)."""
+    ssh_cmd = (
+        f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} "
+        f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes "
+        f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} 'kubectl get --raw {path}'"
+    )
+    try:
+        r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20)
+        if r.returncode != 0:
+            return None
+        return json.loads(r.stdout)
+    except Exception:
+        return None
+
+
+def _parse_k8s_ts(s):
+    if not s:
+        return None
+    try:
+        return datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)
+    except Exception:
+        return None
+
+
+def _pod_health_raw(pod):
+    statuses = (pod.get('status') or {}).get('containerStatuses') or []
+    for cs in statuses:
+        waiting = (cs.get('state') or {}).get('waiting')
+        if waiting and waiting.get('reason') in _CRASH_REASONS:
+            return 'failed'
+    phase = (pod.get('status') or {}).get('phase') or 'Unknown'
+    if phase == 'Failed':
+        return 'failed'
+    if phase == 'Running':
+        all_ready = all(cs.get('ready') for cs in statuses) if statuses else True
+        return 'healthy' if all_ready else 'degraded'
+    return 'degraded'
+
+
+def _ready_count_raw(pod):
+    statuses = (pod.get('status') or {}).get('containerStatuses') or []
+    ready = sum(1 for cs in statuses if cs.get('ready'))
+    return f'{ready}/{len(statuses)}'
+
+
+def _restart_count_raw(pod):
+    statuses = (pod.get('status') or {}).get('containerStatuses') or []
+    return sum(cs.get('restartCount', 0) for cs in statuses)
+
+
 def _init_client():
     global _core_v1, _apps_v1, _custom_objects, _networking_v1, K8S_AVAILABLE
     if _core_v1 is not None or not _K8S_IMPORT_OK:
@@ -135,6 +224,24 @@ def _parse_mem_mib(v):
 # ────────────────────────────────────────────────────────────────
 
 def list_pods():
+    if not RUNNING_ON_MAIN_SERVER:
+        pods = []
+        for ns in NAMESPACES:
+            data = _ssh_kubectl(['get', 'pods', '-n', ns])
+            if not data:
+                continue
+            for pod in data.get('items', []):
+                pods.append({
+                    'name':      pod['metadata']['name'],
+                    'namespace': ns,
+                    'phase':     (pod.get('status') or {}).get('phase') or 'Unknown',
+                    'health':    _pod_health_raw(pod),
+                    'ready':     _ready_count_raw(pod),
+                    'restarts':  _restart_count_raw(pod),
+                    'age':       _age_str(_parse_k8s_ts(pod['metadata'].get('creationTimestamp'))),
+                })
+        return pods
+
     _init_client()
     pods = []
     if not K8S_AVAILABLE:
@@ -159,6 +266,25 @@ def list_pods():
 
 
 def list_deployments():
+    if not RUNNING_ON_MAIN_SERVER:
+        deployments = []
+        for ns in NAMESPACES:
+            data = _ssh_kubectl(['get', 'deployments', '-n', ns])
+            if not data:
+                continue
+            for dep in data.get('items', []):
+                desired = (dep.get('spec') or {}).get('replicas') or 0
+                ready = (dep.get('status') or {}).get('readyReplicas') or 0
+                deployments.append({
+                    'name':      dep['metadata']['name'],
+                    'namespace': ns,
+                    'ready':     ready,
+                    'desired':   desired,
+                    'health':    _deploy_health(desired, ready),
+                    'age':       _age_str(_parse_k8s_ts(dep['metadata'].get('creationTimestamp'))),
+                })
+        return deployments
+
     _init_client()
     deployments = []
     if not K8S_AVAILABLE:
@@ -187,6 +313,19 @@ def get_ingress_info(name, namespace):
     """Real domain/TLS for one Ingress, or None if unavailable/not found.
     Used by modules/sites.py instead of the old hardcoded Docker-era
     domain fields — reflects actual live k8s Ingress state."""
+    if not RUNNING_ON_MAIN_SERVER:
+        data = _ssh_kubectl(['get', 'ingress', name, '-n', namespace])
+        if not data:
+            return None
+        host = None
+        rules = (data.get('spec') or {}).get('rules') or []
+        for rule in rules:
+            if rule.get('host'):
+                host = rule['host']
+                break
+        tls = bool((data.get('spec') or {}).get('tls'))
+        return {'host': host, 'tls': tls}
+
     _init_client()
     if not K8S_AVAILABLE:
         return None
@@ -207,6 +346,25 @@ def get_ingress_info(name, namespace):
 def list_deployments_for_namespace(ns):
     """Targeted single-namespace deployment list (vs. list_deployments()'s
     fixed NAMESPACES sweep) — used by modules/sites.py per app."""
+    if not RUNNING_ON_MAIN_SERVER:
+        data = _ssh_kubectl(['get', 'deployments', '-n', ns])
+        if not data:
+            return []
+        out = []
+        for dep in data.get('items', []):
+            desired = (dep.get('spec') or {}).get('replicas') or 0
+            ready = (dep.get('status') or {}).get('readyReplicas') or 0
+            containers = (((dep.get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or []
+            image = containers[0]['image'] if containers else '—'
+            out.append({
+                'name':    dep['metadata']['name'],
+                'desired': desired,
+                'ready':   ready,
+                'health':  _deploy_health(desired, ready),
+                'image':   image,
+            })
+        return out
+
     _init_client()
     if not K8S_AVAILABLE:
         return []
@@ -235,6 +393,20 @@ def get_pod_metrics():
     """Best-effort CPU/mem per pod via metrics-server. Returns {} if it's not
     installed or unreachable — callers must treat an empty dict as 'no data',
     never as an error."""
+    if not RUNNING_ON_MAIN_SERVER:
+        metrics = {}
+        for ns in NAMESPACES:
+            resp = _ssh_kubectl_raw(f'/apis/metrics.k8s.io/v1beta1/namespaces/{ns}/pods')
+            if not resp:
+                continue
+            for item in resp.get('items', []):
+                name = item.get('metadata', {}).get('name')
+                containers = item.get('containers', [])
+                cpu = sum(_parse_cpu(c.get('usage', {}).get('cpu')) for c in containers)
+                mem = sum(_parse_mem_mib(c.get('usage', {}).get('memory')) for c in containers)
+                metrics[f'{ns}/{name}'] = {'cpu_millicores': round(cpu), 'mem_mib': round(mem)}
+        return metrics
+
     _init_client()
     metrics = {}
     if not K8S_AVAILABLE:
@@ -283,8 +455,13 @@ def get_cluster_overview():
         'deployments_healthy': sum(1 for d in deployments if d['health'] == 'healthy'),
     }
 
+    # K8S_AVAILABLE only reflects the in-cluster client (main server); on
+    # the standby, "available" instead means the SSH+kubectl fetch above
+    # actually returned something.
+    available = K8S_AVAILABLE if RUNNING_ON_MAIN_SERVER else bool(pods or deployments)
+
     return {
-        'available':        K8S_AVAILABLE,
+        'available':        available,
         'namespaces':        by_namespace,
         'summary':           summary,
         'metrics_available': bool(metrics),