Add SSH+kubectl remote fallback so the standby actually shows cluster/app health
Two real gaps found after testing the live standby platform: 1. modules/kubernetes.py's in-cluster client (load_incluster_config()) can only ever work inside the real k8s pod. On the standby it silently returned K8S_AVAILABLE=False with zero fallback — Cluster, Containers- via-sites, and any cluster/app health data was just empty on that page, not erroring, so it was easy to miss. This holds even if the standby later runs as a pod on a *local* k3s on that server (per plan for the monitoring migration) — in-cluster config there would only ever see that local, unrelated cluster, never the main one. Fixed by adding an SSH+kubectl remote path to list_pods(), list_ deployments(), get_ingress_info(), list_deployments_for_namespace(), and get_pod_metrics(): when not RUNNING_ON_MAIN_SERVER, each runs `kubectl <verb> -o json` on the main server over the tunnel (main-to-vm-tunnel.service) instead of using the python client, parses the raw (camelCase) JSON directly rather than trying to force it through the client-library's snake_case object model, and feeds the same _pod_health/_deploy_health/_age_str helpers either way. The in-cluster path for the real main-server pod is untouched. 2. get_system_info() called psutil.* unconditionally with no RUNNING_ON_MAIN_SERVER awareness at all — on the standby it was silently reporting the VM's OWN cpu/mem/disk/hostname mislabeled as "system info" (hostname field literally showed the VM's hostname). Added _get_main_server_system_info_remote(): SSHes to the main server and gathers the same stats via vmstat/free/df/procfs (the main server's bare host has no psutil — that's only in this app's own container image — so this avoids depending on it being present remotely). Verified end-to-end on the live standby: /api/system now reports the real main server's hostname/cpu/mem/disk/uptime; /api/cluster reports real counts (19 pods, 16/16 deployments healthy, 3 degraded, plus live per-pod metrics via the metrics-server raw API); /api/sites shows real per-app container/role status. All previously silently empty.
This commit is contained in:
root
@@ -895,8 +895,65 @@ def get_all_stats():
|
||||
# SYSTEM INFO (FIXED WITH PSUTIL)
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _get_main_server_system_info_remote():
|
||||
"""get_system_info()'s equivalent for the standby: the main server's bare
|
||||
host has no psutil (system python3, not this app's container image), so
|
||||
this uses plain coreutils/procps instead — reusing _ssh_main means this
|
||||
genuinely queries the real main server over the tunnel, not the VM
|
||||
itself (the previous unconditional psutil.* calls silently reported the
|
||||
VM's own stats mislabeled as "system info" whenever running as standby)."""
|
||||
fallback = {
|
||||
'cpu_pct': '0', 'memory': 'N/A', 'mem_pct': '0', 'disk': 'N/A',
|
||||
'disk_pct': '0', 'load': 'N/A', 'uptime': 'N/A', 'docker_v': 'N/A',
|
||||
'hostname': 'unreachable',
|
||||
}
|
||||
remote_cmd = (
|
||||
"hostname; "
|
||||
"vmstat 1 2 | tail -1 | awk '{print 100-$15}'; "
|
||||
"free -m | awk 'NR==2{printf \"%d %d %d\\n\", $3, $2, $3*100/$2}'; "
|
||||
"df -BM --output=used,size,pcent / | tail -1 | tr -d 'M%'; "
|
||||
"awk '{printf \"%.2f %.2f %.2f\\n\", $1, $2, $3}' /proc/loadavg; "
|
||||
"awk '{print $1}' /proc/uptime; "
|
||||
"docker --version 2>/dev/null | awk '{print $3}' | tr -d ','"
|
||||
)
|
||||
out, err = _ssh_main(remote_cmd, timeout=15)
|
||||
lines = out.split('\n') if out else []
|
||||
if len(lines) < 7:
|
||||
return fallback
|
||||
try:
|
||||
hostname = lines[0].strip()
|
||||
cpu_pct = lines[1].strip()
|
||||
mem_used_mb, mem_total_mb, mem_pct = lines[2].split()
|
||||
disk_used_mb, disk_total_mb, disk_pct = lines[3].split()
|
||||
load = lines[4].strip()
|
||||
uptime_s = float(lines[5].strip())
|
||||
docker_v = lines[6].strip() or 'N/A'
|
||||
|
||||
days = int(uptime_s // 86400)
|
||||
hours = int((uptime_s % 86400) // 3600)
|
||||
minutes = int((uptime_s % 3600) // 60)
|
||||
uptime = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
|
||||
|
||||
return {
|
||||
'cpu_pct': cpu_pct,
|
||||
'memory': f"{int(mem_used_mb) / 1024:.1f}G/{int(mem_total_mb) / 1024:.1f}G",
|
||||
'mem_pct': mem_pct,
|
||||
'disk': f"{int(disk_used_mb) / 1024:.0f}G/{int(disk_total_mb) / 1024:.0f}G",
|
||||
'disk_pct': disk_pct,
|
||||
'load': load,
|
||||
'uptime': uptime,
|
||||
'docker_v': docker_v,
|
||||
'hostname': hostname,
|
||||
}
|
||||
except Exception as e:
|
||||
print(f"[backups] remote system info parse error: {e} (raw: {out!r}, err: {err!r})")
|
||||
return fallback
|
||||
|
||||
|
||||
def get_system_info():
|
||||
"""Get system information using psutil for reliable metrics inside container"""
|
||||
if not RUNNING_ON_MAIN_SERVER:
|
||||
return _get_main_server_system_info_remote()
|
||||
|
||||
info = {
|
||||
'cpu_pct': '0',
|
||||
|
||||
@@ -4,6 +4,18 @@
|
||||
# deployments.apps — no secrets, no write, no exec/log), bound per-namespace
|
||||
# across the 8 app namespaces below. There is no cluster-wide RoleBinding,
|
||||
# so every list call below is scoped to one namespace at a time.
|
||||
#
|
||||
# On the standby (RUNNING_ON_MAIN_SERVER False), there is no in-cluster
|
||||
# ServiceAccount to load at all — load_incluster_config() always fails
|
||||
# there, by design, even if the standby itself later runs as a pod on a
|
||||
# *local* k3s on that VM (that cluster isn't this one; in-cluster config
|
||||
# would only ever see the wrong, essentially-empty cluster). Every function
|
||||
# below instead SSHes to the main server (over main-to-vm-tunnel.service)
|
||||
# and runs `kubectl` there directly, where real cluster-admin access
|
||||
# already exists — same real-vs-display separation as MAIN_SERVER_SSH_HOST
|
||||
# in config.py.
|
||||
import json
|
||||
import subprocess
|
||||
from datetime import datetime, timezone
|
||||
|
||||
try:
|
||||
@@ -13,6 +25,11 @@ try:
|
||||
except ImportError:
|
||||
_K8S_IMPORT_OK = False
|
||||
|
||||
from config import (
|
||||
RUNNING_ON_MAIN_SERVER, MAIN_SERVER_USER, MAIN_SERVER_KEY,
|
||||
MAIN_SERVER_SSH_HOST, MAIN_SERVER_SSH_PORT,
|
||||
)
|
||||
|
||||
NAMESPACES = [
|
||||
'n8n', 'odoo', 'mautic', 'erpnext', 'nextcloud',
|
||||
'jenkins-agents', 'jenkins', 'management-platform',
|
||||
@@ -25,6 +42,78 @@ _networking_v1 = None
|
||||
K8S_AVAILABLE = False
|
||||
|
||||
|
||||
def _ssh_kubectl(args):
|
||||
"""Run `kubectl <args> -o json` on the main server over the reverse
|
||||
tunnel. Returns the parsed JSON (dict) or None on any failure —
|
||||
callers must treat None/empty the same as the in-cluster client
|
||||
returning nothing, never as a hard error."""
|
||||
remote_cmd = "kubectl " + " ".join(args) + " -o json"
|
||||
ssh_cmd = (
|
||||
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} "
|
||||
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes "
|
||||
f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} '{remote_cmd}'"
|
||||
)
|
||||
try:
|
||||
r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return json.loads(r.stdout)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _ssh_kubectl_raw(path):
|
||||
"""Same as _ssh_kubectl but for `kubectl get --raw <path>` (used for
|
||||
the metrics-server API, which isn't a normal `get <resource>` call)."""
|
||||
ssh_cmd = (
|
||||
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} "
|
||||
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes "
|
||||
f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} 'kubectl get --raw {path}'"
|
||||
)
|
||||
try:
|
||||
r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20)
|
||||
if r.returncode != 0:
|
||||
return None
|
||||
return json.loads(r.stdout)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_k8s_ts(s):
|
||||
if not s:
|
||||
return None
|
||||
try:
|
||||
return datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def _pod_health_raw(pod):
|
||||
statuses = (pod.get('status') or {}).get('containerStatuses') or []
|
||||
for cs in statuses:
|
||||
waiting = (cs.get('state') or {}).get('waiting')
|
||||
if waiting and waiting.get('reason') in _CRASH_REASONS:
|
||||
return 'failed'
|
||||
phase = (pod.get('status') or {}).get('phase') or 'Unknown'
|
||||
if phase == 'Failed':
|
||||
return 'failed'
|
||||
if phase == 'Running':
|
||||
all_ready = all(cs.get('ready') for cs in statuses) if statuses else True
|
||||
return 'healthy' if all_ready else 'degraded'
|
||||
return 'degraded'
|
||||
|
||||
|
||||
def _ready_count_raw(pod):
|
||||
statuses = (pod.get('status') or {}).get('containerStatuses') or []
|
||||
ready = sum(1 for cs in statuses if cs.get('ready'))
|
||||
return f'{ready}/{len(statuses)}'
|
||||
|
||||
|
||||
def _restart_count_raw(pod):
|
||||
statuses = (pod.get('status') or {}).get('containerStatuses') or []
|
||||
return sum(cs.get('restartCount', 0) for cs in statuses)
|
||||
|
||||
|
||||
def _init_client():
|
||||
global _core_v1, _apps_v1, _custom_objects, _networking_v1, K8S_AVAILABLE
|
||||
if _core_v1 is not None or not _K8S_IMPORT_OK:
|
||||
@@ -135,6 +224,24 @@ def _parse_mem_mib(v):
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def list_pods():
|
||||
if not RUNNING_ON_MAIN_SERVER:
|
||||
pods = []
|
||||
for ns in NAMESPACES:
|
||||
data = _ssh_kubectl(['get', 'pods', '-n', ns])
|
||||
if not data:
|
||||
continue
|
||||
for pod in data.get('items', []):
|
||||
pods.append({
|
||||
'name': pod['metadata']['name'],
|
||||
'namespace': ns,
|
||||
'phase': (pod.get('status') or {}).get('phase') or 'Unknown',
|
||||
'health': _pod_health_raw(pod),
|
||||
'ready': _ready_count_raw(pod),
|
||||
'restarts': _restart_count_raw(pod),
|
||||
'age': _age_str(_parse_k8s_ts(pod['metadata'].get('creationTimestamp'))),
|
||||
})
|
||||
return pods
|
||||
|
||||
_init_client()
|
||||
pods = []
|
||||
if not K8S_AVAILABLE:
|
||||
@@ -159,6 +266,25 @@ def list_pods():
|
||||
|
||||
|
||||
def list_deployments():
|
||||
if not RUNNING_ON_MAIN_SERVER:
|
||||
deployments = []
|
||||
for ns in NAMESPACES:
|
||||
data = _ssh_kubectl(['get', 'deployments', '-n', ns])
|
||||
if not data:
|
||||
continue
|
||||
for dep in data.get('items', []):
|
||||
desired = (dep.get('spec') or {}).get('replicas') or 0
|
||||
ready = (dep.get('status') or {}).get('readyReplicas') or 0
|
||||
deployments.append({
|
||||
'name': dep['metadata']['name'],
|
||||
'namespace': ns,
|
||||
'ready': ready,
|
||||
'desired': desired,
|
||||
'health': _deploy_health(desired, ready),
|
||||
'age': _age_str(_parse_k8s_ts(dep['metadata'].get('creationTimestamp'))),
|
||||
})
|
||||
return deployments
|
||||
|
||||
_init_client()
|
||||
deployments = []
|
||||
if not K8S_AVAILABLE:
|
||||
@@ -187,6 +313,19 @@ def get_ingress_info(name, namespace):
|
||||
"""Real domain/TLS for one Ingress, or None if unavailable/not found.
|
||||
Used by modules/sites.py instead of the old hardcoded Docker-era
|
||||
domain fields — reflects actual live k8s Ingress state."""
|
||||
if not RUNNING_ON_MAIN_SERVER:
|
||||
data = _ssh_kubectl(['get', 'ingress', name, '-n', namespace])
|
||||
if not data:
|
||||
return None
|
||||
host = None
|
||||
rules = (data.get('spec') or {}).get('rules') or []
|
||||
for rule in rules:
|
||||
if rule.get('host'):
|
||||
host = rule['host']
|
||||
break
|
||||
tls = bool((data.get('spec') or {}).get('tls'))
|
||||
return {'host': host, 'tls': tls}
|
||||
|
||||
_init_client()
|
||||
if not K8S_AVAILABLE:
|
||||
return None
|
||||
@@ -207,6 +346,25 @@ def get_ingress_info(name, namespace):
|
||||
def list_deployments_for_namespace(ns):
|
||||
"""Targeted single-namespace deployment list (vs. list_deployments()'s
|
||||
fixed NAMESPACES sweep) — used by modules/sites.py per app."""
|
||||
if not RUNNING_ON_MAIN_SERVER:
|
||||
data = _ssh_kubectl(['get', 'deployments', '-n', ns])
|
||||
if not data:
|
||||
return []
|
||||
out = []
|
||||
for dep in data.get('items', []):
|
||||
desired = (dep.get('spec') or {}).get('replicas') or 0
|
||||
ready = (dep.get('status') or {}).get('readyReplicas') or 0
|
||||
containers = (((dep.get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or []
|
||||
image = containers[0]['image'] if containers else '—'
|
||||
out.append({
|
||||
'name': dep['metadata']['name'],
|
||||
'desired': desired,
|
||||
'ready': ready,
|
||||
'health': _deploy_health(desired, ready),
|
||||
'image': image,
|
||||
})
|
||||
return out
|
||||
|
||||
_init_client()
|
||||
if not K8S_AVAILABLE:
|
||||
return []
|
||||
@@ -235,6 +393,20 @@ def get_pod_metrics():
|
||||
"""Best-effort CPU/mem per pod via metrics-server. Returns {} if it's not
|
||||
installed or unreachable — callers must treat an empty dict as 'no data',
|
||||
never as an error."""
|
||||
if not RUNNING_ON_MAIN_SERVER:
|
||||
metrics = {}
|
||||
for ns in NAMESPACES:
|
||||
resp = _ssh_kubectl_raw(f'/apis/metrics.k8s.io/v1beta1/namespaces/{ns}/pods')
|
||||
if not resp:
|
||||
continue
|
||||
for item in resp.get('items', []):
|
||||
name = item.get('metadata', {}).get('name')
|
||||
containers = item.get('containers', [])
|
||||
cpu = sum(_parse_cpu(c.get('usage', {}).get('cpu')) for c in containers)
|
||||
mem = sum(_parse_mem_mib(c.get('usage', {}).get('memory')) for c in containers)
|
||||
metrics[f'{ns}/{name}'] = {'cpu_millicores': round(cpu), 'mem_mib': round(mem)}
|
||||
return metrics
|
||||
|
||||
_init_client()
|
||||
metrics = {}
|
||||
if not K8S_AVAILABLE:
|
||||
@@ -283,8 +455,13 @@ def get_cluster_overview():
|
||||
'deployments_healthy': sum(1 for d in deployments if d['health'] == 'healthy'),
|
||||
}
|
||||
|
||||
# K8S_AVAILABLE only reflects the in-cluster client (main server); on
|
||||
# the standby, "available" instead means the SSH+kubectl fetch above
|
||||
# actually returned something.
|
||||
available = K8S_AVAILABLE if RUNNING_ON_MAIN_SERVER else bool(pods or deployments)
|
||||
|
||||
return {
|
||||
'available': K8S_AVAILABLE,
|
||||
'available': available,
|
||||
'namespaces': by_namespace,
|
||||
'summary': summary,
|
||||
'metrics_available': bool(metrics),
|
||||
|
||||
Reference in New Issue
Block a user