11b9f6f0c7
Two real gaps found after testing the live standby platform: 1. modules/kubernetes.py's in-cluster client (load_incluster_config()) can only ever work inside the real k8s pod. On the standby it silently returned K8S_AVAILABLE=False with zero fallback — Cluster, Containers- via-sites, and any cluster/app health data was just empty on that page, not erroring, so it was easy to miss. This holds even if the standby later runs as a pod on a *local* k3s on that server (per plan for the monitoring migration) — in-cluster config there would only ever see that local, unrelated cluster, never the main one. Fixed by adding an SSH+kubectl remote path to list_pods(), list_ deployments(), get_ingress_info(), list_deployments_for_namespace(), and get_pod_metrics(): when not RUNNING_ON_MAIN_SERVER, each runs `kubectl <verb> -o json` on the main server over the tunnel (main-to-vm-tunnel.service) instead of using the python client, parses the raw (camelCase) JSON directly rather than trying to force it through the client-library's snake_case object model, and feeds the same _pod_health/_deploy_health/_age_str helpers either way. The in-cluster path for the real main-server pod is untouched. 2. get_system_info() called psutil.* unconditionally with no RUNNING_ON_MAIN_SERVER awareness at all — on the standby it was silently reporting the VM's OWN cpu/mem/disk/hostname mislabeled as "system info" (hostname field literally showed the VM's hostname). Added _get_main_server_system_info_remote(): SSHes to the main server and gathers the same stats via vmstat/free/df/procfs (the main server's bare host has no psutil — that's only in this app's own container image — so this avoids depending on it being present remotely). Verified end-to-end on the live standby: /api/system now reports the real main server's hostname/cpu/mem/disk/uptime; /api/cluster reports real counts (19 pods, 16/16 deployments healthy, 3 degraded, plus live per-pod metrics via the metrics-server raw API); /api/sites shows real per-app container/role status. All previously silently empty.
469 lines
17 KiB
Python
469 lines
17 KiB
Python
# modules/kubernetes.py — read-only cluster view via the in-cluster ServiceAccount
|
|
#
|
|
# Auth: management-platform-viewer-sa (get/list/watch on pods, services,
|
|
# deployments.apps — no secrets, no write, no exec/log), bound per-namespace
|
|
# across the 8 app namespaces below. There is no cluster-wide RoleBinding,
|
|
# so every list call below is scoped to one namespace at a time.
|
|
#
|
|
# On the standby (RUNNING_ON_MAIN_SERVER False), there is no in-cluster
|
|
# ServiceAccount to load at all — load_incluster_config() always fails
|
|
# there, by design, even if the standby itself later runs as a pod on a
|
|
# *local* k3s on that VM (that cluster isn't this one; in-cluster config
|
|
# would only ever see the wrong, essentially-empty cluster). Every function
|
|
# below instead SSHes to the main server (over main-to-vm-tunnel.service)
|
|
# and runs `kubectl` there directly, where real cluster-admin access
|
|
# already exists — same real-vs-display separation as MAIN_SERVER_SSH_HOST
|
|
# in config.py.
|
|
import json
|
|
import subprocess
|
|
from datetime import datetime, timezone
|
|
|
|
try:
|
|
from kubernetes import client, config as k8s_config
|
|
from kubernetes.client.rest import ApiException
|
|
_K8S_IMPORT_OK = True
|
|
except ImportError:
|
|
_K8S_IMPORT_OK = False
|
|
|
|
from config import (
|
|
RUNNING_ON_MAIN_SERVER, MAIN_SERVER_USER, MAIN_SERVER_KEY,
|
|
MAIN_SERVER_SSH_HOST, MAIN_SERVER_SSH_PORT,
|
|
)
|
|
|
|
NAMESPACES = [
|
|
'n8n', 'odoo', 'mautic', 'erpnext', 'nextcloud',
|
|
'jenkins-agents', 'jenkins', 'management-platform',
|
|
]
|
|
|
|
_core_v1 = None
|
|
_apps_v1 = None
|
|
_custom_objects = None
|
|
_networking_v1 = None
|
|
K8S_AVAILABLE = False
|
|
|
|
|
|
def _ssh_kubectl(args):
|
|
"""Run `kubectl <args> -o json` on the main server over the reverse
|
|
tunnel. Returns the parsed JSON (dict) or None on any failure —
|
|
callers must treat None/empty the same as the in-cluster client
|
|
returning nothing, never as a hard error."""
|
|
remote_cmd = "kubectl " + " ".join(args) + " -o json"
|
|
ssh_cmd = (
|
|
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes "
|
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} '{remote_cmd}'"
|
|
)
|
|
try:
|
|
r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20)
|
|
if r.returncode != 0:
|
|
return None
|
|
return json.loads(r.stdout)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _ssh_kubectl_raw(path):
|
|
"""Same as _ssh_kubectl but for `kubectl get --raw <path>` (used for
|
|
the metrics-server API, which isn't a normal `get <resource>` call)."""
|
|
ssh_cmd = (
|
|
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes "
|
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} 'kubectl get --raw {path}'"
|
|
)
|
|
try:
|
|
r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20)
|
|
if r.returncode != 0:
|
|
return None
|
|
return json.loads(r.stdout)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _parse_k8s_ts(s):
|
|
if not s:
|
|
return None
|
|
try:
|
|
return datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc)
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def _pod_health_raw(pod):
|
|
statuses = (pod.get('status') or {}).get('containerStatuses') or []
|
|
for cs in statuses:
|
|
waiting = (cs.get('state') or {}).get('waiting')
|
|
if waiting and waiting.get('reason') in _CRASH_REASONS:
|
|
return 'failed'
|
|
phase = (pod.get('status') or {}).get('phase') or 'Unknown'
|
|
if phase == 'Failed':
|
|
return 'failed'
|
|
if phase == 'Running':
|
|
all_ready = all(cs.get('ready') for cs in statuses) if statuses else True
|
|
return 'healthy' if all_ready else 'degraded'
|
|
return 'degraded'
|
|
|
|
|
|
def _ready_count_raw(pod):
|
|
statuses = (pod.get('status') or {}).get('containerStatuses') or []
|
|
ready = sum(1 for cs in statuses if cs.get('ready'))
|
|
return f'{ready}/{len(statuses)}'
|
|
|
|
|
|
def _restart_count_raw(pod):
|
|
statuses = (pod.get('status') or {}).get('containerStatuses') or []
|
|
return sum(cs.get('restartCount', 0) for cs in statuses)
|
|
|
|
|
|
def _init_client():
|
|
global _core_v1, _apps_v1, _custom_objects, _networking_v1, K8S_AVAILABLE
|
|
if _core_v1 is not None or not _K8S_IMPORT_OK:
|
|
return
|
|
try:
|
|
k8s_config.load_incluster_config()
|
|
_core_v1 = client.CoreV1Api()
|
|
_apps_v1 = client.AppsV1Api()
|
|
_custom_objects = client.CustomObjectsApi()
|
|
_networking_v1 = client.NetworkingV1Api()
|
|
K8S_AVAILABLE = True
|
|
except Exception as e:
|
|
print(f"[kubernetes] in-cluster config unavailable: {e}")
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# HELPERS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def _age_str(dt):
|
|
if not dt:
|
|
return '—'
|
|
seconds = int((datetime.now(timezone.utc) - dt).total_seconds())
|
|
if seconds < 60:
|
|
return f'{seconds}s'
|
|
minutes = seconds // 60
|
|
if minutes < 60:
|
|
return f'{minutes}m'
|
|
hours = minutes // 60
|
|
if hours < 24:
|
|
return f'{hours}h {minutes % 60}m'
|
|
days = hours // 24
|
|
return f'{days}d {hours % 24}h'
|
|
|
|
|
|
_CRASH_REASONS = ('CrashLoopBackOff', 'ImagePullBackOff', 'ErrImagePull', 'Error')
|
|
|
|
|
|
def _pod_health(pod):
|
|
statuses = pod.status.container_statuses or []
|
|
for cs in statuses:
|
|
waiting = cs.state.waiting if cs.state else None
|
|
if waiting and waiting.reason in _CRASH_REASONS:
|
|
return 'failed'
|
|
phase = pod.status.phase or 'Unknown'
|
|
if phase == 'Failed':
|
|
return 'failed'
|
|
if phase == 'Running':
|
|
all_ready = all(cs.ready for cs in statuses) if statuses else True
|
|
return 'healthy' if all_ready else 'degraded'
|
|
return 'degraded' # Pending, Unknown, etc.
|
|
|
|
|
|
def _ready_count(pod):
|
|
statuses = pod.status.container_statuses or []
|
|
ready = sum(1 for cs in statuses if cs.ready)
|
|
return f'{ready}/{len(statuses)}'
|
|
|
|
|
|
def _restart_count(pod):
|
|
statuses = pod.status.container_statuses or []
|
|
return sum(cs.restart_count for cs in statuses)
|
|
|
|
|
|
def _deploy_health(desired, ready):
|
|
if desired == 0:
|
|
return 'degraded'
|
|
if ready == desired:
|
|
return 'healthy'
|
|
if ready == 0:
|
|
return 'failed'
|
|
return 'degraded'
|
|
|
|
|
|
def _parse_cpu(v):
|
|
v = (v or '0').strip()
|
|
try:
|
|
if v.endswith('n'):
|
|
return int(v[:-1]) / 1_000_000
|
|
if v.endswith('u'):
|
|
return int(v[:-1]) / 1_000
|
|
if v.endswith('m'):
|
|
return int(v[:-1])
|
|
return float(v) * 1000
|
|
except ValueError:
|
|
return 0
|
|
|
|
|
|
_MEM_UNITS = {'Ki': 1 / 1024, 'Mi': 1, 'Gi': 1024, 'K': 1 / 1024, 'M': 1, 'G': 1024}
|
|
|
|
|
|
def _parse_mem_mib(v):
|
|
v = (v or '0').strip()
|
|
for suffix, factor in _MEM_UNITS.items():
|
|
if v.endswith(suffix):
|
|
try:
|
|
return float(v[:-len(suffix)]) * factor
|
|
except ValueError:
|
|
return 0
|
|
try:
|
|
return float(v) / (1024 * 1024)
|
|
except ValueError:
|
|
return 0
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# PODS / DEPLOYMENTS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def list_pods():
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
pods = []
|
|
for ns in NAMESPACES:
|
|
data = _ssh_kubectl(['get', 'pods', '-n', ns])
|
|
if not data:
|
|
continue
|
|
for pod in data.get('items', []):
|
|
pods.append({
|
|
'name': pod['metadata']['name'],
|
|
'namespace': ns,
|
|
'phase': (pod.get('status') or {}).get('phase') or 'Unknown',
|
|
'health': _pod_health_raw(pod),
|
|
'ready': _ready_count_raw(pod),
|
|
'restarts': _restart_count_raw(pod),
|
|
'age': _age_str(_parse_k8s_ts(pod['metadata'].get('creationTimestamp'))),
|
|
})
|
|
return pods
|
|
|
|
_init_client()
|
|
pods = []
|
|
if not K8S_AVAILABLE:
|
|
return pods
|
|
for ns in NAMESPACES:
|
|
try:
|
|
resp = _core_v1.list_namespaced_pod(ns)
|
|
except ApiException as e:
|
|
print(f"[kubernetes] list pods failed for {ns}: {e}")
|
|
continue
|
|
for pod in resp.items:
|
|
pods.append({
|
|
'name': pod.metadata.name,
|
|
'namespace': ns,
|
|
'phase': pod.status.phase or 'Unknown',
|
|
'health': _pod_health(pod),
|
|
'ready': _ready_count(pod),
|
|
'restarts': _restart_count(pod),
|
|
'age': _age_str(pod.metadata.creation_timestamp),
|
|
})
|
|
return pods
|
|
|
|
|
|
def list_deployments():
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
deployments = []
|
|
for ns in NAMESPACES:
|
|
data = _ssh_kubectl(['get', 'deployments', '-n', ns])
|
|
if not data:
|
|
continue
|
|
for dep in data.get('items', []):
|
|
desired = (dep.get('spec') or {}).get('replicas') or 0
|
|
ready = (dep.get('status') or {}).get('readyReplicas') or 0
|
|
deployments.append({
|
|
'name': dep['metadata']['name'],
|
|
'namespace': ns,
|
|
'ready': ready,
|
|
'desired': desired,
|
|
'health': _deploy_health(desired, ready),
|
|
'age': _age_str(_parse_k8s_ts(dep['metadata'].get('creationTimestamp'))),
|
|
})
|
|
return deployments
|
|
|
|
_init_client()
|
|
deployments = []
|
|
if not K8S_AVAILABLE:
|
|
return deployments
|
|
for ns in NAMESPACES:
|
|
try:
|
|
resp = _apps_v1.list_namespaced_deployment(ns)
|
|
except ApiException as e:
|
|
print(f"[kubernetes] list deployments failed for {ns}: {e}")
|
|
continue
|
|
for dep in resp.items:
|
|
desired = dep.spec.replicas or 0
|
|
ready = dep.status.ready_replicas or 0
|
|
deployments.append({
|
|
'name': dep.metadata.name,
|
|
'namespace': ns,
|
|
'ready': ready,
|
|
'desired': desired,
|
|
'health': _deploy_health(desired, ready),
|
|
'age': _age_str(dep.metadata.creation_timestamp),
|
|
})
|
|
return deployments
|
|
|
|
|
|
def get_ingress_info(name, namespace):
|
|
"""Real domain/TLS for one Ingress, or None if unavailable/not found.
|
|
Used by modules/sites.py instead of the old hardcoded Docker-era
|
|
domain fields — reflects actual live k8s Ingress state."""
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
data = _ssh_kubectl(['get', 'ingress', name, '-n', namespace])
|
|
if not data:
|
|
return None
|
|
host = None
|
|
rules = (data.get('spec') or {}).get('rules') or []
|
|
for rule in rules:
|
|
if rule.get('host'):
|
|
host = rule['host']
|
|
break
|
|
tls = bool((data.get('spec') or {}).get('tls'))
|
|
return {'host': host, 'tls': tls}
|
|
|
|
_init_client()
|
|
if not K8S_AVAILABLE:
|
|
return None
|
|
try:
|
|
ing = _networking_v1.read_namespaced_ingress(name, namespace)
|
|
except Exception:
|
|
return None
|
|
host = None
|
|
if ing.spec and ing.spec.rules:
|
|
for rule in ing.spec.rules:
|
|
if rule.host:
|
|
host = rule.host
|
|
break
|
|
tls = bool(ing.spec and ing.spec.tls)
|
|
return {'host': host, 'tls': tls}
|
|
|
|
|
|
def list_deployments_for_namespace(ns):
|
|
"""Targeted single-namespace deployment list (vs. list_deployments()'s
|
|
fixed NAMESPACES sweep) — used by modules/sites.py per app."""
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
data = _ssh_kubectl(['get', 'deployments', '-n', ns])
|
|
if not data:
|
|
return []
|
|
out = []
|
|
for dep in data.get('items', []):
|
|
desired = (dep.get('spec') or {}).get('replicas') or 0
|
|
ready = (dep.get('status') or {}).get('readyReplicas') or 0
|
|
containers = (((dep.get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or []
|
|
image = containers[0]['image'] if containers else '—'
|
|
out.append({
|
|
'name': dep['metadata']['name'],
|
|
'desired': desired,
|
|
'ready': ready,
|
|
'health': _deploy_health(desired, ready),
|
|
'image': image,
|
|
})
|
|
return out
|
|
|
|
_init_client()
|
|
if not K8S_AVAILABLE:
|
|
return []
|
|
try:
|
|
resp = _apps_v1.list_namespaced_deployment(ns)
|
|
except Exception as e:
|
|
print(f"[kubernetes] list deployments failed for {ns}: {e}")
|
|
return []
|
|
out = []
|
|
for dep in resp.items:
|
|
desired = dep.spec.replicas or 0
|
|
ready = dep.status.ready_replicas or 0
|
|
containers = dep.spec.template.spec.containers or []
|
|
image = containers[0].image if containers else '—'
|
|
out.append({
|
|
'name': dep.metadata.name,
|
|
'desired': desired,
|
|
'ready': ready,
|
|
'health': _deploy_health(desired, ready),
|
|
'image': image,
|
|
})
|
|
return out
|
|
|
|
|
|
def get_pod_metrics():
|
|
"""Best-effort CPU/mem per pod via metrics-server. Returns {} if it's not
|
|
installed or unreachable — callers must treat an empty dict as 'no data',
|
|
never as an error."""
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
metrics = {}
|
|
for ns in NAMESPACES:
|
|
resp = _ssh_kubectl_raw(f'/apis/metrics.k8s.io/v1beta1/namespaces/{ns}/pods')
|
|
if not resp:
|
|
continue
|
|
for item in resp.get('items', []):
|
|
name = item.get('metadata', {}).get('name')
|
|
containers = item.get('containers', [])
|
|
cpu = sum(_parse_cpu(c.get('usage', {}).get('cpu')) for c in containers)
|
|
mem = sum(_parse_mem_mib(c.get('usage', {}).get('memory')) for c in containers)
|
|
metrics[f'{ns}/{name}'] = {'cpu_millicores': round(cpu), 'mem_mib': round(mem)}
|
|
return metrics
|
|
|
|
_init_client()
|
|
metrics = {}
|
|
if not K8S_AVAILABLE:
|
|
return metrics
|
|
for ns in NAMESPACES:
|
|
try:
|
|
resp = _custom_objects.list_namespaced_custom_object(
|
|
'metrics.k8s.io', 'v1beta1', ns, 'pods'
|
|
)
|
|
except Exception:
|
|
continue
|
|
for item in resp.get('items', []):
|
|
name = item.get('metadata', {}).get('name')
|
|
containers = item.get('containers', [])
|
|
cpu = sum(_parse_cpu(c.get('usage', {}).get('cpu')) for c in containers)
|
|
mem = sum(_parse_mem_mib(c.get('usage', {}).get('memory')) for c in containers)
|
|
metrics[f'{ns}/{name}'] = {'cpu_millicores': round(cpu), 'mem_mib': round(mem)}
|
|
return metrics
|
|
|
|
|
|
def get_cluster_overview():
|
|
"""Single call powering the Cluster page: pods + deployments grouped by
|
|
namespace, plus summary counts. Mirrors the shape of get_all_stats() /
|
|
get_sites_list() in the docker-facing modules."""
|
|
pods = list_pods()
|
|
deployments = list_deployments()
|
|
metrics = get_pod_metrics()
|
|
|
|
for pod in pods:
|
|
key = f"{pod['namespace']}/{pod['name']}"
|
|
if key in metrics:
|
|
pod['metrics'] = metrics[key]
|
|
|
|
by_namespace = {ns: {'pods': [], 'deployments': []} for ns in NAMESPACES}
|
|
for pod in pods:
|
|
by_namespace[pod['namespace']]['pods'].append(pod)
|
|
for dep in deployments:
|
|
by_namespace[dep['namespace']]['deployments'].append(dep)
|
|
|
|
summary = {
|
|
'total_pods': len(pods),
|
|
'healthy': sum(1 for p in pods if p['health'] == 'healthy'),
|
|
'degraded': sum(1 for p in pods if p['health'] == 'degraded'),
|
|
'failed': sum(1 for p in pods if p['health'] == 'failed'),
|
|
'total_deployments': len(deployments),
|
|
'deployments_healthy': sum(1 for d in deployments if d['health'] == 'healthy'),
|
|
}
|
|
|
|
# K8S_AVAILABLE only reflects the in-cluster client (main server); on
|
|
# the standby, "available" instead means the SSH+kubectl fetch above
|
|
# actually returned something.
|
|
available = K8S_AVAILABLE if RUNNING_ON_MAIN_SERVER else bool(pods or deployments)
|
|
|
|
return {
|
|
'available': available,
|
|
'namespaces': by_namespace,
|
|
'summary': summary,
|
|
'metrics_available': bool(metrics),
|
|
}
|