diff --git a/platform/modules/backups.py b/platform/modules/backups.py index 9252f0d..a1517f2 100644 --- a/platform/modules/backups.py +++ b/platform/modules/backups.py @@ -895,8 +895,65 @@ def get_all_stats(): # SYSTEM INFO (FIXED WITH PSUTIL) # ──────────────────────────────────────────────────────────────── +def _get_main_server_system_info_remote(): + """get_system_info()'s equivalent for the standby: the main server's bare + host has no psutil (system python3, not this app's container image), so + this uses plain coreutils/procps instead — reusing _ssh_main means this + genuinely queries the real main server over the tunnel, not the VM + itself (the previous unconditional psutil.* calls silently reported the + VM's own stats mislabeled as "system info" whenever running as standby).""" + fallback = { + 'cpu_pct': '0', 'memory': 'N/A', 'mem_pct': '0', 'disk': 'N/A', + 'disk_pct': '0', 'load': 'N/A', 'uptime': 'N/A', 'docker_v': 'N/A', + 'hostname': 'unreachable', + } + remote_cmd = ( + "hostname; " + "vmstat 1 2 | tail -1 | awk '{print 100-$15}'; " + "free -m | awk 'NR==2{printf \"%d %d %d\\n\", $3, $2, $3*100/$2}'; " + "df -BM --output=used,size,pcent / | tail -1 | tr -d 'M%'; " + "awk '{printf \"%.2f %.2f %.2f\\n\", $1, $2, $3}' /proc/loadavg; " + "awk '{print $1}' /proc/uptime; " + "docker --version 2>/dev/null | awk '{print $3}' | tr -d ','" + ) + out, err = _ssh_main(remote_cmd, timeout=15) + lines = out.split('\n') if out else [] + if len(lines) < 7: + return fallback + try: + hostname = lines[0].strip() + cpu_pct = lines[1].strip() + mem_used_mb, mem_total_mb, mem_pct = lines[2].split() + disk_used_mb, disk_total_mb, disk_pct = lines[3].split() + load = lines[4].strip() + uptime_s = float(lines[5].strip()) + docker_v = lines[6].strip() or 'N/A' + + days = int(uptime_s // 86400) + hours = int((uptime_s % 86400) // 3600) + minutes = int((uptime_s % 3600) // 60) + uptime = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m" + + return { + 'cpu_pct': cpu_pct, + 'memory': f"{int(mem_used_mb) / 1024:.1f}G/{int(mem_total_mb) / 1024:.1f}G", + 'mem_pct': mem_pct, + 'disk': f"{int(disk_used_mb) / 1024:.0f}G/{int(disk_total_mb) / 1024:.0f}G", + 'disk_pct': disk_pct, + 'load': load, + 'uptime': uptime, + 'docker_v': docker_v, + 'hostname': hostname, + } + except Exception as e: + print(f"[backups] remote system info parse error: {e} (raw: {out!r}, err: {err!r})") + return fallback + + def get_system_info(): """Get system information using psutil for reliable metrics inside container""" + if not RUNNING_ON_MAIN_SERVER: + return _get_main_server_system_info_remote() info = { 'cpu_pct': '0', diff --git a/platform/modules/kubernetes.py b/platform/modules/kubernetes.py index 819d2d8..fba6533 100644 --- a/platform/modules/kubernetes.py +++ b/platform/modules/kubernetes.py @@ -4,6 +4,18 @@ # deployments.apps — no secrets, no write, no exec/log), bound per-namespace # across the 8 app namespaces below. There is no cluster-wide RoleBinding, # so every list call below is scoped to one namespace at a time. +# +# On the standby (RUNNING_ON_MAIN_SERVER False), there is no in-cluster +# ServiceAccount to load at all — load_incluster_config() always fails +# there, by design, even if the standby itself later runs as a pod on a +# *local* k3s on that VM (that cluster isn't this one; in-cluster config +# would only ever see the wrong, essentially-empty cluster). Every function +# below instead SSHes to the main server (over main-to-vm-tunnel.service) +# and runs `kubectl` there directly, where real cluster-admin access +# already exists — same real-vs-display separation as MAIN_SERVER_SSH_HOST +# in config.py. +import json +import subprocess from datetime import datetime, timezone try: @@ -13,6 +25,11 @@ try: except ImportError: _K8S_IMPORT_OK = False +from config import ( + RUNNING_ON_MAIN_SERVER, MAIN_SERVER_USER, MAIN_SERVER_KEY, + MAIN_SERVER_SSH_HOST, MAIN_SERVER_SSH_PORT, +) + NAMESPACES = [ 'n8n', 'odoo', 'mautic', 'erpnext', 'nextcloud', 'jenkins-agents', 'jenkins', 'management-platform', @@ -25,6 +42,78 @@ _networking_v1 = None K8S_AVAILABLE = False +def _ssh_kubectl(args): + """Run `kubectl -o json` on the main server over the reverse + tunnel. Returns the parsed JSON (dict) or None on any failure — + callers must treat None/empty the same as the in-cluster client + returning nothing, never as a hard error.""" + remote_cmd = "kubectl " + " ".join(args) + " -o json" + ssh_cmd = ( + f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} " + f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes " + f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} '{remote_cmd}'" + ) + try: + r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20) + if r.returncode != 0: + return None + return json.loads(r.stdout) + except Exception: + return None + + +def _ssh_kubectl_raw(path): + """Same as _ssh_kubectl but for `kubectl get --raw ` (used for + the metrics-server API, which isn't a normal `get ` call).""" + ssh_cmd = ( + f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_SSH_PORT} " + f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes " + f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST} 'kubectl get --raw {path}'" + ) + try: + r = subprocess.run(ssh_cmd, shell=True, capture_output=True, text=True, timeout=20) + if r.returncode != 0: + return None + return json.loads(r.stdout) + except Exception: + return None + + +def _parse_k8s_ts(s): + if not s: + return None + try: + return datetime.strptime(s, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc) + except Exception: + return None + + +def _pod_health_raw(pod): + statuses = (pod.get('status') or {}).get('containerStatuses') or [] + for cs in statuses: + waiting = (cs.get('state') or {}).get('waiting') + if waiting and waiting.get('reason') in _CRASH_REASONS: + return 'failed' + phase = (pod.get('status') or {}).get('phase') or 'Unknown' + if phase == 'Failed': + return 'failed' + if phase == 'Running': + all_ready = all(cs.get('ready') for cs in statuses) if statuses else True + return 'healthy' if all_ready else 'degraded' + return 'degraded' + + +def _ready_count_raw(pod): + statuses = (pod.get('status') or {}).get('containerStatuses') or [] + ready = sum(1 for cs in statuses if cs.get('ready')) + return f'{ready}/{len(statuses)}' + + +def _restart_count_raw(pod): + statuses = (pod.get('status') or {}).get('containerStatuses') or [] + return sum(cs.get('restartCount', 0) for cs in statuses) + + def _init_client(): global _core_v1, _apps_v1, _custom_objects, _networking_v1, K8S_AVAILABLE if _core_v1 is not None or not _K8S_IMPORT_OK: @@ -135,6 +224,24 @@ def _parse_mem_mib(v): # ──────────────────────────────────────────────────────────────── def list_pods(): + if not RUNNING_ON_MAIN_SERVER: + pods = [] + for ns in NAMESPACES: + data = _ssh_kubectl(['get', 'pods', '-n', ns]) + if not data: + continue + for pod in data.get('items', []): + pods.append({ + 'name': pod['metadata']['name'], + 'namespace': ns, + 'phase': (pod.get('status') or {}).get('phase') or 'Unknown', + 'health': _pod_health_raw(pod), + 'ready': _ready_count_raw(pod), + 'restarts': _restart_count_raw(pod), + 'age': _age_str(_parse_k8s_ts(pod['metadata'].get('creationTimestamp'))), + }) + return pods + _init_client() pods = [] if not K8S_AVAILABLE: @@ -159,6 +266,25 @@ def list_pods(): def list_deployments(): + if not RUNNING_ON_MAIN_SERVER: + deployments = [] + for ns in NAMESPACES: + data = _ssh_kubectl(['get', 'deployments', '-n', ns]) + if not data: + continue + for dep in data.get('items', []): + desired = (dep.get('spec') or {}).get('replicas') or 0 + ready = (dep.get('status') or {}).get('readyReplicas') or 0 + deployments.append({ + 'name': dep['metadata']['name'], + 'namespace': ns, + 'ready': ready, + 'desired': desired, + 'health': _deploy_health(desired, ready), + 'age': _age_str(_parse_k8s_ts(dep['metadata'].get('creationTimestamp'))), + }) + return deployments + _init_client() deployments = [] if not K8S_AVAILABLE: @@ -187,6 +313,19 @@ def get_ingress_info(name, namespace): """Real domain/TLS for one Ingress, or None if unavailable/not found. Used by modules/sites.py instead of the old hardcoded Docker-era domain fields — reflects actual live k8s Ingress state.""" + if not RUNNING_ON_MAIN_SERVER: + data = _ssh_kubectl(['get', 'ingress', name, '-n', namespace]) + if not data: + return None + host = None + rules = (data.get('spec') or {}).get('rules') or [] + for rule in rules: + if rule.get('host'): + host = rule['host'] + break + tls = bool((data.get('spec') or {}).get('tls')) + return {'host': host, 'tls': tls} + _init_client() if not K8S_AVAILABLE: return None @@ -207,6 +346,25 @@ def get_ingress_info(name, namespace): def list_deployments_for_namespace(ns): """Targeted single-namespace deployment list (vs. list_deployments()'s fixed NAMESPACES sweep) — used by modules/sites.py per app.""" + if not RUNNING_ON_MAIN_SERVER: + data = _ssh_kubectl(['get', 'deployments', '-n', ns]) + if not data: + return [] + out = [] + for dep in data.get('items', []): + desired = (dep.get('spec') or {}).get('replicas') or 0 + ready = (dep.get('status') or {}).get('readyReplicas') or 0 + containers = (((dep.get('spec') or {}).get('template') or {}).get('spec') or {}).get('containers') or [] + image = containers[0]['image'] if containers else '—' + out.append({ + 'name': dep['metadata']['name'], + 'desired': desired, + 'ready': ready, + 'health': _deploy_health(desired, ready), + 'image': image, + }) + return out + _init_client() if not K8S_AVAILABLE: return [] @@ -235,6 +393,20 @@ def get_pod_metrics(): """Best-effort CPU/mem per pod via metrics-server. Returns {} if it's not installed or unreachable — callers must treat an empty dict as 'no data', never as an error.""" + if not RUNNING_ON_MAIN_SERVER: + metrics = {} + for ns in NAMESPACES: + resp = _ssh_kubectl_raw(f'/apis/metrics.k8s.io/v1beta1/namespaces/{ns}/pods') + if not resp: + continue + for item in resp.get('items', []): + name = item.get('metadata', {}).get('name') + containers = item.get('containers', []) + cpu = sum(_parse_cpu(c.get('usage', {}).get('cpu')) for c in containers) + mem = sum(_parse_mem_mib(c.get('usage', {}).get('memory')) for c in containers) + metrics[f'{ns}/{name}'] = {'cpu_millicores': round(cpu), 'mem_mib': round(mem)} + return metrics + _init_client() metrics = {} if not K8S_AVAILABLE: @@ -283,8 +455,13 @@ def get_cluster_overview(): 'deployments_healthy': sum(1 for d in deployments if d['health'] == 'healthy'), } + # K8S_AVAILABLE only reflects the in-cluster client (main server); on + # the standby, "available" instead means the SSH+kubectl fetch above + # actually returned something. + available = K8S_AVAILABLE if RUNNING_ON_MAIN_SERVER else bool(pods or deployments) + return { - 'available': K8S_AVAILABLE, + 'available': available, 'namespaces': by_namespace, 'summary': summary, 'metrics_available': bool(metrics),