Sync from main server - 2026-04-16 13:42:46
This commit is contained in:
root
@@ -1,8 +1,13 @@
|
||||
# modules/backups.py
|
||||
import os
|
||||
import glob
|
||||
import subprocess
|
||||
import json
|
||||
from config import RUNNING_ON_MAIN_SERVER, VM_HOST, VM_PORT, VM_KEY, VM_USER
|
||||
from config import (
|
||||
RUNNING_ON_MAIN_SERVER,
|
||||
MAIN_SERVER_IP, MAIN_SERVER_USER, MAIN_SERVER_KEY, MAIN_SERVER_PORT,
|
||||
VM_HOST, VM_PORT, VM_KEY, VM_USER,
|
||||
)
|
||||
|
||||
|
||||
def _run(cmd, timeout=20):
|
||||
@@ -13,12 +18,35 @@ def _run(cmd, timeout=20):
|
||||
return '', str(e)
|
||||
|
||||
|
||||
def _ssh_main(remote_cmd, timeout=20):
|
||||
"""
|
||||
Run a command ON THE MAIN SERVER.
|
||||
- If we're already on the main server: run it locally.
|
||||
- If we're on the VM: SSH to main server first.
|
||||
"""
|
||||
if RUNNING_ON_MAIN_SERVER:
|
||||
return _run(remote_cmd, timeout=timeout)
|
||||
else:
|
||||
ssh = (
|
||||
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_PORT} "
|
||||
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
||||
f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}"
|
||||
)
|
||||
return _run(f"{ssh} '{remote_cmd}'", timeout=timeout)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# BACKUPS
|
||||
# BACKUPS (local = on main server; vm = on the VM)
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_local_backups():
|
||||
stdout, _ = _run("ls -t /root/backups/myapps-backup-*.tar.gz 2>/dev/null | head -20")
|
||||
"""
|
||||
Backups stored on the MAIN SERVER at /root/backups/.
|
||||
Always fetched from main server regardless of where platform runs.
|
||||
"""
|
||||
stdout, _ = _ssh_main(
|
||||
"ls -t /root/backups/myapps-backup-*.tar.gz 2>/dev/null | head -20"
|
||||
)
|
||||
files = []
|
||||
if stdout:
|
||||
for line in stdout.split('\n'):
|
||||
@@ -29,7 +57,13 @@ def get_local_backups():
|
||||
|
||||
|
||||
def get_vm_backups():
|
||||
"""
|
||||
Backups stored on the VM at /backups/main-server/.
|
||||
- On main server → SSH through tunnel (localhost:2223)
|
||||
- On VM → read local directory directly
|
||||
"""
|
||||
vm_backups = []
|
||||
|
||||
if RUNNING_ON_MAIN_SERVER:
|
||||
try:
|
||||
cmd = (
|
||||
@@ -38,7 +72,7 @@ def get_vm_backups():
|
||||
f"{VM_USER}@{VM_HOST} "
|
||||
f"'ls -t /backups/main-server/myapps-backup-*.tar.gz 2>/dev/null | head -20'"
|
||||
)
|
||||
stdout, _ = _run(cmd, timeout=20)
|
||||
stdout, _ = _run(cmd, timeout=25)
|
||||
if stdout:
|
||||
for line in stdout.split('\n'):
|
||||
line = line.strip()
|
||||
@@ -47,80 +81,121 @@ def get_vm_backups():
|
||||
except Exception as e:
|
||||
print(f"[backups] VM backup fetch error: {e}")
|
||||
else:
|
||||
# We ARE on the VM — read directly
|
||||
backup_dir = '/backups/main-server'
|
||||
if os.path.exists(backup_dir):
|
||||
files = glob.glob(f'{backup_dir}/myapps-backup-*.tar.gz')
|
||||
files.sort(key=os.path.getmtime, reverse=True)
|
||||
vm_backups = [os.path.basename(f) for f in files[:20]]
|
||||
|
||||
return vm_backups
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# ROOT CONTAINERS
|
||||
# CONTAINERS (always from main server)
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def _parse_containers(raw, owner='root'):
|
||||
containers = []
|
||||
if raw:
|
||||
for line in raw.split('\n'):
|
||||
if '|' not in line:
|
||||
continue
|
||||
parts = line.split('|')
|
||||
containers.append({
|
||||
'name': parts[0].strip(),
|
||||
'status': parts[1].strip() if len(parts) > 1 else '',
|
||||
'image': parts[2].strip() if len(parts) > 2 else '',
|
||||
'ports': parts[3].strip() if len(parts) > 3 else '',
|
||||
'owner': owner,
|
||||
})
|
||||
return containers
|
||||
|
||||
|
||||
def get_containers():
|
||||
"""Root app containers only (filtered)."""
|
||||
stdout, _ = _run(
|
||||
"""App containers only (frappe/nextcloud/mautic/n8n/odoo) — always from main server."""
|
||||
stdout, _ = _ssh_main(
|
||||
"docker ps -a --format '{{.Names}}|{{.Status}}|{{.Image}}|{{.Ports}}' 2>/dev/null | "
|
||||
"grep -E 'frappe|nextcloud|mautic|n8n|odoo'"
|
||||
)
|
||||
containers = []
|
||||
if stdout:
|
||||
for line in stdout.split('\n'):
|
||||
if '|' not in line:
|
||||
continue
|
||||
parts = line.split('|')
|
||||
containers.append({
|
||||
'name': parts[0].strip(),
|
||||
'status': parts[1].strip(),
|
||||
'image': parts[2].strip(),
|
||||
'ports': parts[3].strip() if len(parts) > 3 else '',
|
||||
'owner': 'root',
|
||||
})
|
||||
return containers
|
||||
return _parse_containers(stdout)
|
||||
|
||||
|
||||
def get_all_root_containers():
|
||||
"""ALL root docker containers (unfiltered)."""
|
||||
stdout, _ = _run(
|
||||
"""ALL root docker containers (unfiltered) — always from main server."""
|
||||
stdout, _ = _ssh_main(
|
||||
"docker ps -a --format '{{.Names}}|{{.Status}}|{{.Image}}|{{.Ports}}' 2>/dev/null"
|
||||
)
|
||||
return _parse_containers(stdout)
|
||||
|
||||
|
||||
def get_rootless_user_containers_remote():
|
||||
"""
|
||||
Get containers from all rootless-docker users ON THE MAIN SERVER.
|
||||
Root reads /run/user/*/docker.sock via SSH.
|
||||
"""
|
||||
# List all non-system UIDs that have a docker socket on main server
|
||||
cmd = "ls /run/user/*/docker.sock 2>/dev/null"
|
||||
stdout, _ = _ssh_main(cmd)
|
||||
containers = []
|
||||
if stdout:
|
||||
for line in stdout.split('\n'):
|
||||
if '|' not in line:
|
||||
continue
|
||||
parts = line.split('|')
|
||||
containers.append({
|
||||
'name': parts[0].strip(),
|
||||
'status': parts[1].strip(),
|
||||
'image': parts[2].strip(),
|
||||
'ports': parts[3].strip() if len(parts) > 3 else '',
|
||||
'owner': 'root',
|
||||
})
|
||||
if not stdout:
|
||||
return containers
|
||||
|
||||
for sock_path in stdout.split('\n'):
|
||||
sock_path = sock_path.strip()
|
||||
if not sock_path:
|
||||
continue
|
||||
# Determine username from uid
|
||||
try:
|
||||
uid = sock_path.split('/run/user/')[1].split('/')[0]
|
||||
except (IndexError, ValueError):
|
||||
continue
|
||||
|
||||
name_out, _ = _ssh_main(f"getent passwd {uid} | cut -d: -f1")
|
||||
username = name_out.strip() or f"uid{uid}"
|
||||
|
||||
ctr_out, _ = _ssh_main(
|
||||
f"DOCKER_HOST=unix://{sock_path} "
|
||||
f"docker ps -a --format '{{{{.Names}}}}|{{{{.Status}}}}|{{{{.Image}}}}|{{{{.Ports}}}}' 2>/dev/null"
|
||||
)
|
||||
containers.extend(_parse_containers(ctr_out, owner=username))
|
||||
|
||||
return containers
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# CONTAINER STATS
|
||||
# CONTAINER ACTIONS (start / stop / restart) — on main server
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_container_stats(docker_socket=None):
|
||||
"""One-shot stats snapshot. Returns dict keyed by container name."""
|
||||
if docker_socket:
|
||||
cmd = (
|
||||
f"DOCKER_HOST=unix://{docker_socket} "
|
||||
f"docker stats --no-stream --format "
|
||||
f"'{{{{.Name}}}}|{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.NetIO}}}}|{{{{.BlockIO}}}}' 2>/dev/null"
|
||||
)
|
||||
else:
|
||||
cmd = (
|
||||
"docker stats --no-stream --format "
|
||||
"'{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}|{{.NetIO}}|{{.BlockIO}}' 2>/dev/null"
|
||||
)
|
||||
def container_action(container_name, action):
|
||||
"""
|
||||
action: 'start' | 'stop' | 'restart'
|
||||
Returns (success: bool, output: str)
|
||||
"""
|
||||
if action not in ('start', 'stop', 'restart'):
|
||||
return False, "Invalid action"
|
||||
|
||||
stdout, _ = _run(cmd, timeout=30)
|
||||
safe_name = container_name.replace('"', '').replace(';', '').replace('|', '')
|
||||
stdout, stderr = _ssh_main(
|
||||
f"docker {action} {safe_name} 2>&1",
|
||||
timeout=30
|
||||
)
|
||||
output = (stdout + stderr).strip()
|
||||
success = safe_name in output or 'started' in output.lower() or stderr == ''
|
||||
return True, output
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# STATS — from main server
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_container_stats_remote():
|
||||
"""One-shot stats for all root containers on main server."""
|
||||
stdout, _ = _ssh_main(
|
||||
"docker stats --no-stream --format "
|
||||
"'{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}|{{.NetIO}}|{{.BlockIO}}' 2>/dev/null",
|
||||
timeout=35
|
||||
)
|
||||
stats = {}
|
||||
if stdout:
|
||||
for line in stdout.split('\n'):
|
||||
@@ -141,40 +216,63 @@ def get_container_stats(docker_socket=None):
|
||||
|
||||
|
||||
def get_all_stats():
|
||||
"""Stats for root + all rootless-user containers combined."""
|
||||
all_stats = get_container_stats()
|
||||
try:
|
||||
import pwd
|
||||
for pw in pwd.getpwall():
|
||||
if pw.pw_uid < 1000 or pw.pw_name == 'nobody':
|
||||
"""Stats for root containers on main server + rootless users on main server."""
|
||||
all_stats = get_container_stats_remote()
|
||||
|
||||
# Also get stats from rootless user sockets on main server
|
||||
socks_out, _ = _ssh_main("ls /run/user/*/docker.sock 2>/dev/null")
|
||||
if socks_out:
|
||||
for sock in socks_out.split('\n'):
|
||||
sock = sock.strip()
|
||||
if not sock:
|
||||
continue
|
||||
sock = f"/run/user/{pw.pw_uid}/docker.sock"
|
||||
if os.path.exists(sock):
|
||||
user_stats = get_container_stats(docker_socket=sock)
|
||||
all_stats.update(user_stats)
|
||||
except Exception as e:
|
||||
print(f"[stats] Error: {e}")
|
||||
stdout, _ = _ssh_main(
|
||||
f"DOCKER_HOST=unix://{sock} "
|
||||
f"docker stats --no-stream --format "
|
||||
f"'{{{{.Name}}}}|{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.NetIO}}}}|{{{{.BlockIO}}}}' 2>/dev/null",
|
||||
timeout=35
|
||||
)
|
||||
if stdout:
|
||||
for line in stdout.split('\n'):
|
||||
if '|' not in line:
|
||||
continue
|
||||
parts = line.split('|')
|
||||
if len(parts) < 6:
|
||||
continue
|
||||
all_stats[parts[0].strip()] = {
|
||||
'cpu': parts[1].strip(),
|
||||
'mem': parts[2].strip(),
|
||||
'mem_pct': parts[3].strip(),
|
||||
'net': parts[4].strip(),
|
||||
'block': parts[5].strip(),
|
||||
}
|
||||
return all_stats
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# SYSTEM INFO — from main server
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_system_info():
|
||||
"""Host-level system stats."""
|
||||
cpu_out, _ = _run("top -bn1 | grep 'Cpu(s)' | awk '{print $2+$4}'")
|
||||
mem_out, _ = _run("free -m | awk 'NR==2{printf \"%s/%sMB\", $3, $2}'")
|
||||
mem_pct, _ = _run("free | awk 'NR==2{printf \"%.0f\", $3/$2*100}'")
|
||||
disk_out, _ = _run("df -h / | awk 'NR==2{printf \"%s/%s\", $3, $2}'")
|
||||
disk_pct, _ = _run("df / | awk 'NR==2{print $5}' | tr -d '%'")
|
||||
load_out, _ = _run("cat /proc/loadavg | awk '{print $1, $2, $3}'")
|
||||
uptime_out, _ = _run("uptime -p")
|
||||
docker_v, _ = _run("docker --version | cut -d' ' -f3 | tr -d ','")
|
||||
"""Host-level system stats — always fetched from main server."""
|
||||
cpu_out, _ = _ssh_main("top -bn1 | grep 'Cpu(s)' | awk '{print $2+$4}'")
|
||||
mem_out, _ = _ssh_main("free -m | awk 'NR==2{printf \"%s/%sMB\", $3, $2}'")
|
||||
mem_pct, _ = _ssh_main("free | awk 'NR==2{printf \"%.0f\", $3/$2*100}'")
|
||||
disk_out, _ = _ssh_main("df -h / | awk 'NR==2{printf \"%s/%s\", $3, $2}'")
|
||||
disk_pct, _ = _ssh_main("df / | awk 'NR==2{print $5}' | tr -d '%'")
|
||||
load_out, _ = _ssh_main("cat /proc/loadavg | awk '{print $1, $2, $3}'")
|
||||
uptime, _ = _ssh_main("uptime -p")
|
||||
docker_v, _ = _ssh_main("docker --version | cut -d' ' -f3 | tr -d ','")
|
||||
hostname, _ = _run("hostname -f 2>/dev/null || hostname") # THIS host, not main server
|
||||
|
||||
return {
|
||||
'cpu_pct': cpu_out or '0',
|
||||
'memory': mem_out or 'N/A',
|
||||
'mem_pct': mem_pct or '0',
|
||||
'cpu_pct': cpu_out or '0',
|
||||
'memory': mem_out or 'N/A',
|
||||
'mem_pct': mem_pct or '0',
|
||||
'disk': disk_out or 'N/A',
|
||||
'disk_pct': disk_pct or '0',
|
||||
'load': load_out or 'N/A',
|
||||
'uptime': uptime_out or 'N/A',
|
||||
'uptime': uptime or 'N/A',
|
||||
'docker_v': docker_v or 'N/A',
|
||||
'hostname': hostname or 'this server',
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user