279 lines
11 KiB
Python
279 lines
11 KiB
Python
# modules/backups.py
|
|
import os
|
|
import glob
|
|
import subprocess
|
|
import json
|
|
from config import (
|
|
RUNNING_ON_MAIN_SERVER,
|
|
MAIN_SERVER_IP, MAIN_SERVER_USER, MAIN_SERVER_KEY, MAIN_SERVER_PORT,
|
|
VM_HOST, VM_PORT, VM_KEY, VM_USER,
|
|
)
|
|
|
|
|
|
def _run(cmd, timeout=20):
|
|
try:
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
|
return r.stdout.strip(), r.stderr.strip()
|
|
except Exception as e:
|
|
return '', str(e)
|
|
|
|
|
|
def _ssh_main(remote_cmd, timeout=20):
|
|
"""
|
|
Run a command ON THE MAIN SERVER.
|
|
- If we're already on the main server: run it locally.
|
|
- If we're on the VM: SSH to main server first.
|
|
"""
|
|
if RUNNING_ON_MAIN_SERVER:
|
|
return _run(remote_cmd, timeout=timeout)
|
|
else:
|
|
ssh = (
|
|
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}"
|
|
)
|
|
return _run(f"{ssh} '{remote_cmd}'", timeout=timeout)
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUPS (local = on main server; vm = on the VM)
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_local_backups():
|
|
"""
|
|
Backups stored on the MAIN SERVER at /root/backups/.
|
|
Always fetched from main server regardless of where platform runs.
|
|
"""
|
|
stdout, _ = _ssh_main(
|
|
"ls -t /root/backups/myapps-backup-*.tar.gz 2>/dev/null | head -20"
|
|
)
|
|
files = []
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
line = line.strip()
|
|
if line:
|
|
files.append(os.path.basename(line))
|
|
return files
|
|
|
|
|
|
def get_vm_backups():
|
|
"""
|
|
Backups stored on the VM at /backups/main-server/.
|
|
- On main server → SSH through tunnel (localhost:2223)
|
|
- On VM → read local directory directly
|
|
"""
|
|
vm_backups = []
|
|
|
|
if RUNNING_ON_MAIN_SERVER:
|
|
try:
|
|
cmd = (
|
|
f"ssh -i {VM_KEY} -p {VM_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"{VM_USER}@{VM_HOST} "
|
|
f"'ls -t /backups/main-server/myapps-backup-*.tar.gz 2>/dev/null | head -20'"
|
|
)
|
|
stdout, _ = _run(cmd, timeout=25)
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
line = line.strip()
|
|
if line and '.tar.gz' in line:
|
|
vm_backups.append(os.path.basename(line))
|
|
except Exception as e:
|
|
print(f"[backups] VM backup fetch error: {e}")
|
|
else:
|
|
# We ARE on the VM — read directly
|
|
backup_dir = '/backups/main-server'
|
|
if os.path.exists(backup_dir):
|
|
files = glob.glob(f'{backup_dir}/myapps-backup-*.tar.gz')
|
|
files.sort(key=os.path.getmtime, reverse=True)
|
|
vm_backups = [os.path.basename(f) for f in files[:20]]
|
|
|
|
return vm_backups
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# CONTAINERS (always from main server)
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def _parse_containers(raw, owner='root'):
|
|
containers = []
|
|
if raw:
|
|
for line in raw.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
containers.append({
|
|
'name': parts[0].strip(),
|
|
'status': parts[1].strip() if len(parts) > 1 else '',
|
|
'image': parts[2].strip() if len(parts) > 2 else '',
|
|
'ports': parts[3].strip() if len(parts) > 3 else '',
|
|
'owner': owner,
|
|
})
|
|
return containers
|
|
|
|
|
|
def get_containers():
|
|
"""App containers only (frappe/nextcloud/mautic/n8n/odoo) — always from main server."""
|
|
stdout, _ = _ssh_main(
|
|
"docker ps -a --format '{{.Names}}|{{.Status}}|{{.Image}}|{{.Ports}}' 2>/dev/null | "
|
|
"grep -E 'frappe|nextcloud|mautic|n8n|odoo'"
|
|
)
|
|
return _parse_containers(stdout)
|
|
|
|
|
|
def get_all_root_containers():
|
|
"""ALL root docker containers (unfiltered) — always from main server."""
|
|
stdout, _ = _ssh_main(
|
|
"docker ps -a --format '{{.Names}}|{{.Status}}|{{.Image}}|{{.Ports}}' 2>/dev/null"
|
|
)
|
|
return _parse_containers(stdout)
|
|
|
|
|
|
def get_rootless_user_containers_remote():
|
|
"""
|
|
Get containers from all rootless-docker users ON THE MAIN SERVER.
|
|
Root reads /run/user/*/docker.sock via SSH.
|
|
"""
|
|
# List all non-system UIDs that have a docker socket on main server
|
|
cmd = "ls /run/user/*/docker.sock 2>/dev/null"
|
|
stdout, _ = _ssh_main(cmd)
|
|
containers = []
|
|
if not stdout:
|
|
return containers
|
|
|
|
for sock_path in stdout.split('\n'):
|
|
sock_path = sock_path.strip()
|
|
if not sock_path:
|
|
continue
|
|
# Determine username from uid
|
|
try:
|
|
uid = sock_path.split('/run/user/')[1].split('/')[0]
|
|
except (IndexError, ValueError):
|
|
continue
|
|
|
|
name_out, _ = _ssh_main(f"getent passwd {uid} | cut -d: -f1")
|
|
username = name_out.strip() or f"uid{uid}"
|
|
|
|
ctr_out, _ = _ssh_main(
|
|
f"DOCKER_HOST=unix://{sock_path} "
|
|
f"docker ps -a --format '{{{{.Names}}}}|{{{{.Status}}}}|{{{{.Image}}}}|{{{{.Ports}}}}' 2>/dev/null"
|
|
)
|
|
containers.extend(_parse_containers(ctr_out, owner=username))
|
|
|
|
return containers
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# CONTAINER ACTIONS (start / stop / restart) — on main server
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def container_action(container_name, action):
|
|
"""
|
|
action: 'start' | 'stop' | 'restart'
|
|
Returns (success: bool, output: str)
|
|
"""
|
|
if action not in ('start', 'stop', 'restart'):
|
|
return False, "Invalid action"
|
|
|
|
safe_name = container_name.replace('"', '').replace(';', '').replace('|', '')
|
|
stdout, stderr = _ssh_main(
|
|
f"docker {action} {safe_name} 2>&1",
|
|
timeout=30
|
|
)
|
|
output = (stdout + stderr).strip()
|
|
success = safe_name in output or 'started' in output.lower() or stderr == ''
|
|
return True, output
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# STATS — from main server
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_container_stats_remote():
|
|
"""One-shot stats for all root containers on main server."""
|
|
stdout, _ = _ssh_main(
|
|
"docker stats --no-stream --format "
|
|
"'{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}|{{.NetIO}}|{{.BlockIO}}' 2>/dev/null",
|
|
timeout=35
|
|
)
|
|
stats = {}
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
if len(parts) < 6:
|
|
continue
|
|
name = parts[0].strip()
|
|
stats[name] = {
|
|
'cpu': parts[1].strip(),
|
|
'mem': parts[2].strip(),
|
|
'mem_pct': parts[3].strip(),
|
|
'net': parts[4].strip(),
|
|
'block': parts[5].strip(),
|
|
}
|
|
return stats
|
|
|
|
|
|
def get_all_stats():
|
|
"""Stats for root containers on main server + rootless users on main server."""
|
|
all_stats = get_container_stats_remote()
|
|
|
|
# Also get stats from rootless user sockets on main server
|
|
socks_out, _ = _ssh_main("ls /run/user/*/docker.sock 2>/dev/null")
|
|
if socks_out:
|
|
for sock in socks_out.split('\n'):
|
|
sock = sock.strip()
|
|
if not sock:
|
|
continue
|
|
stdout, _ = _ssh_main(
|
|
f"DOCKER_HOST=unix://{sock} "
|
|
f"docker stats --no-stream --format "
|
|
f"'{{{{.Name}}}}|{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.NetIO}}}}|{{{{.BlockIO}}}}' 2>/dev/null",
|
|
timeout=35
|
|
)
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
if len(parts) < 6:
|
|
continue
|
|
all_stats[parts[0].strip()] = {
|
|
'cpu': parts[1].strip(),
|
|
'mem': parts[2].strip(),
|
|
'mem_pct': parts[3].strip(),
|
|
'net': parts[4].strip(),
|
|
'block': parts[5].strip(),
|
|
}
|
|
return all_stats
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# SYSTEM INFO — from main server
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_system_info():
|
|
"""Host-level system stats — always fetched from main server."""
|
|
cpu_out, _ = _ssh_main("top -bn1 | grep 'Cpu(s)' | awk '{print $2+$4}'")
|
|
mem_out, _ = _ssh_main("free -m | awk 'NR==2{printf \"%s/%sMB\", $3, $2}'")
|
|
mem_pct, _ = _ssh_main("free | awk 'NR==2{printf \"%.0f\", $3/$2*100}'")
|
|
disk_out, _ = _ssh_main("df -h / | awk 'NR==2{printf \"%s/%s\", $3, $2}'")
|
|
disk_pct, _ = _ssh_main("df / | awk 'NR==2{print $5}' | tr -d '%'")
|
|
load_out, _ = _ssh_main("cat /proc/loadavg | awk '{print $1, $2, $3}'")
|
|
uptime, _ = _ssh_main("uptime -p")
|
|
docker_v, _ = _ssh_main("docker --version | cut -d' ' -f3 | tr -d ','")
|
|
hostname, _ = _run("hostname -f 2>/dev/null || hostname") # THIS host, not main server
|
|
|
|
return {
|
|
'cpu_pct': cpu_out or '0',
|
|
'memory': mem_out or 'N/A',
|
|
'mem_pct': mem_pct or '0',
|
|
'disk': disk_out or 'N/A',
|
|
'disk_pct': disk_pct or '0',
|
|
'load': load_out or 'N/A',
|
|
'uptime': uptime or 'N/A',
|
|
'docker_v': docker_v or 'N/A',
|
|
'hostname': hostname or 'this server',
|
|
}
|