817 lines
30 KiB
Python
817 lines
30 KiB
Python
# modules/backups.py
|
|
import os
|
|
import glob
|
|
import subprocess
|
|
import json
|
|
import hashlib
|
|
import tarfile
|
|
import re
|
|
import time
|
|
import socket
|
|
import psutil
|
|
from config import (
|
|
RUNNING_ON_MAIN_SERVER,
|
|
MAIN_SERVER_IP, MAIN_SERVER_USER, MAIN_SERVER_KEY, MAIN_SERVER_PORT,
|
|
VM_HOST, VM_PORT, VM_KEY, VM_USER,
|
|
)
|
|
|
|
|
|
def _run(cmd, timeout=30):
|
|
try:
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
|
return r.stdout.strip(), r.stderr.strip()
|
|
except Exception as e:
|
|
return '', str(e)
|
|
|
|
|
|
def _human_bytes(n):
|
|
n = int(n)
|
|
if n < 1024:
|
|
return f'{n} B'
|
|
if n < 1024 ** 2:
|
|
return f'{n / 1024:.1f} KB'
|
|
if n < 1024 ** 3:
|
|
return f'{n / (1024 ** 2):.1f} MB'
|
|
if n < 1024 ** 4:
|
|
return f'{n / (1024 ** 3):.2f} GB'
|
|
return f'{n / (1024 ** 4):.2f} TB'
|
|
|
|
|
|
def _ssh_main(remote_cmd, timeout=30):
|
|
if RUNNING_ON_MAIN_SERVER:
|
|
return _run(remote_cmd, timeout=timeout)
|
|
else:
|
|
escaped = remote_cmd.replace("'", "'\\''")
|
|
ssh = (
|
|
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"-o BatchMode=yes "
|
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}"
|
|
)
|
|
return _run(f"{ssh} '{escaped}'", timeout=timeout)
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUPS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_local_backups():
|
|
stdout, _ = _ssh_main(
|
|
"ls -t /root/backups/myapps-backup-*.tar.gz 2>/dev/null | head -20"
|
|
)
|
|
files = []
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
line = line.strip()
|
|
if line:
|
|
files.append(os.path.basename(line))
|
|
return files
|
|
|
|
|
|
def get_vm_backups():
|
|
vm_backups = []
|
|
if RUNNING_ON_MAIN_SERVER:
|
|
try:
|
|
cmd = (
|
|
f"ssh -i {VM_KEY} -p {VM_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"-o BatchMode=yes "
|
|
f"{VM_USER}@{VM_HOST} "
|
|
f"'ls -t /backups/cloudproject/myapps-backup-*.tar.gz 2>/dev/null | head -20'"
|
|
)
|
|
stdout, _ = _run(cmd, timeout=25)
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
line = line.strip()
|
|
if line and '.tar.gz' in line:
|
|
vm_backups.append(os.path.basename(line))
|
|
except Exception as e:
|
|
print(f"[backups] VM backup fetch error: {e}")
|
|
else:
|
|
backup_dir = '/backups/cloudproject'
|
|
if os.path.exists(backup_dir):
|
|
files = glob.glob(f'{backup_dir}/myapps-backup-*.tar.gz')
|
|
files.sort(key=os.path.getmtime, reverse=True)
|
|
vm_backups = [os.path.basename(f) for f in files[:20]]
|
|
return vm_backups
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUP HEALTH AUDIT
|
|
# ────────────────────────────────────────────────────────────────
|
|
#
|
|
# audit_backup() used to be one large function (cognitive complexity 18).
|
|
# It's now an orchestrator that calls one small, single-purpose helper per
|
|
# check. Each helper takes the shared `add()` callback and any state it
|
|
# needs, and is responsible for exactly one check — easy to read, easy to
|
|
# test, easy to extend without pushing complexity back up.
|
|
|
|
def _resolve_archive_path(backup_file, source, add):
|
|
"""Figure out the local path to the archive, pulling it from the main
|
|
server first if needed. Returns the path, or None if it could not be
|
|
resolved (in which case an error result dict has already been added
|
|
via `add` and the caller should bail out)."""
|
|
if source == 'local':
|
|
archive_path = f"/root/backups/{backup_file}"
|
|
else:
|
|
archive_path = f"/backups/cloudproject/{backup_file}"
|
|
|
|
if not RUNNING_ON_MAIN_SERVER and source == 'local':
|
|
tmp_path = f"/tmp/audit_{backup_file}"
|
|
if not os.path.exists(tmp_path):
|
|
pull_cmd = (
|
|
f"scp -i {MAIN_SERVER_KEY} -P {MAIN_SERVER_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=15 "
|
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}:/root/backups/{backup_file} "
|
|
f"{tmp_path}"
|
|
)
|
|
_out, err = _run(pull_cmd, timeout=120)
|
|
if not os.path.exists(tmp_path):
|
|
add('File Access', 'fail', f'Could not pull from main server: {err}')
|
|
return None
|
|
archive_path = tmp_path
|
|
|
|
return archive_path
|
|
|
|
|
|
def _check_file_size(archive_path, add):
|
|
size_bytes = os.path.getsize(archive_path)
|
|
size_mb = size_bytes / (1024 * 1024)
|
|
size_human = _human_bytes(size_bytes)
|
|
size_more = [
|
|
f'Exact size: {size_bytes:,} bytes ({size_human})',
|
|
'We flag archives under 1 MB as corrupt and under ~50 MB as unusually small for a full stack backup.',
|
|
]
|
|
if size_bytes < 1024 * 1024:
|
|
add('File Size', 'fail', f'{size_human} — suspiciously tiny, likely corrupt', more=size_more)
|
|
elif size_mb < 50:
|
|
add('File Size', 'warn', f'{size_human} — smaller than expected (typical full backup > 50 MB)', more=size_more)
|
|
else:
|
|
add('File Size', 'pass', f'{size_human} — within expected range', more=size_more)
|
|
return size_bytes, size_human
|
|
|
|
|
|
def _check_checksum(archive_path, add):
|
|
sha_file = archive_path + '.sha256'
|
|
if not os.path.exists(sha_file):
|
|
add('Checksum (SHA256)', 'warn', 'No .sha256 sidecar found — run a new backup to get checksums')
|
|
return
|
|
try:
|
|
with open(sha_file, 'r') as f:
|
|
expected_hash = f.read().split()[0].strip()
|
|
actual_hash = _sha256_file(archive_path)
|
|
if actual_hash == expected_hash:
|
|
add('Checksum (SHA256)', 'pass', f'Hash verified — {actual_hash[:20]}…')
|
|
else:
|
|
add('Checksum (SHA256)', 'fail',
|
|
f'MISMATCH — expected {expected_hash[:20]}… got {actual_hash[:20]}…')
|
|
except Exception as e:
|
|
add('Checksum (SHA256)', 'warn', f'Could not verify: {e}')
|
|
|
|
|
|
def _check_archive_integrity(archive_path, add):
|
|
try:
|
|
result = subprocess.run(
|
|
['gzip', '--test', archive_path],
|
|
capture_output=True, text=True, timeout=120
|
|
)
|
|
if result.returncode == 0:
|
|
add('Archive Integrity', 'pass', 'gzip test passed — archive is not corrupted', more=[
|
|
'Runs gzip --test on the .tar.gz so the compressed stream is readable end-to-end.',
|
|
])
|
|
else:
|
|
add('Archive Integrity', 'fail',
|
|
f'gzip test failed: {(result.stderr or result.stdout)[:200]}')
|
|
except FileNotFoundError:
|
|
_check_archive_integrity_fallback(archive_path, add)
|
|
except subprocess.TimeoutExpired:
|
|
add('Archive Integrity', 'warn', 'Integrity check timed out — file is large, probably OK')
|
|
except Exception as e:
|
|
add('Archive Integrity', 'warn', f'Could not test: {e}')
|
|
|
|
|
|
def _check_archive_integrity_fallback(archive_path, add):
|
|
try:
|
|
import gzip
|
|
with gzip.open(archive_path, 'rb') as f:
|
|
f.read(1024 * 1024)
|
|
add('Archive Integrity', 'pass', 'gzip header valid')
|
|
except Exception as e:
|
|
add('Archive Integrity', 'fail', f'Archive appears corrupt: {e}')
|
|
|
|
|
|
def _list_archive_members(archive_path):
|
|
try:
|
|
with tarfile.open(archive_path, 'r:gz') as tf:
|
|
return tf.getnames()
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _check_internal_structure(members, add):
|
|
if not members:
|
|
add('Internal Structure', 'warn', 'Could not inspect archive members')
|
|
return
|
|
|
|
has_volumes = any('volumes/' in m for m in members)
|
|
has_info = any('backup-info.txt' in m for m in members)
|
|
has_compose = any('compose-files/' in m for m in members)
|
|
vol_count = len([m for m in members if '/volumes/' in m and m.endswith('.tar.gz')])
|
|
|
|
issues = []
|
|
if not has_volumes:
|
|
issues.append('volumes/ missing')
|
|
if not has_info:
|
|
issues.append('backup-info.txt missing')
|
|
|
|
if issues:
|
|
add('Internal Structure', 'fail', ' · '.join(issues))
|
|
return
|
|
|
|
detail = 'volumes/ ✓ backup-info.txt ✓'
|
|
if has_compose:
|
|
detail += ' compose-files/ ✓'
|
|
detail += f' ({vol_count} volume archives)'
|
|
add('Internal Structure', 'pass', detail)
|
|
|
|
|
|
_SUSPICIOUS_PATTERNS = [
|
|
(r'\.\./', 'path traversal (..)'),
|
|
(r'^/', 'absolute path in archive'),
|
|
(r'/etc/passwd', '/etc/passwd reference'),
|
|
(r'/etc/shadow', '/etc/shadow reference'),
|
|
(r'\.ssh/', '.ssh directory reference'),
|
|
(r'id_rsa(?!\.pub)', 'private SSH key reference'),
|
|
(r'authorized_keys', 'authorized_keys reference'),
|
|
]
|
|
|
|
|
|
def _check_security_scan(members, add):
|
|
found_suspicious = []
|
|
for m in members:
|
|
for pat, label in _SUSPICIOUS_PATTERNS:
|
|
if re.search(pat, m):
|
|
found_suspicious.append(f'{m} ({label})')
|
|
break
|
|
|
|
if found_suspicious:
|
|
add('Security Scan', 'fail', f'Suspicious entries found: {found_suspicious[:3]}')
|
|
else:
|
|
add('Security Scan', 'pass', 'No path traversal or dangerous entries detected', more=[
|
|
'Member paths are checked for .. segments, absolute roots, and sensitive paths '
|
|
'(e.g. .ssh, /etc/shadow).',
|
|
])
|
|
|
|
|
|
_SCRIPT_EXTENSIONS = ('.sh', '.py', '.pl', '.rb', '.bash', '.zsh')
|
|
_SAFE_PREFIXES = ('compose-files/', 'volumes/', 'container-configs/', 'configs/')
|
|
|
|
|
|
def _is_unexpected_executable_script(member):
|
|
name = member.name
|
|
if any(name.startswith(p) or f'/{p}' in name for p in _SAFE_PREFIXES):
|
|
return False
|
|
name_lower = name.lower()
|
|
has_script_ext = any(name_lower.endswith(ext) for ext in _SCRIPT_EXTENSIONS)
|
|
has_exec_bit = bool(member.mode & 0o111)
|
|
return has_script_ext and has_exec_bit
|
|
|
|
|
|
def _check_executable_scripts(archive_path, add):
|
|
suspicious_scripts = []
|
|
try:
|
|
with tarfile.open(archive_path, 'r:gz') as tf:
|
|
for member in tf.getmembers():
|
|
if member.isfile() and _is_unexpected_executable_script(member):
|
|
suspicious_scripts.append(os.path.basename(member.name))
|
|
except Exception:
|
|
pass
|
|
|
|
if suspicious_scripts:
|
|
add('Executable Scripts', 'warn',
|
|
f'Scripts with execute bit outside expected dirs: {suspicious_scripts[:3]}')
|
|
else:
|
|
add('Executable Scripts', 'pass', 'No unexpected executable scripts found')
|
|
|
|
|
|
def _check_volume_count(members, add):
|
|
vol_archives = [m for m in members if 'volumes/' in m and m.endswith('.tar.gz')]
|
|
v = len(vol_archives)
|
|
if v == 0:
|
|
add('Volume Count', 'fail', 'No volume archives found in backup')
|
|
elif v < 5:
|
|
add('Volume Count', 'warn', f'Only {v} volumes (expected ≥5 for a full backup)')
|
|
else:
|
|
add('Volume Count', 'pass', f'{v} volume archives present')
|
|
|
|
|
|
def _score_checks(checks):
|
|
weights = {'pass': 10, 'warn': 5, 'fail': 0}
|
|
total = len(checks) * 10
|
|
earned = sum(weights.get(c['status'], 0) for c in checks)
|
|
return int((earned / total) * 100) if total > 0 else 0
|
|
|
|
|
|
def _summary_for_score(score):
|
|
if score >= 90:
|
|
return 'Backup looks healthy and is safe to restore.'
|
|
if score >= 70:
|
|
return 'Minor warnings — likely safe, but review before restoring.'
|
|
if score >= 40:
|
|
return 'Significant issues detected — restore with caution.'
|
|
return 'Multiple checks failed — do NOT restore without manual inspection.'
|
|
|
|
|
|
def _health_tier_for(score, has_fails, has_warns):
|
|
if has_fails:
|
|
return 'critical', 'Unhealthy'
|
|
if score == 100:
|
|
return 'excellent', '100% healthy'
|
|
if score >= 90:
|
|
return 'good', ('Healthy' if not has_warns else 'Healthy (with notes)')
|
|
if score >= 70:
|
|
return 'fair', 'Mostly healthy'
|
|
if score >= 40:
|
|
return 'poor', 'At risk'
|
|
return 'critical', 'Unhealthy'
|
|
|
|
|
|
def _missing_result(backup_file, checks, detail=None):
|
|
return {
|
|
'ok': False, 'score': 0, 'checks': checks,
|
|
'backup_file': backup_file,
|
|
'file_size_bytes': None,
|
|
'file_size_display': None,
|
|
'health_tier': 'critical',
|
|
'health_label': 'Unhealthy',
|
|
'summary': detail or 'Cannot access backup file from this host.',
|
|
}
|
|
|
|
|
|
def audit_backup(backup_file, source='local'):
|
|
checks = []
|
|
|
|
def add(name, status, detail='', more=None):
|
|
entry = {'name': name, 'status': status, 'detail': detail}
|
|
if more:
|
|
entry['more'] = more
|
|
checks.append(entry)
|
|
|
|
archive_path = _resolve_archive_path(backup_file, source, add)
|
|
if archive_path is None:
|
|
return _missing_result(backup_file, checks, 'Cannot access backup file from this host.')
|
|
|
|
if not os.path.exists(archive_path):
|
|
add('File Exists', 'fail', f'Not found: {archive_path}')
|
|
return _missing_result(backup_file, checks, 'Backup file does not exist on disk.')
|
|
add('File Exists', 'pass', archive_path)
|
|
|
|
size_bytes, size_human = _check_file_size(archive_path, add)
|
|
_check_checksum(archive_path, add)
|
|
_check_archive_integrity(archive_path, add)
|
|
|
|
members = _list_archive_members(archive_path)
|
|
_check_internal_structure(members, add)
|
|
_check_security_scan(members, add)
|
|
_check_executable_scripts(archive_path, add)
|
|
_check_volume_count(members, add)
|
|
|
|
score = _score_checks(checks)
|
|
has_fails = any(c['status'] == 'fail' for c in checks)
|
|
has_warns = any(c['status'] == 'warn' for c in checks)
|
|
ok = not has_fails and score >= 60
|
|
health_tier, health_label = _health_tier_for(score, has_fails, has_warns)
|
|
|
|
return {
|
|
'ok': ok,
|
|
'score': score,
|
|
'checks': checks,
|
|
'summary': _summary_for_score(score),
|
|
'backup_file': backup_file,
|
|
'file_size_bytes': size_bytes,
|
|
'file_size_display': size_human,
|
|
'health_tier': health_tier,
|
|
'health_label': health_label,
|
|
}
|
|
|
|
|
|
def _sha256_file(path):
|
|
h = hashlib.sha256()
|
|
with open(path, 'rb') as f:
|
|
for chunk in iter(lambda: f.read(65536), b''):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# DELETE BACKUP
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def delete_backup(backup_file, source='local'):
|
|
if not re.match(r'^myapps-backup-\d{8}_\d{6}\.tar\.gz$', backup_file):
|
|
return False, f'Invalid backup filename: {backup_file}'
|
|
|
|
if source == 'local':
|
|
if RUNNING_ON_MAIN_SERVER:
|
|
archive_path = f"/root/backups/{backup_file}"
|
|
if not os.path.exists(archive_path):
|
|
return False, f'File not found: {archive_path}'
|
|
os.remove(archive_path)
|
|
sha = archive_path + '.sha256'
|
|
if os.path.exists(sha): os.remove(sha)
|
|
return True, f'Deleted {backup_file} from main server'
|
|
else:
|
|
cmd = f"rm -f /root/backups/{backup_file} /root/backups/{backup_file}.sha256"
|
|
out, err = _ssh_main(cmd)
|
|
if err and 'No such file' not in err:
|
|
return False, f'Remote delete error: {err}'
|
|
return True, f'Deleted {backup_file} from main server'
|
|
|
|
elif source == 'vm':
|
|
archive_path = f"/backups/cloudproject/{backup_file}"
|
|
if not RUNNING_ON_MAIN_SERVER:
|
|
if not os.path.exists(archive_path):
|
|
return False, f'File not found: {archive_path}'
|
|
os.remove(archive_path)
|
|
sha = archive_path + '.sha256'
|
|
if os.path.exists(sha): os.remove(sha)
|
|
return True, f'Deleted {backup_file} from VM'
|
|
else:
|
|
cmd = (
|
|
f"ssh -i {VM_KEY} -p {VM_PORT} "
|
|
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
|
f"-o BatchMode=yes "
|
|
f"{VM_USER}@{VM_HOST} "
|
|
f"'rm -f /backups/cloudproject/{backup_file} "
|
|
f"/backups/cloudproject/{backup_file}.sha256'"
|
|
)
|
|
out, err = _run(cmd, timeout=30)
|
|
if err and 'No such file' not in err:
|
|
return False, f'VM delete error: {err}'
|
|
return True, f'Deleted {backup_file} from VM'
|
|
|
|
return False, 'Unknown source'
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# BACKUP STATUS LOG
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_backup_log_entries(limit=20):
|
|
stdout, _ = _ssh_main(
|
|
f"tail -n {limit} /root/backups/backup-status.log 2>/dev/null || echo ''"
|
|
)
|
|
entries = []
|
|
if not stdout:
|
|
return entries
|
|
for line in stdout.strip().split('\n'):
|
|
if not line.strip():
|
|
continue
|
|
parts = line.split('|')
|
|
entries.append({
|
|
'timestamp': parts[0].strip() if len(parts) > 0 else '',
|
|
'status': parts[1].strip() if len(parts) > 1 else '',
|
|
'name': parts[2].strip() if len(parts) > 2 else '',
|
|
'message': parts[3].strip() if len(parts) > 3 else '',
|
|
})
|
|
return list(reversed(entries))
|
|
|
|
|
|
def get_backup_script_path():
|
|
candidates = ['/root/backup-myapps.sh']
|
|
for p in candidates:
|
|
out, _ = _ssh_main(f"[ -f {p} ] && echo yes || echo no")
|
|
if out.strip() == 'yes':
|
|
return p
|
|
return None
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# CONTAINERS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def _parse_uptime(status):
|
|
"""Human uptime snippet from docker ps Status field."""
|
|
if not status:
|
|
return '—'
|
|
if status.startswith('Up'):
|
|
m = re.match(r'Up\s+([^(]+)', status)
|
|
return ('Up ' + m.group(1).strip()) if m else status.split('(')[0].strip()
|
|
if status.startswith('Exited'):
|
|
return status.split('(')[0].strip() or status
|
|
return status
|
|
|
|
|
|
def _parse_containers(raw, owner='root'):
|
|
containers = []
|
|
if raw:
|
|
for line in raw.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
status = parts[1].strip() if len(parts) > 1 else ''
|
|
containers.append({
|
|
'name': parts[0].strip(),
|
|
'status': status,
|
|
'image': parts[2].strip() if len(parts) > 2 else '',
|
|
'ports': parts[3].strip() if len(parts) > 3 else '',
|
|
'created_at': parts[4].strip() if len(parts) > 4 else '',
|
|
'uptime': _parse_uptime(status),
|
|
'running': status.startswith('Up'),
|
|
'owner': owner,
|
|
})
|
|
return containers
|
|
|
|
|
|
_DOCKER_PS_FMT = (
|
|
"{{.Names}}|{{.Status}}|{{.Image}}|{{.Ports}}|{{.CreatedAt}}"
|
|
)
|
|
|
|
|
|
def get_containers():
|
|
stdout, _ = _ssh_main(
|
|
f"docker ps -a --format '{_DOCKER_PS_FMT}' 2>/dev/null | "
|
|
"grep -E 'frappe|nextcloud|mautic|n8n|odoo'"
|
|
)
|
|
return _parse_containers(stdout)
|
|
|
|
|
|
def get_all_root_containers():
|
|
stdout, _ = _ssh_main(
|
|
f"docker ps -a --format '{_DOCKER_PS_FMT}' 2>/dev/null"
|
|
)
|
|
return _parse_containers(stdout)
|
|
|
|
|
|
def get_rootless_user_containers_remote():
|
|
cmd = "ls /run/user/*/docker.sock 2>/dev/null"
|
|
stdout, _ = _ssh_main(cmd)
|
|
containers = []
|
|
if not stdout:
|
|
return containers
|
|
for sock_path in stdout.split('\n'):
|
|
sock_path = sock_path.strip()
|
|
if not sock_path:
|
|
continue
|
|
try:
|
|
uid = sock_path.split('/run/user/')[1].split('/')[0]
|
|
except (IndexError, ValueError):
|
|
continue
|
|
name_out, _ = _ssh_main(f"getent passwd {uid} | cut -d: -f1")
|
|
username = name_out.strip() or f"uid{uid}"
|
|
ctr_out, _ = _ssh_main(
|
|
f"DOCKER_HOST=unix://{sock_path} "
|
|
f"docker ps -a --format '{_DOCKER_PS_FMT}' 2>/dev/null"
|
|
)
|
|
containers.extend(_parse_containers(ctr_out, owner=username))
|
|
return containers
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# CONTAINER ACTIONS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def container_action(container_name, action):
|
|
if action not in ('start', 'stop', 'restart'):
|
|
return False, "Invalid action"
|
|
safe_name = container_name.replace('"', '').replace(';', '').replace('|', '')
|
|
stdout, stderr = _ssh_main(f"docker {action} {safe_name} 2>&1", timeout=30)
|
|
output = (stdout + stderr).strip()
|
|
return True, output
|
|
|
|
|
|
def get_container_status(container_name):
|
|
safe_name = container_name.replace('"', '').replace(';', '').replace('|', '')
|
|
stdout, _ = _ssh_main(
|
|
f"docker inspect --format='{{{{.State.Status}}}}' {safe_name} 2>/dev/null"
|
|
)
|
|
raw = stdout.strip().lower()
|
|
if raw in ('running', 'restarting'):
|
|
status = 'running'
|
|
elif raw in ('exited', 'stopped', 'dead', 'paused'):
|
|
status = 'stopped'
|
|
else:
|
|
status = 'unknown'
|
|
return {'name': container_name, 'status': status, 'raw': raw}
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# STATS
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_container_stats_remote():
|
|
stdout, _ = _ssh_main(
|
|
"docker stats --no-stream --format "
|
|
"'{{.Name}}|{{.CPUPerc}}|{{.MemUsage}}|{{.MemPerc}}|{{.NetIO}}|{{.BlockIO}}' 2>/dev/null",
|
|
timeout=35
|
|
)
|
|
stats = {}
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
if len(parts) < 6:
|
|
continue
|
|
name = parts[0].strip()
|
|
stats[name] = {
|
|
'cpu': parts[1].strip(),
|
|
'mem': parts[2].strip(),
|
|
'mem_pct': parts[3].strip(),
|
|
'net': parts[4].strip(),
|
|
'block': parts[5].strip(),
|
|
}
|
|
return stats
|
|
|
|
|
|
def get_all_stats():
|
|
all_stats = get_container_stats_remote()
|
|
socks_out, _ = _ssh_main("ls /run/user/*/docker.sock 2>/dev/null")
|
|
if socks_out:
|
|
for sock in socks_out.split('\n'):
|
|
sock = sock.strip()
|
|
if not sock:
|
|
continue
|
|
stdout, _ = _ssh_main(
|
|
f"DOCKER_HOST=unix://{sock} "
|
|
f"docker stats --no-stream --format "
|
|
f"'{{{{.Name}}}}|{{{{.CPUPerc}}}}|{{{{.MemUsage}}}}|{{{{.MemPerc}}}}|{{{{.NetIO}}}}|{{{{.BlockIO}}}}' 2>/dev/null",
|
|
timeout=35
|
|
)
|
|
if stdout:
|
|
for line in stdout.split('\n'):
|
|
if '|' not in line:
|
|
continue
|
|
parts = line.split('|')
|
|
if len(parts) < 6:
|
|
continue
|
|
all_stats[parts[0].strip()] = {
|
|
'cpu': parts[1].strip(),
|
|
'mem': parts[2].strip(),
|
|
'mem_pct': parts[3].strip(),
|
|
'net': parts[4].strip(),
|
|
'block': parts[5].strip(),
|
|
}
|
|
return all_stats
|
|
|
|
|
|
# ────────────────────────────────────────────────────────────────
|
|
# SYSTEM INFO (FIXED WITH PSUTIL)
|
|
# ────────────────────────────────────────────────────────────────
|
|
|
|
def get_system_info():
|
|
"""Get system information using psutil for reliable metrics inside container"""
|
|
|
|
info = {
|
|
'cpu_pct': '0',
|
|
'memory': 'N/A',
|
|
'mem_pct': '0',
|
|
'disk': 'N/A',
|
|
'disk_pct': '0',
|
|
'load': 'N/A',
|
|
'uptime': 'N/A',
|
|
'docker_v': 'N/A',
|
|
'hostname': socket.gethostname()
|
|
}
|
|
|
|
try:
|
|
# CPU usage - psutil handles /proc access properly
|
|
info['cpu_pct'] = str(psutil.cpu_percent(interval=0.5))
|
|
|
|
# Memory usage
|
|
mem = psutil.virtual_memory()
|
|
mem_used_gb = mem.used / (1024**3)
|
|
mem_total_gb = mem.total / (1024**3)
|
|
info['memory'] = f"{mem_used_gb:.1f}G/{mem_total_gb:.1f}G"
|
|
info['mem_pct'] = str(int(mem.percent))
|
|
|
|
# Load average
|
|
try:
|
|
load = psutil.getloadavg()
|
|
info['load'] = f"{load[0]:.2f} {load[1]:.2f} {load[2]:.2f}"
|
|
except AttributeError:
|
|
# Fallback for systems without getloadavg
|
|
with open('/proc/loadavg', 'r') as f:
|
|
load = f.read().split()[:3]
|
|
info['load'] = ' '.join(load)
|
|
|
|
# Disk usage
|
|
try:
|
|
disk = psutil.disk_usage('/')
|
|
disk_used_gb = disk.used / (1024**3)
|
|
disk_total_gb = disk.total / (1024**3)
|
|
info['disk'] = f"{disk_used_gb:.0f}G/{disk_total_gb:.0f}G"
|
|
info['disk_pct'] = str(int(disk.percent))
|
|
except Exception as e:
|
|
print(f"Disk read error: {e}")
|
|
|
|
# Uptime
|
|
try:
|
|
boot_time = psutil.boot_time()
|
|
uptime_seconds = time.time() - boot_time
|
|
days = int(uptime_seconds // 86400)
|
|
hours = int((uptime_seconds % 86400) // 3600)
|
|
minutes = int((uptime_seconds % 3600) // 60)
|
|
info['uptime'] = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
|
|
except Exception as e:
|
|
print(f"Uptime read error: {e}")
|
|
|
|
# Docker version
|
|
try:
|
|
docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True)
|
|
info['docker_v'] = docker_v.split()[-1].strip(',')
|
|
except Exception as e:
|
|
print(f"Docker version error: {e}")
|
|
|
|
except Exception as e:
|
|
print(f"System info error: {e}")
|
|
# Fallback to /proc method if psutil fails
|
|
info = _get_system_info_fallback()
|
|
|
|
return info
|
|
|
|
|
|
def _get_system_info_fallback():
|
|
"""Fallback method using /proc directly when psutil is not available"""
|
|
info = {
|
|
'cpu_pct': '0',
|
|
'memory': 'N/A',
|
|
'mem_pct': '0',
|
|
'disk': 'N/A',
|
|
'disk_pct': '0',
|
|
'load': 'N/A',
|
|
'uptime': 'N/A',
|
|
'docker_v': 'N/A',
|
|
'hostname': socket.gethostname()
|
|
}
|
|
|
|
try:
|
|
# CPU calculation with delta
|
|
def _read_cpu():
|
|
with open('/proc/stat', 'r') as f:
|
|
fields = f.readline().split()
|
|
idle = int(fields[4])
|
|
total = sum(int(x) for x in fields[1:])
|
|
return idle, total
|
|
|
|
idle1, total1 = _read_cpu()
|
|
time.sleep(0.5)
|
|
idle2, total2 = _read_cpu()
|
|
|
|
idle_delta = idle2 - idle1
|
|
total_delta = total2 - total1
|
|
if total_delta > 0:
|
|
cpu_pct = round(100 * (1 - idle_delta / total_delta), 1)
|
|
info['cpu_pct'] = str(cpu_pct)
|
|
|
|
# Memory
|
|
mem = {}
|
|
with open('/proc/meminfo', 'r') as f:
|
|
for line in f:
|
|
parts = line.split(':')
|
|
if len(parts) == 2:
|
|
mem[parts[0].strip()] = int(parts[1].strip().split()[0])
|
|
|
|
if 'MemTotal' in mem and 'MemAvailable' in mem:
|
|
mem_total_mb = mem['MemTotal'] // 1024
|
|
mem_used_mb = (mem['MemTotal'] - mem['MemAvailable']) // 1024
|
|
info['memory'] = f"{mem_used_mb}M/{mem_total_mb}M"
|
|
info['mem_pct'] = str(int(100 * mem_used_mb / mem_total_mb))
|
|
|
|
# Load
|
|
with open('/proc/loadavg', 'r') as f:
|
|
load = f.read().split()[:3]
|
|
info['load'] = ' '.join(load)
|
|
|
|
# Uptime
|
|
with open('/proc/uptime', 'r') as f:
|
|
secs = int(float(f.read().split()[0]))
|
|
days = secs // 86400
|
|
hours = (secs % 86400) // 3600
|
|
minutes = (secs % 3600) // 60
|
|
info['uptime'] = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m"
|
|
|
|
# Disk
|
|
try:
|
|
disk = psutil.disk_usage('/')
|
|
disk_used_gb = disk.used / (1024**3)
|
|
disk_total_gb = disk.total / (1024**3)
|
|
info['disk'] = f"{disk_used_gb:.0f}G/{disk_total_gb:.0f}G"
|
|
info['disk_pct'] = str(int(disk.percent))
|
|
except Exception:
|
|
import shutil
|
|
disk = shutil.disk_usage('/')
|
|
info['disk'] = f"{disk.used // (1024**3)}G/{disk.total // (1024**3)}G"
|
|
info['disk_pct'] = str(int(100 * disk.used / disk.total))
|
|
|
|
# Docker version
|
|
try:
|
|
docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True)
|
|
info['docker_v'] = docker_v.split()[-1].strip(',')
|
|
except Exception:
|
|
pass
|
|
|
|
except Exception as e:
|
|
print(f"Fallback system info error: {e}")
|
|
|
|
return info |