Sync from main server - 2026-05-05 00:20:15
This commit is contained in:
root
@@ -13,7 +13,7 @@ from config import (
|
||||
)
|
||||
|
||||
|
||||
def _run(cmd, timeout=20):
|
||||
def _run(cmd, timeout=30):
|
||||
try:
|
||||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=timeout)
|
||||
return r.stdout.strip(), r.stderr.strip()
|
||||
@@ -22,7 +22,6 @@ def _run(cmd, timeout=20):
|
||||
|
||||
|
||||
def _human_bytes(n):
|
||||
"""Human-readable byte size for audit UI."""
|
||||
n = int(n)
|
||||
if n < 1024:
|
||||
return f'{n} B'
|
||||
@@ -35,16 +34,18 @@ def _human_bytes(n):
|
||||
return f'{n / (1024 ** 4):.2f} TB'
|
||||
|
||||
|
||||
def _ssh_main(remote_cmd, timeout=20):
|
||||
def _ssh_main(remote_cmd, timeout=30):
|
||||
if RUNNING_ON_MAIN_SERVER:
|
||||
return _run(remote_cmd, timeout=timeout)
|
||||
else:
|
||||
escaped = remote_cmd.replace("'", "'\\''")
|
||||
ssh = (
|
||||
f"ssh -i {MAIN_SERVER_KEY} -p {MAIN_SERVER_PORT} "
|
||||
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
||||
f"-o BatchMode=yes "
|
||||
f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}"
|
||||
)
|
||||
return _run(f"{ssh} '{remote_cmd}'", timeout=timeout)
|
||||
return _run(f"{ssh} '{escaped}'", timeout=timeout)
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
@@ -71,6 +72,7 @@ def get_vm_backups():
|
||||
cmd = (
|
||||
f"ssh -i {VM_KEY} -p {VM_PORT} "
|
||||
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
||||
f"-o BatchMode=yes "
|
||||
f"{VM_USER}@{VM_HOST} "
|
||||
f"'ls -t /backups/main-server/myapps-backup-*.tar.gz 2>/dev/null | head -20'"
|
||||
)
|
||||
@@ -96,22 +98,6 @@ def get_vm_backups():
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def audit_backup(backup_file, source='local'):
|
||||
"""
|
||||
Perform a health and integrity audit on a backup archive.
|
||||
|
||||
Checks:
|
||||
1. File exists
|
||||
2. File size sanity
|
||||
3. SHA256 checksum (if .sha256 sidecar exists)
|
||||
4. tar archive integrity (gzip test only — portable, no conflicting flags)
|
||||
5. Expected internal structure
|
||||
6. Path traversal / suspicious paths
|
||||
7. Suspicious script files at unexpected locations (scripts only, not binaries)
|
||||
8. Volume count
|
||||
|
||||
Returns:
|
||||
{ ok, score, checks, summary }
|
||||
"""
|
||||
checks = []
|
||||
|
||||
def add(name, status, detail='', more=None):
|
||||
@@ -120,13 +106,11 @@ def audit_backup(backup_file, source='local'):
|
||||
entry['more'] = more
|
||||
checks.append(entry)
|
||||
|
||||
# ── Resolve archive path ─────────────────────────────────────────────────
|
||||
if source == 'local':
|
||||
archive_path = f"/root/backups/{backup_file}"
|
||||
else:
|
||||
archive_path = f"/backups/main-server/{backup_file}"
|
||||
|
||||
# On VM auditing a "local" (main server) backup → pull to /tmp first
|
||||
if not RUNNING_ON_MAIN_SERVER and source == 'local':
|
||||
tmp_path = f"/tmp/audit_{backup_file}"
|
||||
if not os.path.exists(tmp_path):
|
||||
@@ -151,7 +135,6 @@ def audit_backup(backup_file, source='local'):
|
||||
}
|
||||
archive_path = tmp_path
|
||||
|
||||
# ── CHECK 1: File exists ─────────────────────────────────────────────────
|
||||
if not os.path.exists(archive_path):
|
||||
add('File Exists', 'fail', f'Not found: {archive_path}')
|
||||
return {
|
||||
@@ -165,7 +148,6 @@ def audit_backup(backup_file, source='local'):
|
||||
}
|
||||
add('File Exists', 'pass', archive_path)
|
||||
|
||||
# ── CHECK 2: File size ───────────────────────────────────────────────────
|
||||
size_bytes = os.path.getsize(archive_path)
|
||||
size_mb = size_bytes / (1024 * 1024)
|
||||
size_human = _human_bytes(size_bytes)
|
||||
@@ -183,7 +165,6 @@ def audit_backup(backup_file, source='local'):
|
||||
add('File Size', 'pass',
|
||||
f'{size_human} — within expected range', more=size_more)
|
||||
|
||||
# ── CHECK 3: SHA256 checksum ─────────────────────────────────────────────
|
||||
sha_file = archive_path + '.sha256'
|
||||
if os.path.exists(sha_file):
|
||||
try:
|
||||
@@ -201,8 +182,6 @@ def audit_backup(backup_file, source='local'):
|
||||
add('Checksum (SHA256)', 'warn',
|
||||
'No .sha256 sidecar found — run a new backup to get checksums')
|
||||
|
||||
# ── CHECK 4: Archive integrity ───────────────────────────────────────────
|
||||
# Use gzip --test which works everywhere without conflicting tar flags
|
||||
try:
|
||||
result = subprocess.run(
|
||||
['gzip', '--test', archive_path],
|
||||
@@ -216,11 +195,9 @@ def audit_backup(backup_file, source='local'):
|
||||
add('Archive Integrity', 'fail',
|
||||
f'gzip test failed: {(result.stderr or result.stdout)[:200]}')
|
||||
except FileNotFoundError:
|
||||
# gzip not available — try python gzip
|
||||
try:
|
||||
import gzip
|
||||
with gzip.open(archive_path, 'rb') as f:
|
||||
# Read just the first few MB to check header validity
|
||||
f.read(1024 * 1024)
|
||||
add('Archive Integrity', 'pass', 'gzip header valid')
|
||||
except Exception as e:
|
||||
@@ -230,7 +207,6 @@ def audit_backup(backup_file, source='local'):
|
||||
except Exception as e:
|
||||
add('Archive Integrity', 'warn', f'Could not test: {e}')
|
||||
|
||||
# ── Read archive member list (used by checks 5, 6, 7, 8) ─────────────────
|
||||
members = []
|
||||
try:
|
||||
with tarfile.open(archive_path, 'r:gz') as tf:
|
||||
@@ -238,7 +214,6 @@ def audit_backup(backup_file, source='local'):
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# ── CHECK 5: Internal structure ──────────────────────────────────────────
|
||||
if members:
|
||||
has_volumes = any('volumes/' in m for m in members)
|
||||
has_info = any('backup-info.txt' in m for m in members)
|
||||
@@ -259,7 +234,6 @@ def audit_backup(backup_file, source='local'):
|
||||
else:
|
||||
add('Internal Structure', 'warn', 'Could not inspect archive members')
|
||||
|
||||
# ── CHECK 6: Path traversal / suspicious paths ────────────────────────────
|
||||
SUSPICIOUS = [
|
||||
(r'\.\./', 'path traversal (..)'),
|
||||
(r'^/', 'absolute path in archive'),
|
||||
@@ -285,10 +259,6 @@ def audit_backup(backup_file, source='local'):
|
||||
'(e.g. .ssh, /etc/shadow).',
|
||||
])
|
||||
|
||||
# ── CHECK 7: Suspicious scripts (smart — scripts only, not data files) ────
|
||||
# Only flag actual text script files (.sh .py .pl .rb) with execute bits
|
||||
# placed outside compose-files/ and outside known vendor directories.
|
||||
# .bin, .so, .exe data files are intentionally excluded (too many false positives)
|
||||
SCRIPT_EXTENSIONS = ('.sh', '.py', '.pl', '.rb', '.bash', '.zsh')
|
||||
SAFE_PREFIXES = (
|
||||
'compose-files/',
|
||||
@@ -303,10 +273,8 @@ def audit_backup(backup_file, source='local'):
|
||||
if not member.isfile():
|
||||
continue
|
||||
name = member.name
|
||||
# Skip files in known-safe directories
|
||||
if any(name.startswith(p) or f'/{p}' in name for p in SAFE_PREFIXES):
|
||||
continue
|
||||
# Only flag actual script extensions with execute bits
|
||||
name_lower = name.lower()
|
||||
has_script_ext = any(name_lower.endswith(ext) for ext in SCRIPT_EXTENSIONS)
|
||||
has_exec_bit = bool(member.mode & 0o111)
|
||||
@@ -321,7 +289,6 @@ def audit_backup(backup_file, source='local'):
|
||||
else:
|
||||
add('Executable Scripts', 'pass', 'No unexpected executable scripts found')
|
||||
|
||||
# ── CHECK 8: Volume count ────────────────────────────────────────────────
|
||||
vol_archives = [m for m in members if 'volumes/' in m and m.endswith('.tar.gz')]
|
||||
v = len(vol_archives)
|
||||
if v == 0:
|
||||
@@ -331,7 +298,6 @@ def audit_backup(backup_file, source='local'):
|
||||
else:
|
||||
add('Volume Count', 'pass', f'{v} volume archives present')
|
||||
|
||||
# ── Score ─────────────────────────────────────────────────────────────────
|
||||
weights = {'pass': 10, 'warn': 5, 'fail': 0}
|
||||
total = len(checks) * 10
|
||||
earned = sum(weights.get(c['status'], 0) for c in checks)
|
||||
@@ -427,6 +393,7 @@ def delete_backup(backup_file, source='local'):
|
||||
cmd = (
|
||||
f"ssh -i {VM_KEY} -p {VM_PORT} "
|
||||
f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 "
|
||||
f"-o BatchMode=yes "
|
||||
f"{VM_USER}@{VM_HOST} "
|
||||
f"'rm -f /backups/main-server/{backup_file} "
|
||||
f"/backups/main-server/{backup_file}.sha256'"
|
||||
@@ -621,19 +588,59 @@ def get_all_stats():
|
||||
|
||||
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
# SYSTEM INFO
|
||||
# SYSTEM INFO — single batched SSH call
|
||||
# ────────────────────────────────────────────────────────────────
|
||||
|
||||
def get_system_info():
|
||||
cpu_out, _ = _ssh_main("top -bn1 | grep 'Cpu(s)' | awk '{print $2+$4}'")
|
||||
mem_out, _ = _ssh_main("free -m | awk 'NR==2{printf \"%s/%sMB\", $3, $2}'")
|
||||
mem_pct, _ = _ssh_main("free | awk 'NR==2{printf \"%.0f\", $3/$2*100}'")
|
||||
disk_out, _ = _ssh_main("df -h / | awk 'NR==2{printf \"%s/%s\", $3, $2}'")
|
||||
disk_pct, _ = _ssh_main("df / | awk 'NR==2{print $5}' | tr -d '%'")
|
||||
load_out, _ = _ssh_main("cat /proc/loadavg | awk '{print $1, $2, $3}'")
|
||||
uptime, _ = _ssh_main("uptime -p")
|
||||
docker_v, _ = _ssh_main("docker --version | cut -d' ' -f3 | tr -d ','")
|
||||
hostname, _ = _run("hostname -f 2>/dev/null || hostname")
|
||||
"""
|
||||
Collect all system metrics in a SINGLE SSH call instead of 8 separate ones.
|
||||
Emits a pipe-delimited line: cpu|mem|mem_pct|disk|disk_pct|load|uptime|docker_v|hostname
|
||||
"""
|
||||
batch_cmd = (
|
||||
"printf '%s|%s|%s|%s|%s|%s|%s|%s|%s\\n' "
|
||||
"\"$(top -bn1 | grep 'Cpu(s)' | awk '{print $2+$4}')\" "
|
||||
"\"$(free -m | awk 'NR==2{printf \"%s/%sMB\", $3, $2}')\" "
|
||||
"\"$(free | awk 'NR==2{printf \"%.0f\", $3/$2*100}')\" "
|
||||
"\"$(df -h / | awk 'NR==2{printf \"%s/%s\", $3, $2}')\" "
|
||||
"\"$(df / | awk 'NR==2{print $5}' | tr -d '%')\" "
|
||||
"\"$(cat /proc/loadavg | awk '{print $1, $2, $3}')\" "
|
||||
"\"$(uptime -p)\" "
|
||||
"\"$(docker --version 2>/dev/null | cut -d' ' -f3 | tr -d ',')\" "
|
||||
"\"$(hostname -f 2>/dev/null || hostname)\""
|
||||
)
|
||||
|
||||
stdout, stderr = _ssh_main(batch_cmd, timeout=20)
|
||||
|
||||
# Parse the pipe-delimited result
|
||||
if stdout and '|' in stdout:
|
||||
# Use the last line in case there's extra output
|
||||
for line in reversed(stdout.splitlines()):
|
||||
line = line.strip()
|
||||
if '|' in line:
|
||||
parts = line.split('|')
|
||||
if len(parts) >= 9:
|
||||
return {
|
||||
'cpu_pct': parts[0] or '0',
|
||||
'memory': parts[1] or 'N/A',
|
||||
'mem_pct': parts[2] or '0',
|
||||
'disk': parts[3] or 'N/A',
|
||||
'disk_pct': parts[4] or '0',
|
||||
'load': parts[5] or 'N/A',
|
||||
'uptime': parts[6] or 'N/A',
|
||||
'docker_v': parts[7] or 'N/A',
|
||||
'hostname': parts[8] or 'main server',
|
||||
}
|
||||
|
||||
# Fallback: individual calls if batch failed
|
||||
cpu_out, _ = _ssh_main("top -bn1 | grep 'Cpu(s)' | awk '{print $2+$4}'")
|
||||
mem_out, _ = _ssh_main("free -m | awk 'NR==2{printf \"%s/%sMB\", $3, $2}'")
|
||||
mem_pct, _ = _ssh_main("free | awk 'NR==2{printf \"%.0f\", $3/$2*100}'")
|
||||
disk_out, _ = _ssh_main("df -h / | awk 'NR==2{printf \"%s/%s\", $3, $2}'")
|
||||
disk_pct, _ = _ssh_main("df / | awk 'NR==2{print $5}' | tr -d '%'")
|
||||
load_out, _ = _ssh_main("cat /proc/loadavg | awk '{print $1, $2, $3}'")
|
||||
uptime, _ = _ssh_main("uptime -p")
|
||||
docker_v, _ = _ssh_main("docker --version | cut -d' ' -f3 | tr -d ','")
|
||||
hostname, _ = _run("hostname -f 2>/dev/null || hostname")
|
||||
return {
|
||||
'cpu_pct': cpu_out or '0',
|
||||
'memory': mem_out or 'N/A',
|
||||
@@ -644,4 +651,4 @@ def get_system_info():
|
||||
'uptime': uptime or 'N/A',
|
||||
'docker_v': docker_v or 'N/A',
|
||||
'hostname': hostname or 'main server',
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user