diff --git a/platform/modules/backups.py b/platform/modules/backups.py index 4293b46..ab2905c 100644 --- a/platform/modules/backups.py +++ b/platform/modules/backups.py @@ -99,16 +99,18 @@ def get_vm_backups(): # ──────────────────────────────────────────────────────────────── # BACKUP HEALTH AUDIT # ──────────────────────────────────────────────────────────────── +# +# audit_backup() used to be one large function (cognitive complexity 18). +# It's now an orchestrator that calls one small, single-purpose helper per +# check. Each helper takes the shared `add()` callback and any state it +# needs, and is responsible for exactly one check — easy to read, easy to +# test, easy to extend without pushing complexity back up. -def audit_backup(backup_file, source='local'): - checks = [] - - def add(name, status, detail='', more=None): - entry = {'name': name, 'status': status, 'detail': detail} - if more: - entry['more'] = more - checks.append(entry) - +def _resolve_archive_path(backup_file, source, add): + """Figure out the local path to the archive, pulling it from the main + server first if needed. Returns the path, or None if it could not be + resolved (in which case an error result dict has already been added + via `add` and the caller should bail out).""" if source == 'local': archive_path = f"/root/backups/{backup_file}" else: @@ -123,34 +125,16 @@ def audit_backup(backup_file, source='local'): f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}:/root/backups/{backup_file} " f"{tmp_path}" ) - out, err = _run(pull_cmd, timeout=120) + _out, err = _run(pull_cmd, timeout=120) if not os.path.exists(tmp_path): - return { - 'ok': False, 'score': 0, - 'backup_file': backup_file, - 'file_size_bytes': None, - 'file_size_display': None, - 'health_tier': 'critical', - 'health_label': 'Unhealthy', - 'checks': [{'name': 'File Access', 'status': 'fail', - 'detail': f'Could not pull from main server: {err}'}], - 'summary': 'Cannot access backup file from this host.' - } + add('File Access', 'fail', f'Could not pull from main server: {err}') + return None archive_path = tmp_path - if not os.path.exists(archive_path): - add('File Exists', 'fail', f'Not found: {archive_path}') - return { - 'ok': False, 'score': 0, 'checks': checks, - 'backup_file': backup_file, - 'file_size_bytes': None, - 'file_size_display': None, - 'health_tier': 'critical', - 'health_label': 'Unhealthy', - 'summary': 'Backup file does not exist on disk.', - } - add('File Exists', 'pass', archive_path) + return archive_path + +def _check_file_size(archive_path, add): size_bytes = os.path.getsize(archive_path) size_mb = size_bytes / (1024 * 1024) size_human = _human_bytes(size_bytes) @@ -159,32 +143,33 @@ def audit_backup(backup_file, source='local'): 'We flag archives under 1 MB as corrupt and under ~50 MB as unusually small for a full stack backup.', ] if size_bytes < 1024 * 1024: - add('File Size', 'fail', - f'{size_human} — suspiciously tiny, likely corrupt', more=size_more) + add('File Size', 'fail', f'{size_human} — suspiciously tiny, likely corrupt', more=size_more) elif size_mb < 50: - add('File Size', 'warn', - f'{size_human} — smaller than expected (typical full backup > 50 MB)', more=size_more) + add('File Size', 'warn', f'{size_human} — smaller than expected (typical full backup > 50 MB)', more=size_more) else: - add('File Size', 'pass', - f'{size_human} — within expected range', more=size_more) + add('File Size', 'pass', f'{size_human} — within expected range', more=size_more) + return size_bytes, size_human + +def _check_checksum(archive_path, add): sha_file = archive_path + '.sha256' - if os.path.exists(sha_file): - try: - with open(sha_file, 'r') as f: - expected_hash = f.read().split()[0].strip() - actual_hash = _sha256_file(archive_path) - if actual_hash == expected_hash: - add('Checksum (SHA256)', 'pass', f'Hash verified — {actual_hash[:20]}…') - else: - add('Checksum (SHA256)', 'fail', - f'MISMATCH — expected {expected_hash[:20]}… got {actual_hash[:20]}…') - except Exception as e: - add('Checksum (SHA256)', 'warn', f'Could not verify: {e}') - else: - add('Checksum (SHA256)', 'warn', - 'No .sha256 sidecar found — run a new backup to get checksums') + if not os.path.exists(sha_file): + add('Checksum (SHA256)', 'warn', 'No .sha256 sidecar found — run a new backup to get checksums') + return + try: + with open(sha_file, 'r') as f: + expected_hash = f.read().split()[0].strip() + actual_hash = _sha256_file(archive_path) + if actual_hash == expected_hash: + add('Checksum (SHA256)', 'pass', f'Hash verified — {actual_hash[:20]}…') + else: + add('Checksum (SHA256)', 'fail', + f'MISMATCH — expected {expected_hash[:20]}… got {actual_hash[:20]}…') + except Exception as e: + add('Checksum (SHA256)', 'warn', f'Could not verify: {e}') + +def _check_archive_integrity(archive_path, add): try: result = subprocess.run( ['gzip', '--test', archive_path], @@ -198,91 +183,107 @@ def audit_backup(backup_file, source='local'): add('Archive Integrity', 'fail', f'gzip test failed: {(result.stderr or result.stdout)[:200]}') except FileNotFoundError: - try: - import gzip - with gzip.open(archive_path, 'rb') as f: - f.read(1024 * 1024) - add('Archive Integrity', 'pass', 'gzip header valid') - except Exception as e: - add('Archive Integrity', 'fail', f'Archive appears corrupt: {e}') + _check_archive_integrity_fallback(archive_path, add) except subprocess.TimeoutExpired: add('Archive Integrity', 'warn', 'Integrity check timed out — file is large, probably OK') except Exception as e: add('Archive Integrity', 'warn', f'Could not test: {e}') - members = [] + +def _check_archive_integrity_fallback(archive_path, add): + try: + import gzip + with gzip.open(archive_path, 'rb') as f: + f.read(1024 * 1024) + add('Archive Integrity', 'pass', 'gzip header valid') + except Exception as e: + add('Archive Integrity', 'fail', f'Archive appears corrupt: {e}') + + +def _list_archive_members(archive_path): try: with tarfile.open(archive_path, 'r:gz') as tf: - members = tf.getnames() + return tf.getnames() except Exception: - pass + return [] - if members: - has_volumes = any('volumes/' in m for m in members) - has_info = any('backup-info.txt' in m for m in members) - has_compose = any('compose-files/' in m for m in members) - vol_count = len([m for m in members if '/volumes/' in m and m.endswith('.tar.gz')]) - issues = [] - if not has_volumes: issues.append('volumes/ missing') - if not has_info: issues.append('backup-info.txt missing') - - if not issues: - detail = f'volumes/ ✓ backup-info.txt ✓' - if has_compose: detail += ' compose-files/ ✓' - detail += f' ({vol_count} volume archives)' - add('Internal Structure', 'pass', detail) - else: - add('Internal Structure', 'fail', ' · '.join(issues)) - else: +def _check_internal_structure(members, add): + if not members: add('Internal Structure', 'warn', 'Could not inspect archive members') + return - SUSPICIOUS = [ - (r'\.\./', 'path traversal (..)'), - (r'^/', 'absolute path in archive'), - (r'/etc/passwd', '/etc/passwd reference'), - (r'/etc/shadow', '/etc/shadow reference'), - (r'\.ssh/', '.ssh directory reference'), - (r'id_rsa(?!\.pub)', 'private SSH key reference'), - (r'authorized_keys', 'authorized_keys reference'), - ] + has_volumes = any('volumes/' in m for m in members) + has_info = any('backup-info.txt' in m for m in members) + has_compose = any('compose-files/' in m for m in members) + vol_count = len([m for m in members if '/volumes/' in m and m.endswith('.tar.gz')]) + + issues = [] + if not has_volumes: + issues.append('volumes/ missing') + if not has_info: + issues.append('backup-info.txt missing') + + if issues: + add('Internal Structure', 'fail', ' · '.join(issues)) + return + + detail = 'volumes/ ✓ backup-info.txt ✓' + if has_compose: + detail += ' compose-files/ ✓' + detail += f' ({vol_count} volume archives)' + add('Internal Structure', 'pass', detail) + + +_SUSPICIOUS_PATTERNS = [ + (r'\.\./', 'path traversal (..)'), + (r'^/', 'absolute path in archive'), + (r'/etc/passwd', '/etc/passwd reference'), + (r'/etc/shadow', '/etc/shadow reference'), + (r'\.ssh/', '.ssh directory reference'), + (r'id_rsa(?!\.pub)', 'private SSH key reference'), + (r'authorized_keys', 'authorized_keys reference'), +] + + +def _check_security_scan(members, add): found_suspicious = [] for m in members: - for pat, label in SUSPICIOUS: + for pat, label in _SUSPICIOUS_PATTERNS: if re.search(pat, m): found_suspicious.append(f'{m} ({label})') break if found_suspicious: - add('Security Scan', 'fail', - f'Suspicious entries found: {found_suspicious[:3]}') + add('Security Scan', 'fail', f'Suspicious entries found: {found_suspicious[:3]}') else: add('Security Scan', 'pass', 'No path traversal or dangerous entries detected', more=[ 'Member paths are checked for .. segments, absolute roots, and sensitive paths ' '(e.g. .ssh, /etc/shadow).', ]) - SCRIPT_EXTENSIONS = ('.sh', '.py', '.pl', '.rb', '.bash', '.zsh') - SAFE_PREFIXES = ( - 'compose-files/', - 'volumes/', - 'container-configs/', - 'configs/', - ) + +_SCRIPT_EXTENSIONS = ('.sh', '.py', '.pl', '.rb', '.bash', '.zsh') +_SAFE_PREFIXES = ('compose-files/', 'volumes/', 'container-configs/', 'configs/') + + +def _is_unexpected_executable_script(member): + name = member.name + if any(name.startswith(p) or f'/{p}' in name for p in _SAFE_PREFIXES): + return False + name_lower = name.lower() + has_script_ext = any(name_lower.endswith(ext) for ext in _SCRIPT_EXTENSIONS) + has_exec_bit = bool(member.mode & 0o111) + return has_script_ext and has_exec_bit + + +def _check_executable_scripts(archive_path, add): suspicious_scripts = [] try: with tarfile.open(archive_path, 'r:gz') as tf: for member in tf.getmembers(): - if not member.isfile(): - continue - name = member.name - if any(name.startswith(p) or f'/{p}' in name for p in SAFE_PREFIXES): - continue - name_lower = name.lower() - has_script_ext = any(name_lower.endswith(ext) for ext in SCRIPT_EXTENSIONS) - has_exec_bit = bool(member.mode & 0o111) - if has_script_ext and has_exec_bit: - suspicious_scripts.append(os.path.basename(name)) + if member.isfile() and _is_unexpected_executable_script(member): + suspicious_scripts.append(os.path.basename(member.name)) except Exception: pass @@ -292,6 +293,8 @@ def audit_backup(backup_file, source='local'): else: add('Executable Scripts', 'pass', 'No unexpected executable scripts found') + +def _check_volume_count(members, add): vol_archives = [m for m in members if 'volumes/' in m and m.endswith('.tar.gz')] v = len(vol_archives) if v == 0: @@ -301,48 +304,89 @@ def audit_backup(backup_file, source='local'): else: add('Volume Count', 'pass', f'{v} volume archives present') + +def _score_checks(checks): weights = {'pass': 10, 'warn': 5, 'fail': 0} - total = len(checks) * 10 + total = len(checks) * 10 earned = sum(weights.get(c['status'], 0) for c in checks) - score = int((earned / total) * 100) if total > 0 else 0 + return int((earned / total) * 100) if total > 0 else 0 - has_fails = any(c['status'] == 'fail' for c in checks) - ok = not has_fails and score >= 60 +def _summary_for_score(score): if score >= 90: - summary = 'Backup looks healthy and is safe to restore.' - elif score >= 70: - summary = 'Minor warnings — likely safe, but review before restoring.' - elif score >= 40: - summary = 'Significant issues detected — restore with caution.' - else: - summary = 'Multiple checks failed — do NOT restore without manual inspection.' + return 'Backup looks healthy and is safe to restore.' + if score >= 70: + return 'Minor warnings — likely safe, but review before restoring.' + if score >= 40: + return 'Significant issues detected — restore with caution.' + return 'Multiple checks failed — do NOT restore without manual inspection.' - has_warns = any(c['status'] == 'warn' for c in checks) + +def _health_tier_for(score, has_fails, has_warns): if has_fails: - health_tier = 'critical' - health_label = 'Unhealthy' - elif score == 100: - health_tier = 'excellent' - health_label = '100% healthy' - elif score >= 90: - health_tier = 'good' - health_label = 'Healthy' if not has_warns else 'Healthy (with notes)' - elif score >= 70: - health_tier = 'fair' - health_label = 'Mostly healthy' - elif score >= 40: - health_tier = 'poor' - health_label = 'At risk' - else: - health_tier = 'critical' - health_label = 'Unhealthy' + return 'critical', 'Unhealthy' + if score == 100: + return 'excellent', '100% healthy' + if score >= 90: + return 'good', ('Healthy' if not has_warns else 'Healthy (with notes)') + if score >= 70: + return 'fair', 'Mostly healthy' + if score >= 40: + return 'poor', 'At risk' + return 'critical', 'Unhealthy' + + +def _missing_result(backup_file, checks, detail=None): + return { + 'ok': False, 'score': 0, 'checks': checks, + 'backup_file': backup_file, + 'file_size_bytes': None, + 'file_size_display': None, + 'health_tier': 'critical', + 'health_label': 'Unhealthy', + 'summary': detail or 'Cannot access backup file from this host.', + } + + +def audit_backup(backup_file, source='local'): + checks = [] + + def add(name, status, detail='', more=None): + entry = {'name': name, 'status': status, 'detail': detail} + if more: + entry['more'] = more + checks.append(entry) + + archive_path = _resolve_archive_path(backup_file, source, add) + if archive_path is None: + return _missing_result(backup_file, checks, 'Cannot access backup file from this host.') + + if not os.path.exists(archive_path): + add('File Exists', 'fail', f'Not found: {archive_path}') + return _missing_result(backup_file, checks, 'Backup file does not exist on disk.') + add('File Exists', 'pass', archive_path) + + size_bytes, size_human = _check_file_size(archive_path, add) + _check_checksum(archive_path, add) + _check_archive_integrity(archive_path, add) + + members = _list_archive_members(archive_path) + _check_internal_structure(members, add) + _check_security_scan(members, add) + _check_executable_scripts(archive_path, add) + _check_volume_count(members, add) + + score = _score_checks(checks) + has_fails = any(c['status'] == 'fail' for c in checks) + has_warns = any(c['status'] == 'warn' for c in checks) + ok = not has_fails and score >= 60 + health_tier, health_label = _health_tier_for(score, has_fails, has_warns) return { 'ok': ok, 'score': score, 'checks': checks, - 'summary': summary, + 'summary': _summary_for_score(score), 'backup_file': backup_file, 'file_size_bytes': size_bytes, 'file_size_display': size_human, @@ -617,7 +661,7 @@ def get_all_stats(): def get_system_info(): """Get system information using psutil for reliable metrics inside container""" - + info = { 'cpu_pct': '0', 'memory': 'N/A', @@ -629,18 +673,18 @@ def get_system_info(): 'docker_v': 'N/A', 'hostname': socket.gethostname() } - + try: # CPU usage - psutil handles /proc access properly info['cpu_pct'] = str(psutil.cpu_percent(interval=0.5)) - + # Memory usage mem = psutil.virtual_memory() mem_used_gb = mem.used / (1024**3) mem_total_gb = mem.total / (1024**3) info['memory'] = f"{mem_used_gb:.1f}G/{mem_total_gb:.1f}G" info['mem_pct'] = str(int(mem.percent)) - + # Load average try: load = psutil.getloadavg() @@ -650,7 +694,7 @@ def get_system_info(): with open('/proc/loadavg', 'r') as f: load = f.read().split()[:3] info['load'] = ' '.join(load) - + # Disk usage try: disk = psutil.disk_usage('/') @@ -660,7 +704,7 @@ def get_system_info(): info['disk_pct'] = str(int(disk.percent)) except Exception as e: print(f"Disk read error: {e}") - + # Uptime try: boot_time = psutil.boot_time() @@ -671,19 +715,19 @@ def get_system_info(): info['uptime'] = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m" except Exception as e: print(f"Uptime read error: {e}") - + # Docker version try: docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True) info['docker_v'] = docker_v.split()[-1].strip(',') except Exception as e: print(f"Docker version error: {e}") - + except Exception as e: print(f"System info error: {e}") # Fallback to /proc method if psutil fails info = _get_system_info_fallback() - + return info @@ -700,7 +744,7 @@ def _get_system_info_fallback(): 'docker_v': 'N/A', 'hostname': socket.gethostname() } - + try: # CPU calculation with delta def _read_cpu(): @@ -709,17 +753,17 @@ def _get_system_info_fallback(): idle = int(fields[4]) total = sum(int(x) for x in fields[1:]) return idle, total - + idle1, total1 = _read_cpu() time.sleep(0.5) idle2, total2 = _read_cpu() - + idle_delta = idle2 - idle1 total_delta = total2 - total1 if total_delta > 0: cpu_pct = round(100 * (1 - idle_delta / total_delta), 1) info['cpu_pct'] = str(cpu_pct) - + # Memory mem = {} with open('/proc/meminfo', 'r') as f: @@ -727,18 +771,18 @@ def _get_system_info_fallback(): parts = line.split(':') if len(parts) == 2: mem[parts[0].strip()] = int(parts[1].strip().split()[0]) - + if 'MemTotal' in mem and 'MemAvailable' in mem: mem_total_mb = mem['MemTotal'] // 1024 mem_used_mb = (mem['MemTotal'] - mem['MemAvailable']) // 1024 info['memory'] = f"{mem_used_mb}M/{mem_total_mb}M" info['mem_pct'] = str(int(100 * mem_used_mb / mem_total_mb)) - + # Load with open('/proc/loadavg', 'r') as f: load = f.read().split()[:3] info['load'] = ' '.join(load) - + # Uptime with open('/proc/uptime', 'r') as f: secs = int(float(f.read().split()[0])) @@ -746,7 +790,7 @@ def _get_system_info_fallback(): hours = (secs % 86400) // 3600 minutes = (secs % 3600) // 60 info['uptime'] = f"{days}d {hours}h {minutes}m" if days > 0 else f"{hours}h {minutes}m" - + # Disk try: disk = psutil.disk_usage('/') @@ -754,20 +798,20 @@ def _get_system_info_fallback(): disk_total_gb = disk.total / (1024**3) info['disk'] = f"{disk_used_gb:.0f}G/{disk_total_gb:.0f}G" info['disk_pct'] = str(int(disk.percent)) - except: + except Exception: import shutil disk = shutil.disk_usage('/') info['disk'] = f"{disk.used // (1024**3)}G/{disk.total // (1024**3)}G" info['disk_pct'] = str(int(100 * disk.used / disk.total)) - + # Docker version try: docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True) info['docker_v'] = docker_v.split()[-1].strip(',') - except: + except Exception: pass - + except Exception as e: print(f"Fallback system info error: {e}") - + return info \ No newline at end of file