Compare commits
3 Commits
b43c87de73
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
| 04bbbb192b | |||
| 32feb9d4eb | |||
| ae267a6d9b |
@@ -99,16 +99,18 @@ def get_vm_backups():
|
|||||||
# ────────────────────────────────────────────────────────────────
|
# ────────────────────────────────────────────────────────────────
|
||||||
# BACKUP HEALTH AUDIT
|
# BACKUP HEALTH AUDIT
|
||||||
# ────────────────────────────────────────────────────────────────
|
# ────────────────────────────────────────────────────────────────
|
||||||
|
#
|
||||||
|
# audit_backup() used to be one large function (cognitive complexity 18).
|
||||||
|
# It's now an orchestrator that calls one small, single-purpose helper per
|
||||||
|
# check. Each helper takes the shared `add()` callback and any state it
|
||||||
|
# needs, and is responsible for exactly one check — easy to read, easy to
|
||||||
|
# test, easy to extend without pushing complexity back up.
|
||||||
|
|
||||||
def audit_backup(backup_file, source='local'):
|
def _resolve_archive_path(backup_file, source, add):
|
||||||
checks = []
|
"""Figure out the local path to the archive, pulling it from the main
|
||||||
|
server first if needed. Returns the path, or None if it could not be
|
||||||
def add(name, status, detail='', more=None):
|
resolved (in which case an error result dict has already been added
|
||||||
entry = {'name': name, 'status': status, 'detail': detail}
|
via `add` and the caller should bail out)."""
|
||||||
if more:
|
|
||||||
entry['more'] = more
|
|
||||||
checks.append(entry)
|
|
||||||
|
|
||||||
if source == 'local':
|
if source == 'local':
|
||||||
archive_path = f"/root/backups/{backup_file}"
|
archive_path = f"/root/backups/{backup_file}"
|
||||||
else:
|
else:
|
||||||
@@ -123,34 +125,16 @@ def audit_backup(backup_file, source='local'):
|
|||||||
f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}:/root/backups/{backup_file} "
|
f"{MAIN_SERVER_USER}@{MAIN_SERVER_IP}:/root/backups/{backup_file} "
|
||||||
f"{tmp_path}"
|
f"{tmp_path}"
|
||||||
)
|
)
|
||||||
out, err = _run(pull_cmd, timeout=120)
|
_out, err = _run(pull_cmd, timeout=120)
|
||||||
if not os.path.exists(tmp_path):
|
if not os.path.exists(tmp_path):
|
||||||
return {
|
add('File Access', 'fail', f'Could not pull from main server: {err}')
|
||||||
'ok': False, 'score': 0,
|
return None
|
||||||
'backup_file': backup_file,
|
|
||||||
'file_size_bytes': None,
|
|
||||||
'file_size_display': None,
|
|
||||||
'health_tier': 'critical',
|
|
||||||
'health_label': 'Unhealthy',
|
|
||||||
'checks': [{'name': 'File Access', 'status': 'fail',
|
|
||||||
'detail': f'Could not pull from main server: {err}'}],
|
|
||||||
'summary': 'Cannot access backup file from this host.'
|
|
||||||
}
|
|
||||||
archive_path = tmp_path
|
archive_path = tmp_path
|
||||||
|
|
||||||
if not os.path.exists(archive_path):
|
return archive_path
|
||||||
add('File Exists', 'fail', f'Not found: {archive_path}')
|
|
||||||
return {
|
|
||||||
'ok': False, 'score': 0, 'checks': checks,
|
|
||||||
'backup_file': backup_file,
|
|
||||||
'file_size_bytes': None,
|
|
||||||
'file_size_display': None,
|
|
||||||
'health_tier': 'critical',
|
|
||||||
'health_label': 'Unhealthy',
|
|
||||||
'summary': 'Backup file does not exist on disk.',
|
|
||||||
}
|
|
||||||
add('File Exists', 'pass', archive_path)
|
|
||||||
|
|
||||||
|
|
||||||
|
def _check_file_size(archive_path, add):
|
||||||
size_bytes = os.path.getsize(archive_path)
|
size_bytes = os.path.getsize(archive_path)
|
||||||
size_mb = size_bytes / (1024 * 1024)
|
size_mb = size_bytes / (1024 * 1024)
|
||||||
size_human = _human_bytes(size_bytes)
|
size_human = _human_bytes(size_bytes)
|
||||||
@@ -159,17 +143,19 @@ def audit_backup(backup_file, source='local'):
|
|||||||
'We flag archives under 1 MB as corrupt and under ~50 MB as unusually small for a full stack backup.',
|
'We flag archives under 1 MB as corrupt and under ~50 MB as unusually small for a full stack backup.',
|
||||||
]
|
]
|
||||||
if size_bytes < 1024 * 1024:
|
if size_bytes < 1024 * 1024:
|
||||||
add('File Size', 'fail',
|
add('File Size', 'fail', f'{size_human} — suspiciously tiny, likely corrupt', more=size_more)
|
||||||
f'{size_human} — suspiciously tiny, likely corrupt', more=size_more)
|
|
||||||
elif size_mb < 50:
|
elif size_mb < 50:
|
||||||
add('File Size', 'warn',
|
add('File Size', 'warn', f'{size_human} — smaller than expected (typical full backup > 50 MB)', more=size_more)
|
||||||
f'{size_human} — smaller than expected (typical full backup > 50 MB)', more=size_more)
|
|
||||||
else:
|
else:
|
||||||
add('File Size', 'pass',
|
add('File Size', 'pass', f'{size_human} — within expected range', more=size_more)
|
||||||
f'{size_human} — within expected range', more=size_more)
|
return size_bytes, size_human
|
||||||
|
|
||||||
|
|
||||||
|
def _check_checksum(archive_path, add):
|
||||||
sha_file = archive_path + '.sha256'
|
sha_file = archive_path + '.sha256'
|
||||||
if os.path.exists(sha_file):
|
if not os.path.exists(sha_file):
|
||||||
|
add('Checksum (SHA256)', 'warn', 'No .sha256 sidecar found — run a new backup to get checksums')
|
||||||
|
return
|
||||||
try:
|
try:
|
||||||
with open(sha_file, 'r') as f:
|
with open(sha_file, 'r') as f:
|
||||||
expected_hash = f.read().split()[0].strip()
|
expected_hash = f.read().split()[0].strip()
|
||||||
@@ -181,10 +167,9 @@ def audit_backup(backup_file, source='local'):
|
|||||||
f'MISMATCH — expected {expected_hash[:20]}… got {actual_hash[:20]}…')
|
f'MISMATCH — expected {expected_hash[:20]}… got {actual_hash[:20]}…')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
add('Checksum (SHA256)', 'warn', f'Could not verify: {e}')
|
add('Checksum (SHA256)', 'warn', f'Could not verify: {e}')
|
||||||
else:
|
|
||||||
add('Checksum (SHA256)', 'warn',
|
|
||||||
'No .sha256 sidecar found — run a new backup to get checksums')
|
|
||||||
|
|
||||||
|
|
||||||
|
def _check_archive_integrity(archive_path, add):
|
||||||
try:
|
try:
|
||||||
result = subprocess.run(
|
result = subprocess.run(
|
||||||
['gzip', '--test', archive_path],
|
['gzip', '--test', archive_path],
|
||||||
@@ -198,6 +183,14 @@ def audit_backup(backup_file, source='local'):
|
|||||||
add('Archive Integrity', 'fail',
|
add('Archive Integrity', 'fail',
|
||||||
f'gzip test failed: {(result.stderr or result.stdout)[:200]}')
|
f'gzip test failed: {(result.stderr or result.stdout)[:200]}')
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
|
_check_archive_integrity_fallback(archive_path, add)
|
||||||
|
except subprocess.TimeoutExpired:
|
||||||
|
add('Archive Integrity', 'warn', 'Integrity check timed out — file is large, probably OK')
|
||||||
|
except Exception as e:
|
||||||
|
add('Archive Integrity', 'warn', f'Could not test: {e}')
|
||||||
|
|
||||||
|
|
||||||
|
def _check_archive_integrity_fallback(archive_path, add):
|
||||||
try:
|
try:
|
||||||
import gzip
|
import gzip
|
||||||
with gzip.open(archive_path, 'rb') as f:
|
with gzip.open(archive_path, 'rb') as f:
|
||||||
@@ -205,39 +198,44 @@ def audit_backup(backup_file, source='local'):
|
|||||||
add('Archive Integrity', 'pass', 'gzip header valid')
|
add('Archive Integrity', 'pass', 'gzip header valid')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
add('Archive Integrity', 'fail', f'Archive appears corrupt: {e}')
|
add('Archive Integrity', 'fail', f'Archive appears corrupt: {e}')
|
||||||
except subprocess.TimeoutExpired:
|
|
||||||
add('Archive Integrity', 'warn', 'Integrity check timed out — file is large, probably OK')
|
|
||||||
except Exception as e:
|
|
||||||
add('Archive Integrity', 'warn', f'Could not test: {e}')
|
|
||||||
|
|
||||||
members = []
|
|
||||||
|
def _list_archive_members(archive_path):
|
||||||
try:
|
try:
|
||||||
with tarfile.open(archive_path, 'r:gz') as tf:
|
with tarfile.open(archive_path, 'r:gz') as tf:
|
||||||
members = tf.getnames()
|
return tf.getnames()
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _check_internal_structure(members, add):
|
||||||
|
if not members:
|
||||||
|
add('Internal Structure', 'warn', 'Could not inspect archive members')
|
||||||
|
return
|
||||||
|
|
||||||
if members:
|
|
||||||
has_volumes = any('volumes/' in m for m in members)
|
has_volumes = any('volumes/' in m for m in members)
|
||||||
has_info = any('backup-info.txt' in m for m in members)
|
has_info = any('backup-info.txt' in m for m in members)
|
||||||
has_compose = any('compose-files/' in m for m in members)
|
has_compose = any('compose-files/' in m for m in members)
|
||||||
vol_count = len([m for m in members if '/volumes/' in m and m.endswith('.tar.gz')])
|
vol_count = len([m for m in members if '/volumes/' in m and m.endswith('.tar.gz')])
|
||||||
|
|
||||||
issues = []
|
issues = []
|
||||||
if not has_volumes: issues.append('volumes/ missing')
|
if not has_volumes:
|
||||||
if not has_info: issues.append('backup-info.txt missing')
|
issues.append('volumes/ missing')
|
||||||
|
if not has_info:
|
||||||
|
issues.append('backup-info.txt missing')
|
||||||
|
|
||||||
if not issues:
|
if issues:
|
||||||
detail = f'volumes/ ✓ backup-info.txt ✓'
|
add('Internal Structure', 'fail', ' · '.join(issues))
|
||||||
if has_compose: detail += ' compose-files/ ✓'
|
return
|
||||||
|
|
||||||
|
detail = 'volumes/ ✓ backup-info.txt ✓'
|
||||||
|
if has_compose:
|
||||||
|
detail += ' compose-files/ ✓'
|
||||||
detail += f' ({vol_count} volume archives)'
|
detail += f' ({vol_count} volume archives)'
|
||||||
add('Internal Structure', 'pass', detail)
|
add('Internal Structure', 'pass', detail)
|
||||||
else:
|
|
||||||
add('Internal Structure', 'fail', ' · '.join(issues))
|
|
||||||
else:
|
|
||||||
add('Internal Structure', 'warn', 'Could not inspect archive members')
|
|
||||||
|
|
||||||
SUSPICIOUS = [
|
|
||||||
|
_SUSPICIOUS_PATTERNS = [
|
||||||
(r'\.\./', 'path traversal (..)'),
|
(r'\.\./', 'path traversal (..)'),
|
||||||
(r'^/', 'absolute path in archive'),
|
(r'^/', 'absolute path in archive'),
|
||||||
(r'/etc/passwd', '/etc/passwd reference'),
|
(r'/etc/passwd', '/etc/passwd reference'),
|
||||||
@@ -246,43 +244,46 @@ def audit_backup(backup_file, source='local'):
|
|||||||
(r'id_rsa(?!\.pub)', 'private SSH key reference'),
|
(r'id_rsa(?!\.pub)', 'private SSH key reference'),
|
||||||
(r'authorized_keys', 'authorized_keys reference'),
|
(r'authorized_keys', 'authorized_keys reference'),
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|
||||||
|
def _check_security_scan(members, add):
|
||||||
found_suspicious = []
|
found_suspicious = []
|
||||||
for m in members:
|
for m in members:
|
||||||
for pat, label in SUSPICIOUS:
|
for pat, label in _SUSPICIOUS_PATTERNS:
|
||||||
if re.search(pat, m):
|
if re.search(pat, m):
|
||||||
found_suspicious.append(f'{m} ({label})')
|
found_suspicious.append(f'{m} ({label})')
|
||||||
break
|
break
|
||||||
|
|
||||||
if found_suspicious:
|
if found_suspicious:
|
||||||
add('Security Scan', 'fail',
|
add('Security Scan', 'fail', f'Suspicious entries found: {found_suspicious[:3]}')
|
||||||
f'Suspicious entries found: {found_suspicious[:3]}')
|
|
||||||
else:
|
else:
|
||||||
add('Security Scan', 'pass', 'No path traversal or dangerous entries detected', more=[
|
add('Security Scan', 'pass', 'No path traversal or dangerous entries detected', more=[
|
||||||
'Member paths are checked for .. segments, absolute roots, and sensitive paths '
|
'Member paths are checked for .. segments, absolute roots, and sensitive paths '
|
||||||
'(e.g. .ssh, /etc/shadow).',
|
'(e.g. .ssh, /etc/shadow).',
|
||||||
])
|
])
|
||||||
|
|
||||||
SCRIPT_EXTENSIONS = ('.sh', '.py', '.pl', '.rb', '.bash', '.zsh')
|
|
||||||
SAFE_PREFIXES = (
|
_SCRIPT_EXTENSIONS = ('.sh', '.py', '.pl', '.rb', '.bash', '.zsh')
|
||||||
'compose-files/',
|
_SAFE_PREFIXES = ('compose-files/', 'volumes/', 'container-configs/', 'configs/')
|
||||||
'volumes/',
|
|
||||||
'container-configs/',
|
|
||||||
'configs/',
|
def _is_unexpected_executable_script(member):
|
||||||
)
|
name = member.name
|
||||||
|
if any(name.startswith(p) or f'/{p}' in name for p in _SAFE_PREFIXES):
|
||||||
|
return False
|
||||||
|
name_lower = name.lower()
|
||||||
|
has_script_ext = any(name_lower.endswith(ext) for ext in _SCRIPT_EXTENSIONS)
|
||||||
|
has_exec_bit = bool(member.mode & 0o111)
|
||||||
|
return has_script_ext and has_exec_bit
|
||||||
|
|
||||||
|
|
||||||
|
def _check_executable_scripts(archive_path, add):
|
||||||
suspicious_scripts = []
|
suspicious_scripts = []
|
||||||
try:
|
try:
|
||||||
with tarfile.open(archive_path, 'r:gz') as tf:
|
with tarfile.open(archive_path, 'r:gz') as tf:
|
||||||
for member in tf.getmembers():
|
for member in tf.getmembers():
|
||||||
if not member.isfile():
|
if member.isfile() and _is_unexpected_executable_script(member):
|
||||||
continue
|
suspicious_scripts.append(os.path.basename(member.name))
|
||||||
name = member.name
|
|
||||||
if any(name.startswith(p) or f'/{p}' in name for p in SAFE_PREFIXES):
|
|
||||||
continue
|
|
||||||
name_lower = name.lower()
|
|
||||||
has_script_ext = any(name_lower.endswith(ext) for ext in SCRIPT_EXTENSIONS)
|
|
||||||
has_exec_bit = bool(member.mode & 0o111)
|
|
||||||
if has_script_ext and has_exec_bit:
|
|
||||||
suspicious_scripts.append(os.path.basename(name))
|
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
@@ -292,6 +293,8 @@ def audit_backup(backup_file, source='local'):
|
|||||||
else:
|
else:
|
||||||
add('Executable Scripts', 'pass', 'No unexpected executable scripts found')
|
add('Executable Scripts', 'pass', 'No unexpected executable scripts found')
|
||||||
|
|
||||||
|
|
||||||
|
def _check_volume_count(members, add):
|
||||||
vol_archives = [m for m in members if 'volumes/' in m and m.endswith('.tar.gz')]
|
vol_archives = [m for m in members if 'volumes/' in m and m.endswith('.tar.gz')]
|
||||||
v = len(vol_archives)
|
v = len(vol_archives)
|
||||||
if v == 0:
|
if v == 0:
|
||||||
@@ -301,48 +304,89 @@ def audit_backup(backup_file, source='local'):
|
|||||||
else:
|
else:
|
||||||
add('Volume Count', 'pass', f'{v} volume archives present')
|
add('Volume Count', 'pass', f'{v} volume archives present')
|
||||||
|
|
||||||
|
|
||||||
|
def _score_checks(checks):
|
||||||
weights = {'pass': 10, 'warn': 5, 'fail': 0}
|
weights = {'pass': 10, 'warn': 5, 'fail': 0}
|
||||||
total = len(checks) * 10
|
total = len(checks) * 10
|
||||||
earned = sum(weights.get(c['status'], 0) for c in checks)
|
earned = sum(weights.get(c['status'], 0) for c in checks)
|
||||||
score = int((earned / total) * 100) if total > 0 else 0
|
return int((earned / total) * 100) if total > 0 else 0
|
||||||
|
|
||||||
has_fails = any(c['status'] == 'fail' for c in checks)
|
|
||||||
ok = not has_fails and score >= 60
|
|
||||||
|
|
||||||
|
def _summary_for_score(score):
|
||||||
if score >= 90:
|
if score >= 90:
|
||||||
summary = 'Backup looks healthy and is safe to restore.'
|
return 'Backup looks healthy and is safe to restore.'
|
||||||
elif score >= 70:
|
if score >= 70:
|
||||||
summary = 'Minor warnings — likely safe, but review before restoring.'
|
return 'Minor warnings — likely safe, but review before restoring.'
|
||||||
elif score >= 40:
|
if score >= 40:
|
||||||
summary = 'Significant issues detected — restore with caution.'
|
return 'Significant issues detected — restore with caution.'
|
||||||
else:
|
return 'Multiple checks failed — do NOT restore without manual inspection.'
|
||||||
summary = 'Multiple checks failed — do NOT restore without manual inspection.'
|
|
||||||
|
|
||||||
has_warns = any(c['status'] == 'warn' for c in checks)
|
|
||||||
|
def _health_tier_for(score, has_fails, has_warns):
|
||||||
if has_fails:
|
if has_fails:
|
||||||
health_tier = 'critical'
|
return 'critical', 'Unhealthy'
|
||||||
health_label = 'Unhealthy'
|
if score == 100:
|
||||||
elif score == 100:
|
return 'excellent', '100% healthy'
|
||||||
health_tier = 'excellent'
|
if score >= 90:
|
||||||
health_label = '100% healthy'
|
return 'good', ('Healthy' if not has_warns else 'Healthy (with notes)')
|
||||||
elif score >= 90:
|
if score >= 70:
|
||||||
health_tier = 'good'
|
return 'fair', 'Mostly healthy'
|
||||||
health_label = 'Healthy' if not has_warns else 'Healthy (with notes)'
|
if score >= 40:
|
||||||
elif score >= 70:
|
return 'poor', 'At risk'
|
||||||
health_tier = 'fair'
|
return 'critical', 'Unhealthy'
|
||||||
health_label = 'Mostly healthy'
|
|
||||||
elif score >= 40:
|
|
||||||
health_tier = 'poor'
|
def _missing_result(backup_file, checks, detail=None):
|
||||||
health_label = 'At risk'
|
return {
|
||||||
else:
|
'ok': False, 'score': 0, 'checks': checks,
|
||||||
health_tier = 'critical'
|
'backup_file': backup_file,
|
||||||
health_label = 'Unhealthy'
|
'file_size_bytes': None,
|
||||||
|
'file_size_display': None,
|
||||||
|
'health_tier': 'critical',
|
||||||
|
'health_label': 'Unhealthy',
|
||||||
|
'summary': detail or 'Cannot access backup file from this host.',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def audit_backup(backup_file, source='local'):
|
||||||
|
checks = []
|
||||||
|
|
||||||
|
def add(name, status, detail='', more=None):
|
||||||
|
entry = {'name': name, 'status': status, 'detail': detail}
|
||||||
|
if more:
|
||||||
|
entry['more'] = more
|
||||||
|
checks.append(entry)
|
||||||
|
|
||||||
|
archive_path = _resolve_archive_path(backup_file, source, add)
|
||||||
|
if archive_path is None:
|
||||||
|
return _missing_result(backup_file, checks, 'Cannot access backup file from this host.')
|
||||||
|
|
||||||
|
if not os.path.exists(archive_path):
|
||||||
|
add('File Exists', 'fail', f'Not found: {archive_path}')
|
||||||
|
return _missing_result(backup_file, checks, 'Backup file does not exist on disk.')
|
||||||
|
add('File Exists', 'pass', archive_path)
|
||||||
|
|
||||||
|
size_bytes, size_human = _check_file_size(archive_path, add)
|
||||||
|
_check_checksum(archive_path, add)
|
||||||
|
_check_archive_integrity(archive_path, add)
|
||||||
|
|
||||||
|
members = _list_archive_members(archive_path)
|
||||||
|
_check_internal_structure(members, add)
|
||||||
|
_check_security_scan(members, add)
|
||||||
|
_check_executable_scripts(archive_path, add)
|
||||||
|
_check_volume_count(members, add)
|
||||||
|
|
||||||
|
score = _score_checks(checks)
|
||||||
|
has_fails = any(c['status'] == 'fail' for c in checks)
|
||||||
|
has_warns = any(c['status'] == 'warn' for c in checks)
|
||||||
|
ok = not has_fails and score >= 60
|
||||||
|
health_tier, health_label = _health_tier_for(score, has_fails, has_warns)
|
||||||
|
|
||||||
return {
|
return {
|
||||||
'ok': ok,
|
'ok': ok,
|
||||||
'score': score,
|
'score': score,
|
||||||
'checks': checks,
|
'checks': checks,
|
||||||
'summary': summary,
|
'summary': _summary_for_score(score),
|
||||||
'backup_file': backup_file,
|
'backup_file': backup_file,
|
||||||
'file_size_bytes': size_bytes,
|
'file_size_bytes': size_bytes,
|
||||||
'file_size_display': size_human,
|
'file_size_display': size_human,
|
||||||
@@ -754,7 +798,7 @@ def _get_system_info_fallback():
|
|||||||
disk_total_gb = disk.total / (1024**3)
|
disk_total_gb = disk.total / (1024**3)
|
||||||
info['disk'] = f"{disk_used_gb:.0f}G/{disk_total_gb:.0f}G"
|
info['disk'] = f"{disk_used_gb:.0f}G/{disk_total_gb:.0f}G"
|
||||||
info['disk_pct'] = str(int(disk.percent))
|
info['disk_pct'] = str(int(disk.percent))
|
||||||
except:
|
except Exception:
|
||||||
import shutil
|
import shutil
|
||||||
disk = shutil.disk_usage('/')
|
disk = shutil.disk_usage('/')
|
||||||
info['disk'] = f"{disk.used // (1024**3)}G/{disk.total // (1024**3)}G"
|
info['disk'] = f"{disk.used // (1024**3)}G/{disk.total // (1024**3)}G"
|
||||||
@@ -764,7 +808,7 @@ def _get_system_info_fallback():
|
|||||||
try:
|
try:
|
||||||
docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True)
|
docker_v = subprocess.check_output(['docker', '--version'], stderr=subprocess.DEVNULL, text=True)
|
||||||
info['docker_v'] = docker_v.split()[-1].strip(',')
|
info['docker_v'] = docker_v.split()[-1].strip(',')
|
||||||
except:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -158,6 +158,55 @@ container_is_healthy() {
|
|||||||
return 1
|
return 1
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# NEW: Detect a crash-looping Postgres container and
|
||||||
|
# attempt automatic recovery via pg_resetwal if the
|
||||||
|
# logs show WAL/checkpoint corruption. This is a safety
|
||||||
|
# net — the real fix is that backups now stop DB
|
||||||
|
# containers before tar'ing their volumes — but this
|
||||||
|
# protects against older/already-corrupt backups too.
|
||||||
|
# --------------------------------------------------
|
||||||
|
repair_postgres_if_crashlooping() {
|
||||||
|
local ctr="$1"
|
||||||
|
local vol="$2"
|
||||||
|
|
||||||
|
[ -z "$ctr" ] || [ -z "$vol" ] && return 0
|
||||||
|
docker ps -a --format '{{.Names}}' | grep -qx "$ctr" || return 0
|
||||||
|
|
||||||
|
sleep 8 # give it a moment to attempt startup
|
||||||
|
|
||||||
|
local restart_count status
|
||||||
|
restart_count=$(docker inspect --format='{{.RestartCount}}' "$ctr" 2>/dev/null || echo 0)
|
||||||
|
status=$(docker inspect --format='{{.State.Status}}' "$ctr" 2>/dev/null || echo "missing")
|
||||||
|
|
||||||
|
if [ "$status" = "restarting" ] || [ "$restart_count" -gt 2 ]; then
|
||||||
|
echo " ⚠️ $ctr is crash-looping (status=$status, restarts=$restart_count)"
|
||||||
|
|
||||||
|
local last_log
|
||||||
|
last_log=$(docker logs "$ctr" --tail 30 2>&1)
|
||||||
|
|
||||||
|
if echo "$last_log" | grep -qi "could not locate a valid checkpoint record\|invalid primary checkpoint\|unexpected pageaddr"; then
|
||||||
|
echo " 🔧 Detected WAL corruption — attempting pg_resetwal recovery on volume '$vol'..."
|
||||||
|
docker stop "$ctr" &>/dev/null || true
|
||||||
|
docker run --rm -v "${vol}:/var/lib/postgresql/data" postgres:15 \
|
||||||
|
bash -c "chown -R postgres:postgres /var/lib/postgresql/data && \
|
||||||
|
su postgres -c 'pg_resetwal -f /var/lib/postgresql/data'" \
|
||||||
|
&& echo " ✅ pg_resetwal completed" \
|
||||||
|
|| echo " ❌ pg_resetwal failed — manual intervention needed"
|
||||||
|
docker start "$ctr" &>/dev/null || true
|
||||||
|
sleep 8
|
||||||
|
status=$(docker inspect --format='{{.State.Status}}' "$ctr" 2>/dev/null || echo "missing")
|
||||||
|
if [ "$status" = "running" ]; then
|
||||||
|
echo " ✅ $ctr recovered and running (note: pg_resetwal can drop the most recent transactions — verify your data)"
|
||||||
|
else
|
||||||
|
echo " ❌ $ctr still not healthy after pg_resetwal — check 'docker logs $ctr' manually. DATA MAY BE LOST."
|
||||||
|
fi
|
||||||
|
else
|
||||||
|
echo " ❌ $ctr crash-looping for a reason other than WAL corruption — check 'docker logs $ctr' manually."
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# STEP 1: Restore Volumes
|
# STEP 1: Restore Volumes
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
@@ -259,6 +308,14 @@ declare -A APP_KEY=(
|
|||||||
["n8n"]="n8n"
|
["n8n"]="n8n"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Postgres-backed apps: map app key -> "container:volume" for the
|
||||||
|
# crash-loop auto-repair check run right after each app starts.
|
||||||
|
declare -A APP_POSTGRES=(
|
||||||
|
["n8n"]="n8n-postgres:n8n-setup_n8n-db-data"
|
||||||
|
["nextcloud"]="nextcloud-postgres:nextcloud-setup_nextcloud-db-data"
|
||||||
|
["odoo"]="odoo-clean-db-1:odoo-clean_db-data"
|
||||||
|
)
|
||||||
|
|
||||||
if [ -d "$SCRIPT_DIR/compose-files" ]; then
|
if [ -d "$SCRIPT_DIR/compose-files" ]; then
|
||||||
cd "$SCRIPT_DIR/compose-files"
|
cd "$SCRIPT_DIR/compose-files"
|
||||||
for app in Frappe Odoo Nextcloud Mautic n8n; do
|
for app in Frappe Odoo Nextcloud Mautic n8n; do
|
||||||
@@ -292,6 +349,14 @@ if [ -d "$SCRIPT_DIR/compose-files" ]; then
|
|||||||
fi
|
fi
|
||||||
docker-compose up -d 2>&1 | tail -3
|
docker-compose up -d 2>&1 | tail -3
|
||||||
cd ..
|
cd ..
|
||||||
|
|
||||||
|
# If this app has a Postgres DB, check it didn't come up crash-looping.
|
||||||
|
pg_spec="${APP_POSTGRES[$app_key]:-}"
|
||||||
|
if [ -n "$pg_spec" ]; then
|
||||||
|
pg_ctr="${pg_spec%%:*}"
|
||||||
|
pg_vol="${pg_spec##*:}"
|
||||||
|
repair_postgres_if_crashlooping "$pg_ctr" "$pg_vol"
|
||||||
|
fi
|
||||||
done
|
done
|
||||||
cd "$SCRIPT_DIR"
|
cd "$SCRIPT_DIR"
|
||||||
else
|
else
|
||||||
@@ -475,7 +540,15 @@ else
|
|||||||
docker start n8n-app 2>/dev/null && echo " ✅ Started n8n-app" || echo " ⚠️ Could not start n8n"
|
docker start n8n-app 2>/dev/null && echo " ✅ Started n8n-app" || echo " ⚠️ Could not start n8n"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
# Re-check postgres after this section's potential restart too.
|
||||||
|
repair_postgres_if_crashlooping "n8n-postgres" "n8n-setup_n8n-db-data"
|
||||||
|
|
||||||
|
if container_is_healthy n8n-postgres; then
|
||||||
echo " ✅ n8n → http://${VM_IP}:5678"
|
echo " ✅ n8n → http://${VM_IP}:5678"
|
||||||
|
else
|
||||||
|
echo " ⚠️ n8n-app is up but n8n-postgres is NOT healthy — check: docker logs n8n-postgres"
|
||||||
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
@@ -485,9 +558,19 @@ echo ""
|
|||||||
echo "========================================="
|
echo "========================================="
|
||||||
echo "✅ RESTORE COMPLETE"
|
echo "✅ RESTORE COMPLETE"
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
if app_selected nextcloud; then echo " Nextcloud → http://${VM_IP}:8082"; fi
|
|
||||||
if app_selected mautic; then echo " Mautic → http://${VM_IP}:8081/s/login (admin / Admin!Password123)"; fi
|
summarize_app() {
|
||||||
if app_selected odoo; then echo " Odoo → https://odooo.nav.ovh/web"; fi
|
local label="$1" url="$2" db_ctr="$3"
|
||||||
if app_selected n8n; then echo " n8n → http://${VM_IP}:5678"; fi
|
if [ -n "$db_ctr" ] && ! container_is_healthy "$db_ctr"; then
|
||||||
if app_selected frappe; then echo " Frappe → http://${VM_IP}:8080"; fi
|
echo " ${label} → ⚠️ DB NOT healthy ($db_ctr) — check: docker logs $db_ctr"
|
||||||
|
else
|
||||||
|
echo " ${label} → ${url}"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
if app_selected nextcloud; then summarize_app "Nextcloud" "http://${VM_IP}:8082" "nextcloud-postgres"; fi
|
||||||
|
if app_selected mautic; then summarize_app "Mautic " "http://${VM_IP}:8081/s/login (admin / Admin!Password123)" "mautic-mariadb"; fi
|
||||||
|
if app_selected odoo; then summarize_app "Odoo " "https://odooo.nav.ovh/web" "odoo-clean-db-1"; fi
|
||||||
|
if app_selected n8n; then summarize_app "n8n " "http://${VM_IP}:5678" "n8n-postgres"; fi
|
||||||
|
if app_selected frappe; then summarize_app "Frappe " "http://${VM_IP}:8080" "frappe-mariadb"; fi
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -403,7 +403,9 @@ function renderBackupList(items, id, source) {
|
|||||||
: '';
|
: '';
|
||||||
return `
|
return `
|
||||||
<div class="backup-item" id="bk-item-${source}-${b.replace(/[^a-z0-9]/gi, '_')}">
|
<div class="backup-item" id="bk-item-${source}-${b.replace(/[^a-z0-9]/gi, '_')}">
|
||||||
|
<div class="backup-item-main">
|
||||||
<span class="backup-name">${escapeHtml(b)}</span>
|
<span class="backup-name">${escapeHtml(b)}</span>
|
||||||
|
</div>
|
||||||
<div class="backup-actions">
|
<div class="backup-actions">
|
||||||
${detailsBtn}
|
${detailsBtn}
|
||||||
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditBackup('${source}','${safe}',this)"><i class="fas fa-shield-check"></i> Audit</button>
|
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditBackup('${source}','${safe}',this)"><i class="fas fa-shield-check"></i> Audit</button>
|
||||||
@@ -533,7 +535,13 @@ async function loadBackupLog() {
|
|||||||
node.innerHTML = '<div class="empty-state"><i class="fas fa-inbox"></i>No backup history yet</div>';
|
node.innerHTML = '<div class="empty-state"><i class="fas fa-inbox"></i>No backup history yet</div>';
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
node.innerHTML = d.entries.map((e) => `<div class="backup-item"><span class="backup-name">${e.name}</span><span class="card-meta">${e.status}</span></div>`).join('');
|
node.innerHTML = d.entries.map((e) => `
|
||||||
|
<div class="backup-item">
|
||||||
|
<div class="backup-item-main">
|
||||||
|
<span class="backup-name">${escapeHtml(e.name)}</span>
|
||||||
|
<span class="backup-meta">${escapeHtml(e.status)}</span>
|
||||||
|
</div>
|
||||||
|
</div>`).join('');
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
node.innerHTML = `<div class="empty-state">Error loading log: ${e}</div>`;
|
node.innerHTML = `<div class="empty-state">Error loading log: ${e}</div>`;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,7 +7,7 @@
|
|||||||
<link rel="preconnect" href="https://fonts.googleapis.com">
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
||||||
<link href="https://fonts.googleapis.com/css2?family=Syne:wght@400;500;600;700;800&family=Geist+Mono:wght@300;400;500&display=swap" rel="stylesheet">
|
<link href="https://fonts.googleapis.com/css2?family=Syne:wght@400;500;600;700;800&family=Geist+Mono:wght@300;400;500&display=swap" rel="stylesheet">
|
||||||
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/css/all.min.css">
|
<link rel="stylesheet" href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.7.2/css/all.min.css">
|
||||||
<link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}">
|
<link rel="stylesheet" href="{{ url_for('static', filename='css/style.css') }}?v=ui4">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
<div class="layout">
|
<div class="layout">
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
<!DOCTYPE html>
|
<!DOCTYPE html>
|
||||||
<html lang="en">
|
<html lang="en" data-theme="dark">
|
||||||
<head>
|
<head>
|
||||||
<meta charset="UTF-8">
|
<meta charset="UTF-8">
|
||||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||||
@@ -10,53 +10,82 @@
|
|||||||
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||||
body {
|
body {
|
||||||
font-family: 'Syne', sans-serif;
|
font-family: 'Syne', sans-serif;
|
||||||
background: #0a0b0e;
|
background: #07080c;
|
||||||
display: flex; align-items: center; justify-content: center;
|
display: flex; align-items: center; justify-content: center;
|
||||||
min-height: 100vh; color: #e8ecf4;
|
min-height: 100vh; color: #eef1f8;
|
||||||
|
-webkit-font-smoothing: antialiased;
|
||||||
}
|
}
|
||||||
body::before {
|
body::before {
|
||||||
content: ''; position: fixed; inset: 0;
|
content: ''; position: fixed; inset: 0; pointer-events: none;
|
||||||
|
background:
|
||||||
|
radial-gradient(ellipse 80% 60% at 50% -10%, rgba(79,140,255,0.12), transparent 55%),
|
||||||
|
radial-gradient(ellipse 50% 40% at 100% 80%, rgba(183,148,255,0.08), transparent 50%);
|
||||||
|
}
|
||||||
|
body::after {
|
||||||
|
content: ''; position: fixed; inset: 0; pointer-events: none;
|
||||||
background-image:
|
background-image:
|
||||||
linear-gradient(rgba(59,130,246,0.03) 1px, transparent 1px),
|
linear-gradient(rgba(79,140,255,0.025) 1px, transparent 1px),
|
||||||
linear-gradient(90deg, rgba(59,130,246,0.03) 1px, transparent 1px);
|
linear-gradient(90deg, rgba(79,140,255,0.025) 1px, transparent 1px);
|
||||||
background-size: 40px 40px; pointer-events: none;
|
background-size: 48px 48px;
|
||||||
}
|
}
|
||||||
.card {
|
.card {
|
||||||
background: #111318; border: 1px solid #1e2330;
|
background: rgba(14,16,23,0.9);
|
||||||
border-radius: 20px; padding: 44px 40px; width: 380px; position: relative;
|
backdrop-filter: blur(24px);
|
||||||
|
border: 1px solid #1a2030;
|
||||||
|
border-radius: 22px;
|
||||||
|
padding: 44px 40px;
|
||||||
|
width: 400px;
|
||||||
|
position: relative;
|
||||||
|
box-shadow: 0 24px 64px rgba(0,0,0,0.45), 0 0 0 1px rgba(255,255,255,0.03);
|
||||||
|
animation: cardIn 0.5s cubic-bezier(0.22, 1, 0.36, 1);
|
||||||
|
}
|
||||||
|
@keyframes cardIn {
|
||||||
|
from { opacity: 0; transform: translateY(16px) scale(0.98); }
|
||||||
|
to { opacity: 1; transform: translateY(0) scale(1); }
|
||||||
}
|
}
|
||||||
.card::before {
|
.card::before {
|
||||||
content: ''; position: absolute; top: 0; left: 40px; right: 40px; height: 2px;
|
content: ''; position: absolute; top: 0; left: 32px; right: 32px; height: 2px;
|
||||||
background: linear-gradient(90deg, #3b82f6, #60a5fa, #a78bfa);
|
background: linear-gradient(90deg, #4f8cff, #7eb3ff, #b794ff);
|
||||||
border-radius: 0 0 4px 4px;
|
border-radius: 0 0 4px 4px;
|
||||||
}
|
}
|
||||||
.brand { display: flex; align-items: center; gap: 12px; margin-bottom: 32px; }
|
.brand { display: flex; align-items: center; gap: 14px; margin-bottom: 36px; }
|
||||||
.brand-mark {
|
.brand-mark {
|
||||||
width: 40px; height: 40px; background: #3b82f6; border-radius: 10px;
|
width: 42px; height: 42px;
|
||||||
|
background: linear-gradient(135deg, #4f8cff, #6366f1);
|
||||||
|
border-radius: 12px;
|
||||||
display: flex; align-items: center; justify-content: center;
|
display: flex; align-items: center; justify-content: center;
|
||||||
font-family: 'Geist Mono', monospace; font-weight: 500; font-size: 13px; color: #fff;
|
font-family: 'Geist Mono', monospace; font-weight: 600; font-size: 13px; color: #fff;
|
||||||
|
box-shadow: 0 4px 16px rgba(79,140,255,0.35);
|
||||||
}
|
}
|
||||||
.brand-name { font-size: 18px; font-weight: 700; }
|
.brand-name { font-size: 19px; font-weight: 700; letter-spacing: -0.02em; }
|
||||||
.brand-sub { font-size: 11px; color: #4a5568; font-family: 'Geist Mono', monospace; letter-spacing: 0.08em; }
|
.brand-sub { font-size: 9px; color: #525d72; font-family: 'Geist Mono', monospace; letter-spacing: 0.14em; margin-top: 2px; }
|
||||||
label { display: block; font-size: 11px; font-weight: 600; color: #8892a4; letter-spacing: 0.1em; margin-bottom: 7px; }
|
label { display: block; font-size: 10px; font-weight: 700; color: #8b95aa; letter-spacing: 0.12em; margin-bottom: 8px; }
|
||||||
input[type="password"] {
|
input[type="password"] {
|
||||||
width: 100%; padding: 11px 14px; background: #181c24;
|
width: 100%; padding: 12px 16px; background: #141820;
|
||||||
border: 1px solid #262d3d; border-radius: 10px;
|
border: 1px solid #252d42; border-radius: 12px;
|
||||||
font-size: 14px; font-family: 'Geist Mono', monospace;
|
font-size: 14px; font-family: 'Geist Mono', monospace;
|
||||||
color: #e8ecf4; letter-spacing: 0.1em; margin-bottom: 20px;
|
color: #eef1f8; letter-spacing: 0.08em; margin-bottom: 22px;
|
||||||
transition: border-color 0.18s;
|
transition: border-color 0.22s, box-shadow 0.22s;
|
||||||
|
}
|
||||||
|
input[type="password"]:focus {
|
||||||
|
outline: none; border-color: #4f8cff;
|
||||||
|
box-shadow: 0 0 0 3px rgba(79,140,255,0.15);
|
||||||
}
|
}
|
||||||
input[type="password"]:focus { outline: none; border-color: #3b82f6; }
|
|
||||||
button {
|
button {
|
||||||
width: 100%; padding: 12px; background: #3b82f6; color: #fff;
|
width: 100%; padding: 13px;
|
||||||
border: none; border-radius: 10px; font-size: 14px; font-weight: 600;
|
background: linear-gradient(135deg, #4f8cff, #6366f1);
|
||||||
font-family: 'Syne', sans-serif; cursor: pointer; transition: background 0.18s;
|
color: #fff; border: none; border-radius: 12px;
|
||||||
|
font-size: 14px; font-weight: 700;
|
||||||
|
font-family: 'Syne', sans-serif; cursor: pointer;
|
||||||
|
transition: filter 0.22s, transform 0.15s, box-shadow 0.22s;
|
||||||
|
box-shadow: 0 4px 18px rgba(79,140,255,0.35);
|
||||||
}
|
}
|
||||||
button:hover { background: #2563eb; }
|
button:hover { filter: brightness(1.08); transform: translateY(-1px); box-shadow: 0 6px 24px rgba(79,140,255,0.45); }
|
||||||
|
button:active { transform: translateY(0); }
|
||||||
.error {
|
.error {
|
||||||
color: #ef4444; font-size: 12px; margin-bottom: 14px;
|
color: #f05252; font-size: 12px; margin-bottom: 16px;
|
||||||
padding: 9px 12px; background: rgba(239,68,68,0.08);
|
padding: 10px 14px; background: rgba(240,82,82,0.08);
|
||||||
border: 1px solid rgba(239,68,68,0.2); border-radius: 8px;
|
border: 1px solid rgba(240,82,82,0.22); border-radius: 10px;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
</head>
|
</head>
|
||||||
|
|||||||
@@ -1,33 +1,9 @@
|
|||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
|
|
||||||
{# ── Extra styles scoped to this page ── #}
|
{# ── Extra styles scoped to this page (modals only) ── #}
|
||||||
<style>
|
<style>
|
||||||
/* ── BACKUP ITEMS ──────────────────────────────────────────────── */
|
/* page-specific modal styles — backup layout lives in style.css */
|
||||||
.backup-item {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: space-between;
|
|
||||||
padding: 10px 14px;
|
|
||||||
border-radius: 10px;
|
|
||||||
background: var(--surface2);
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
gap: 10px;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
transition: border-color 0.15s, background 0.15s;
|
|
||||||
}
|
|
||||||
.backup-item:hover { border-color: var(--border2); background: var(--bg4); }
|
|
||||||
|
|
||||||
.backup-name {
|
|
||||||
font-family: var(--mono);
|
|
||||||
font-size: 12px;
|
|
||||||
color: var(--text2);
|
|
||||||
word-break: break-all;
|
|
||||||
}
|
|
||||||
|
|
||||||
.backup-actions { display: flex; gap: 5px; align-items: center; flex-shrink: 0; }
|
|
||||||
|
|
||||||
/* ── ENHANCED MODAL ─────────────────────────────────────────── */
|
|
||||||
.modal-overlay {
|
.modal-overlay {
|
||||||
position: fixed; inset: 0; z-index: 1000;
|
position: fixed; inset: 0; z-index: 1000;
|
||||||
background: rgba(0,0,0,0.72);
|
background: rgba(0,0,0,0.72);
|
||||||
@@ -298,18 +274,20 @@
|
|||||||
<button class="btn btn-ghost btn-sm" onclick="refreshBackupsList()"><i class="fas fa-sync-alt"></i> Refresh</button>
|
<button class="btn btn-ghost btn-sm" onclick="refreshBackupsList()"><i class="fas fa-sync-alt"></i> Refresh</button>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div style="display:grid;grid-template-columns:1fr 1fr 1fr;gap:16px;">
|
<div class="backup-columns">
|
||||||
|
|
||||||
{# LOCAL #}
|
{# LOCAL #}
|
||||||
<div>
|
<div class="backup-column">
|
||||||
<div class="section-header">
|
<div class="backup-column-header">
|
||||||
<div class="section-title">🖥️ MAIN SERVER</div>
|
<div class="section-title">🖥️ Main Server</div>
|
||||||
<span class="card-meta">/root/backups/</span>
|
<span class="card-meta">/root/backups/</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="backup-list" id="local-backup-list">
|
<div class="backup-list" id="local-backup-list">
|
||||||
{% for b in backups %}
|
{% for b in backups %}
|
||||||
<div class="backup-item">
|
<div class="backup-item">
|
||||||
|
<div class="backup-item-main">
|
||||||
<span class="backup-name">{{ b }}</span>
|
<span class="backup-name">{{ b }}</span>
|
||||||
|
</div>
|
||||||
<div class="backup-actions">
|
<div class="backup-actions">
|
||||||
<button class="btn btn-ghost btn-sm" style="color:var(--text3);" onclick="showBackupDetails('local','{{ b }}',this)" title="Quick details"><i class="fas fa-circle-info"></i></button>
|
<button class="btn btn-ghost btn-sm" style="color:var(--text3);" onclick="showBackupDetails('local','{{ b }}',this)" title="Quick details"><i class="fas fa-circle-info"></i></button>
|
||||||
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditBackup('local','{{ b }}',this)"><i class="fas fa-shield-check"></i> Audit</button>
|
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditBackup('local','{{ b }}',this)"><i class="fas fa-shield-check"></i> Audit</button>
|
||||||
@@ -323,15 +301,17 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{# VM #}
|
{# VM #}
|
||||||
<div>
|
<div class="backup-column">
|
||||||
<div class="section-header">
|
<div class="backup-column-header">
|
||||||
<div class="section-title">💾 VM SERVER</div>
|
<div class="section-title">💾 VM Server</div>
|
||||||
<span class="card-meta">/backups/main-server/</span>
|
<span class="card-meta">/backups/main-server/</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="backup-list" id="vm-backup-list">
|
<div class="backup-list" id="vm-backup-list">
|
||||||
{% for b in vm_backups %}
|
{% for b in vm_backups %}
|
||||||
<div class="backup-item">
|
<div class="backup-item">
|
||||||
|
<div class="backup-item-main">
|
||||||
<span class="backup-name">{{ b }}</span>
|
<span class="backup-name">{{ b }}</span>
|
||||||
|
</div>
|
||||||
<div class="backup-actions">
|
<div class="backup-actions">
|
||||||
<button class="btn btn-ghost btn-sm" style="color:var(--text3);" onclick="showBackupDetails('vm','{{ b }}',this)" title="Quick details"><i class="fas fa-circle-info"></i></button>
|
<button class="btn btn-ghost btn-sm" style="color:var(--text3);" onclick="showBackupDetails('vm','{{ b }}',this)" title="Quick details"><i class="fas fa-circle-info"></i></button>
|
||||||
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditBackup('vm','{{ b }}',this)"><i class="fas fa-shield-check"></i> Audit</button>
|
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditBackup('vm','{{ b }}',this)"><i class="fas fa-shield-check"></i> Audit</button>
|
||||||
@@ -344,9 +324,9 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{# CLOUD R2 #}
|
{# CLOUD R2 #}
|
||||||
<div>
|
<div class="backup-column">
|
||||||
<div class="section-header">
|
<div class="backup-column-header">
|
||||||
<div class="section-title">☁️ CLOUD (R2)</div>
|
<div class="section-title">☁️ Cloud (R2)</div>
|
||||||
<span class="card-meta">navitrends-backups</span>
|
<span class="card-meta">navitrends-backups</span>
|
||||||
</div>
|
</div>
|
||||||
<div class="backup-list" id="cloud-backup-list">
|
<div class="backup-list" id="cloud-backup-list">
|
||||||
@@ -363,7 +343,7 @@
|
|||||||
<div class="card-title"><i class="fas fa-clock-rotate-left"></i> Backup History</div>
|
<div class="card-title"><i class="fas fa-clock-rotate-left"></i> Backup History</div>
|
||||||
<button class="btn btn-ghost btn-sm" onclick="loadBackupLog()"><i class="fas fa-sync-alt"></i> Refresh</button>
|
<button class="btn btn-ghost btn-sm" onclick="loadBackupLog()"><i class="fas fa-sync-alt"></i> Refresh</button>
|
||||||
</div>
|
</div>
|
||||||
<div id="backup-history-list"><div class="empty-state"><i class="fas fa-spinner fa-spin"></i> Loading…</div></div>
|
<div id="backup-history-list" class="cloud-backup-list"><div class="empty-state"><i class="fas fa-spinner fa-spin"></i> Loading…</div></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
||||||
@@ -734,11 +714,9 @@ async function loadCloudBackupsColumn() {
|
|||||||
const archives = data.backups.filter(b => b.name.endsWith('.tar.gz'));
|
const archives = data.backups.filter(b => b.name.endsWith('.tar.gz'));
|
||||||
list.innerHTML = archives.map(b => `
|
list.innerHTML = archives.map(b => `
|
||||||
<div class="backup-item">
|
<div class="backup-item">
|
||||||
<div style="display:flex;flex-direction:column;gap:2px;min-width:0;">
|
<div class="backup-item-main">
|
||||||
<span class="backup-name" style="font-size:11px;">
|
<span class="backup-name"><i class="fas fa-cloud" style="color:var(--accent2);"></i>${b.name}</span>
|
||||||
<i class="fas fa-cloud" style="color:var(--accent2);font-size:10px;margin-right:4px;"></i>${b.name}
|
<span class="backup-meta">${b.size_human} · ${b.last_modified_str}</span>
|
||||||
</span>
|
|
||||||
<span style="font-size:10px;color:var(--text3);font-family:var(--mono);">${b.size_human} · ${b.last_modified_str}</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="backup-actions">
|
<div class="backup-actions">
|
||||||
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditCloudBackup('${b.key}', this)" title="Audit R2 integrity">
|
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditCloudBackup('${b.key}', this)" title="Audit R2 integrity">
|
||||||
|
|||||||
@@ -1,12 +1,18 @@
|
|||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
|
|
||||||
{# ── Connection status banner ─────────────────────────────────────────────── #}
|
{# ── Connection status ─────────────────────────────────────────────────────── #}
|
||||||
<div class="card" id="r2-status-card">
|
<div class="card cloud-status-card" id="r2-status-card">
|
||||||
<div class="card-header">
|
<div class="cloud-status-hero">
|
||||||
<div class="card-title"><i class="fas fa-cloud"></i> Cloudflare R2 — Cloud Storage</div>
|
<div class="cloud-status-hero-left">
|
||||||
<div style="display:flex;align-items:center;gap:10px;">
|
<div class="cloud-status-icon"><i class="fas fa-cloud"></i></div>
|
||||||
<span id="r2-conn-badge" class="badge" style="background:rgba(59,130,246,0.12);color:var(--accent2);">
|
<div>
|
||||||
|
<div class="cloud-status-title">Cloudflare R2</div>
|
||||||
|
<div class="cloud-status-sub">Cold storage for off-site backup archives</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="cloud-status-hero-right">
|
||||||
|
<span id="r2-conn-badge" class="cloud-conn-badge">
|
||||||
<i class="fas fa-spinner fa-spin"></i> Checking…
|
<i class="fas fa-spinner fa-spin"></i> Checking…
|
||||||
</span>
|
</span>
|
||||||
<button class="btn btn-ghost btn-sm" onclick="testR2Connection()">
|
<button class="btn btn-ghost btn-sm" onclick="testR2Connection()">
|
||||||
@@ -14,7 +20,8 @@
|
|||||||
</button>
|
</button>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="stat-row" id="r2-stats-row">
|
|
||||||
|
<div class="cloud-stats-grid" id="r2-stats-row">
|
||||||
<div class="stat-card">
|
<div class="stat-card">
|
||||||
<div class="stat-number" id="r2-stat-objects">—</div>
|
<div class="stat-number" id="r2-stat-objects">—</div>
|
||||||
<div class="stat-label">Cloud Backups</div>
|
<div class="stat-label">Cloud Backups</div>
|
||||||
@@ -24,15 +31,16 @@
|
|||||||
<div class="stat-label">Total Cloud Size</div>
|
<div class="stat-label">Total Cloud Size</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="stat-card">
|
<div class="stat-card">
|
||||||
<div class="stat-number" style="font-size:13px;color:var(--text2);" id="r2-stat-bucket">{{ r2_bucket }}</div>
|
<div class="stat-number is-text" id="r2-stat-bucket">{{ r2_bucket }}</div>
|
||||||
<div class="stat-label">Bucket Name</div>
|
<div class="stat-label">Bucket Name</div>
|
||||||
</div>
|
</div>
|
||||||
<div class="stat-card">
|
<div class="stat-card">
|
||||||
<div class="stat-number" style="font-size:11px;color:var(--text3);">R2</div>
|
<div class="stat-number is-text" style="color:var(--text2);">Cloudflare R2</div>
|
||||||
<div class="stat-label">Provider</div>
|
<div class="stat-label">Provider</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div id="r2-conn-msg" style="color:var(--text3);font-size:12px;margin-top:4px;font-family:var(--mono);padding:0 4px;"></div>
|
|
||||||
|
<div id="r2-conn-msg" class="cloud-conn-msg" style="display:none;"></div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{# ── Upload panel ─────────────────────────────────────────────────────────── #}
|
{# ── Upload panel ─────────────────────────────────────────────────────────── #}
|
||||||
@@ -40,11 +48,11 @@
|
|||||||
<div class="card-header">
|
<div class="card-header">
|
||||||
<div class="card-title"><i class="fas fa-cloud-upload-alt"></i> Upload Backup to R2</div>
|
<div class="card-title"><i class="fas fa-cloud-upload-alt"></i> Upload Backup to R2</div>
|
||||||
</div>
|
</div>
|
||||||
<p style="color:var(--text2);font-size:13px;margin-bottom:14px;">
|
<p style="color:var(--text2);font-size:13px;margin-bottom:18px;line-height:1.55;">
|
||||||
Select a local or VM backup to push to Cloudflare R2 cold storage.
|
Select a local or VM backup to push to Cloudflare R2 cold storage.
|
||||||
</p>
|
</p>
|
||||||
<div style="display:flex;gap:12px;flex-wrap:wrap;align-items:flex-end;max-width:640px;">
|
<div class="cloud-upload-form">
|
||||||
<div class="form-group" style="flex:1;min-width:260px;">
|
<div class="form-group">
|
||||||
<label class="form-label">SOURCE BACKUP</label>
|
<label class="form-label">SOURCE BACKUP</label>
|
||||||
<select id="upload-backup-select" class="form-input">
|
<select id="upload-backup-select" class="form-input">
|
||||||
{% if local_backups %}
|
{% if local_backups %}
|
||||||
@@ -69,24 +77,24 @@
|
|||||||
</div>
|
</div>
|
||||||
|
|
||||||
{% if not r2_configured %}
|
{% if not r2_configured %}
|
||||||
<div style="margin-top:14px;padding:12px 14px;background:rgba(245,158,11,0.08);border:1px solid rgba(245,158,11,0.25);border-radius:8px;font-size:12px;color:var(--yellow);">
|
<div style="margin-top:16px;padding:12px 14px;background:rgba(245,158,11,0.08);border:1px solid rgba(245,158,11,0.25);border-radius:10px;font-size:12px;color:var(--yellow);line-height:1.5;">
|
||||||
<i class="fas fa-triangle-exclamation"></i>
|
<i class="fas fa-triangle-exclamation"></i>
|
||||||
<strong>R2 credentials not configured.</strong>
|
<strong>R2 credentials not configured.</strong>
|
||||||
Set <code>R2_ACCESS_KEY_ID</code> and <code>R2_SECRET_ACCESS_KEY</code> — see Setup Guide below.
|
Set <code>R2_ACCESS_KEY_ID</code> and <code>R2_SECRET_ACCESS_KEY</code> — see Setup Guide below.
|
||||||
</div>
|
</div>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
<div id="r2-upload-wrapper" style="display:none;margin-top:16px;">
|
<div id="r2-upload-wrapper" style="display:none;margin-top:20px;padding-top:20px;border-top:1px solid var(--border);">
|
||||||
<div class="card-header" style="margin-bottom:8px;">
|
<div class="card-header" style="margin-bottom:8px;">
|
||||||
<div class="card-title" style="font-size:13px;"><i class="fas fa-terminal"></i> Upload Log</div>
|
<div class="card-title" style="font-size:13px;"><i class="fas fa-terminal"></i> Upload Log</div>
|
||||||
<span class="badge" id="r2-upload-badge" style="background:rgba(59,130,246,0.15);color:var(--accent2);">Running…</span>
|
<span class="badge" id="r2-upload-badge" style="background:rgba(59,130,246,0.15);color:var(--accent2);">Running…</span>
|
||||||
</div>
|
</div>
|
||||||
<div id="r2-upload-log" class="log-console" style="max-height:200px;"></div>
|
<div id="r2-upload-log" class="log-console" style="max-height:200px;"></div>
|
||||||
<div style="margin-top:8px;">
|
<div style="margin-top:10px;">
|
||||||
<div style="height:6px;background:var(--border);border-radius:4px;overflow:hidden;">
|
<div style="height:6px;background:var(--border);border-radius:99px;overflow:hidden;">
|
||||||
<div id="r2-progress-bar" style="height:100%;background:linear-gradient(90deg,var(--accent2),var(--cyan));width:0%;transition:width 0.4s ease;border-radius:4px;"></div>
|
<div id="r2-progress-bar" style="height:100%;background:linear-gradient(90deg,var(--accent2),var(--cyan));width:0%;transition:width 0.4s ease;border-radius:99px;"></div>
|
||||||
</div>
|
</div>
|
||||||
<div style="display:flex;justify-content:space-between;margin-top:4px;">
|
<div style="display:flex;justify-content:space-between;margin-top:6px;">
|
||||||
<span style="font-size:11px;color:var(--text3);font-family:var(--mono);" id="r2-upload-elapsed"></span>
|
<span style="font-size:11px;color:var(--text3);font-family:var(--mono);" id="r2-upload-elapsed"></span>
|
||||||
<span style="font-size:11px;color:var(--accent2);font-family:var(--mono);" id="r2-progress-pct">0%</span>
|
<span style="font-size:11px;color:var(--accent2);font-family:var(--mono);" id="r2-progress-pct">0%</span>
|
||||||
</div>
|
</div>
|
||||||
@@ -100,7 +108,7 @@
|
|||||||
<div class="card-title"><i class="fas fa-database"></i> Cloud Backups (R2)</div>
|
<div class="card-title"><i class="fas fa-database"></i> Cloud Backups (R2)</div>
|
||||||
<button class="btn btn-ghost btn-sm" onclick="loadR2Backups()"><i class="fas fa-sync-alt"></i> Refresh</button>
|
<button class="btn btn-ghost btn-sm" onclick="loadR2Backups()"><i class="fas fa-sync-alt"></i> Refresh</button>
|
||||||
</div>
|
</div>
|
||||||
<div id="r2-backup-list">
|
<div id="r2-backup-list" class="cloud-backup-list">
|
||||||
<div class="empty-state"><i class="fas fa-spinner fa-spin"></i> Loading…</div>
|
<div class="empty-state"><i class="fas fa-spinner fa-spin"></i> Loading…</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -168,22 +176,30 @@ function toggleSetupGuide() {
|
|||||||
async function testR2Connection() {
|
async function testR2Connection() {
|
||||||
const badge = document.getElementById('r2-conn-badge');
|
const badge = document.getElementById('r2-conn-badge');
|
||||||
const msg = document.getElementById('r2-conn-msg');
|
const msg = document.getElementById('r2-conn-msg');
|
||||||
|
badge.className = 'cloud-conn-badge';
|
||||||
badge.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Testing…';
|
badge.innerHTML = '<i class="fas fa-spinner fa-spin"></i> Testing…';
|
||||||
badge.style.cssText = 'background:rgba(59,130,246,0.12);color:var(--accent2);';
|
badge.style.cssText = '';
|
||||||
try {
|
try {
|
||||||
const res = await fetch('/api/cloud/r2/test');
|
const res = await fetch('/api/cloud/r2/test');
|
||||||
const data = await res.json();
|
const data = await res.json();
|
||||||
if (data.success) {
|
if (data.success) {
|
||||||
badge.innerHTML = '✅ Connected';
|
badge.innerHTML = '<i class="fas fa-circle-check"></i> Connected';
|
||||||
badge.style.cssText = 'background:rgba(34,197,94,0.12);color:var(--green);';
|
badge.style.cssText = 'background:rgba(34,197,94,0.1);border-color:rgba(34,197,94,0.25);color:var(--green);';
|
||||||
} else {
|
} else {
|
||||||
badge.innerHTML = '❌ Disconnected';
|
badge.innerHTML = '<i class="fas fa-circle-xmark"></i> Disconnected';
|
||||||
badge.style.cssText = 'background:rgba(239,68,68,0.12);color:#f87171;';
|
badge.style.cssText = 'background:rgba(239,68,68,0.1);border-color:rgba(239,68,68,0.25);color:var(--red);';
|
||||||
|
}
|
||||||
|
if (data.message) {
|
||||||
|
msg.textContent = data.message;
|
||||||
|
msg.style.display = '';
|
||||||
|
} else {
|
||||||
|
msg.style.display = 'none';
|
||||||
}
|
}
|
||||||
msg.textContent = data.message || '';
|
|
||||||
} catch(e) {
|
} catch(e) {
|
||||||
badge.innerHTML = '❌ Error';
|
badge.innerHTML = '<i class="fas fa-circle-xmark"></i> Error';
|
||||||
|
badge.style.cssText = 'background:rgba(239,68,68,0.1);border-color:rgba(239,68,68,0.25);color:var(--red);';
|
||||||
msg.textContent = e.message;
|
msg.textContent = e.message;
|
||||||
|
msg.style.display = '';
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -208,16 +224,12 @@ async function loadR2Backups() {
|
|||||||
}
|
}
|
||||||
list.innerHTML = data.backups.map(b => `
|
list.innerHTML = data.backups.map(b => `
|
||||||
<div class="backup-item">
|
<div class="backup-item">
|
||||||
<div style="display:flex;flex-direction:column;gap:2px;">
|
<div class="backup-item-main">
|
||||||
<span class="backup-name">
|
<span class="backup-name"><i class="fas fa-cloud" style="color:var(--accent2);"></i>${b.name}</span>
|
||||||
<i class="fas fa-cloud" style="color:var(--accent2);font-size:11px;margin-right:6px;"></i>${b.name}
|
<span class="backup-meta">${b.size_human} · ${b.last_modified_str}</span>
|
||||||
</span>
|
|
||||||
<span style="font-size:11px;color:var(--text3);font-family:var(--mono);">
|
|
||||||
${b.size_human} · ${b.last_modified_str}
|
|
||||||
</span>
|
|
||||||
</div>
|
</div>
|
||||||
<div class="backup-actions">
|
<div class="backup-actions">
|
||||||
<button class="btn btn-ghost btn-sm" onclick="auditR2Backup('${b.key}', this)" title="Audit integrity">
|
<button class="btn btn-ghost btn-sm btn-audit" onclick="auditR2Backup('${b.key}', this)" title="Audit integrity">
|
||||||
<i class="fas fa-shield-check"></i> Audit
|
<i class="fas fa-shield-check"></i> Audit
|
||||||
</button>
|
</button>
|
||||||
<button class="btn btn-ghost btn-sm" onclick="restoreFromR2('${b.key}','${b.name}')">
|
<button class="btn btn-ghost btn-sm" onclick="restoreFromR2('${b.key}','${b.name}')">
|
||||||
@@ -230,7 +242,7 @@ async function loadR2Backups() {
|
|||||||
</div>
|
</div>
|
||||||
`).join('');
|
`).join('');
|
||||||
} catch(e) {
|
} catch(e) {
|
||||||
list.innerHTML = `<div class="empty-state" style="color:#f87171;"><i class="fas fa-circle-exclamation"></i> ${e.message}</div>`;
|
list.innerHTML = `<div class="empty-state" style="color:var(--red);"><i class="fas fa-circle-exclamation"></i> ${e.message}</div>`;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,88 +1,37 @@
|
|||||||
{% extends "base.html" %}
|
{% extends "base.html" %}
|
||||||
{% block content %}
|
{% block content %}
|
||||||
|
|
||||||
<style>
|
|
||||||
.dash-charts {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(3, 1fr);
|
|
||||||
gap: 16px;
|
|
||||||
margin-bottom: 16px;
|
|
||||||
}
|
|
||||||
.dash-chart-card {
|
|
||||||
background: var(--surface);
|
|
||||||
border: 1px solid var(--border);
|
|
||||||
border-radius: var(--radius-lg);
|
|
||||||
padding: 18px;
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
align-items: center;
|
|
||||||
min-height: 260px;
|
|
||||||
}
|
|
||||||
.dash-chart-card h3 {
|
|
||||||
font-size: 12px;
|
|
||||||
font-weight: 600;
|
|
||||||
letter-spacing: 0.06em;
|
|
||||||
color: var(--text2);
|
|
||||||
margin: 0 0 12px;
|
|
||||||
align-self: flex-start;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 8px;
|
|
||||||
}
|
|
||||||
.dash-chart-card h3 i { color: var(--accent2); font-size: 11px; }
|
|
||||||
.dash-chart-wrap {
|
|
||||||
position: relative;
|
|
||||||
width: 100%;
|
|
||||||
max-width: 200px;
|
|
||||||
height: 200px;
|
|
||||||
flex: 1;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
}
|
|
||||||
.dash-chart-legend {
|
|
||||||
display: flex;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: 10px 16px;
|
|
||||||
justify-content: center;
|
|
||||||
margin-top: 12px;
|
|
||||||
font-family: var(--mono);
|
|
||||||
font-size: 10px;
|
|
||||||
color: var(--text2);
|
|
||||||
}
|
|
||||||
.dash-chart-legend span { display: inline-flex; align-items: center; gap: 5px; }
|
|
||||||
.dash-chart-legend .dot { width: 8px; height: 8px; border-radius: 50%; }
|
|
||||||
.dash-quick-links {
|
|
||||||
display: flex;
|
|
||||||
gap: 8px;
|
|
||||||
margin-top: 10px;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
justify-content: center;
|
|
||||||
}
|
|
||||||
@media (max-width: 1100px) {
|
|
||||||
.dash-charts { grid-template-columns: 1fr; }
|
|
||||||
}
|
|
||||||
</style>
|
|
||||||
|
|
||||||
<div class="metrics-row">
|
<div class="metrics-row">
|
||||||
<div class="metric-card cpu">
|
<div class="metric-card cpu">
|
||||||
|
<div class="metric-top">
|
||||||
<div class="metric-label">CPU USAGE</div>
|
<div class="metric-label">CPU USAGE</div>
|
||||||
|
<div class="metric-icon"><i class="fas fa-microchip"></i></div>
|
||||||
|
</div>
|
||||||
<div class="metric-value" id="m-cpu">{{ system.cpu_pct or '…' }}<span>%</span></div>
|
<div class="metric-value" id="m-cpu">{{ system.cpu_pct or '…' }}<span>%</span></div>
|
||||||
<div class="gauge-bar"><div class="gauge-fill" id="g-cpu" style="width:{{ system.cpu_pct or 0 }}%"></div></div>
|
<div class="gauge-bar"><div class="gauge-fill" id="g-cpu" style="width:{{ system.cpu_pct or 0 }}%"></div></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="metric-card mem">
|
<div class="metric-card mem">
|
||||||
|
<div class="metric-top">
|
||||||
<div class="metric-label">MEMORY</div>
|
<div class="metric-label">MEMORY</div>
|
||||||
<div class="metric-value" id="m-mem" style="font-size:16px;">{{ system.memory or '…' }}</div>
|
<div class="metric-icon"><i class="fas fa-memory"></i></div>
|
||||||
|
</div>
|
||||||
|
<div class="metric-value small" id="m-mem">{{ system.memory or '…' }}</div>
|
||||||
<div class="gauge-bar"><div class="gauge-fill" id="g-mem" style="width:{{ system.mem_pct or 0 }}%"></div></div>
|
<div class="gauge-bar"><div class="gauge-fill" id="g-mem" style="width:{{ system.mem_pct or 0 }}%"></div></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="metric-card disk">
|
<div class="metric-card disk">
|
||||||
|
<div class="metric-top">
|
||||||
<div class="metric-label">DISK /</div>
|
<div class="metric-label">DISK /</div>
|
||||||
<div class="metric-value" id="m-disk" style="font-size:16px;">{{ system.disk or '…' }}</div>
|
<div class="metric-icon"><i class="fas fa-hard-drive"></i></div>
|
||||||
|
</div>
|
||||||
|
<div class="metric-value small" id="m-disk">{{ system.disk or '…' }}</div>
|
||||||
<div class="gauge-bar"><div class="gauge-fill" id="g-disk" style="width:{{ system.disk_pct or 0 }}%"></div></div>
|
<div class="gauge-bar"><div class="gauge-fill" id="g-disk" style="width:{{ system.disk_pct or 0 }}%"></div></div>
|
||||||
</div>
|
</div>
|
||||||
<div class="metric-card load">
|
<div class="metric-card load">
|
||||||
|
<div class="metric-top">
|
||||||
<div class="metric-label">LOAD AVG</div>
|
<div class="metric-label">LOAD AVG</div>
|
||||||
<div class="metric-value" id="m-load" style="font-size:16px;">{{ system.load or '…' }}</div>
|
<div class="metric-icon"><i class="fas fa-gauge-high"></i></div>
|
||||||
|
</div>
|
||||||
|
<div class="metric-value small" id="m-load">{{ system.load or '…' }}</div>
|
||||||
<div class="gauge-bar"><div class="gauge-fill" id="g-load" style="width:10%"></div></div>
|
<div class="gauge-bar"><div class="gauge-fill" id="g-load" style="width:10%"></div></div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
@@ -161,8 +110,10 @@
|
|||||||
datasets: [{
|
datasets: [{
|
||||||
data: values,
|
data: values,
|
||||||
backgroundColor: colors,
|
backgroundColor: colors,
|
||||||
borderWidth: 0,
|
borderWidth: 2,
|
||||||
hoverOffset: 4,
|
borderColor: isDark() ? '#0e1017' : '#ffffff',
|
||||||
|
hoverOffset: 8,
|
||||||
|
hoverBorderWidth: 0,
|
||||||
}],
|
}],
|
||||||
},
|
},
|
||||||
options: {
|
options: {
|
||||||
|
|||||||
@@ -2,6 +2,8 @@
|
|||||||
# =============================================
|
# =============================================
|
||||||
# backup-myapps.sh — Run on MAIN SERVER
|
# backup-myapps.sh — Run on MAIN SERVER
|
||||||
# Backs up: Frappe, Nextcloud, Mautic, n8n, Odoo
|
# Backs up: Frappe, Nextcloud, Mautic, n8n, Odoo
|
||||||
|
# + CI/CD (my-jenkins, management-platform)
|
||||||
|
# + Infrastructure (K8s ingress, Traefik, SSL certs)
|
||||||
# Storage tiers:
|
# Storage tiers:
|
||||||
# 1. Local → /root/backups/ (CRITICAL — always runs)
|
# 1. Local → /root/backups/ (CRITICAL — always runs)
|
||||||
# 2. VM → SSH tunnel → /backups/main-server/
|
# 2. VM → SSH tunnel → /backups/main-server/
|
||||||
@@ -65,6 +67,8 @@ log_status() {
|
|||||||
echo "========================================="
|
echo "========================================="
|
||||||
echo "📦 Starting Backup: $BACKUP_NAME"
|
echo "📦 Starting Backup: $BACKUP_NAME"
|
||||||
echo " Apps: Frappe, Nextcloud, Mautic, n8n, Odoo"
|
echo " Apps: Frappe, Nextcloud, Mautic, n8n, Odoo"
|
||||||
|
echo " CI/CD: my-jenkins, management-platform"
|
||||||
|
echo " Infra: K8s ingress, Traefik, SSL certs"
|
||||||
echo " Tiers: Local → VM → ☁ Cloudflare R2"
|
echo " Tiers: Local → VM → ☁ Cloudflare R2"
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
|
|
||||||
@@ -168,7 +172,85 @@ docker exec frappe-erpnext \
|
|||||||
|| echo " ⏭️ Frappe config not found (skipping)"
|
|| echo " ⏭️ Frappe config not found (skipping)"
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# 6. Metadata + checksum
|
# 6. CI/CD & Infrastructure configs ← NEW SECTION
|
||||||
|
# --------------------------------------------------
|
||||||
|
echo ""
|
||||||
|
echo "🚀 [6b/7] Backing up CI/CD & Infrastructure configs..."
|
||||||
|
mkdir -p cicd-configs/k8s
|
||||||
|
|
||||||
|
# my-cicd folder (Jenkinsfile, docker-compose for my-jenkins)
|
||||||
|
if [ -d /root/my-cicd ]; then
|
||||||
|
cp -r /root/my-cicd cicd-configs/my-cicd \
|
||||||
|
&& echo " ✅ /root/my-cicd (Jenkinsfile + compose)" \
|
||||||
|
|| echo " ⚠️ my-cicd copy failed"
|
||||||
|
else
|
||||||
|
echo " ⏭️ /root/my-cicd not found — skipping"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# my-jenkins Docker volume (jobs, credentials, plugins, build history)
|
||||||
|
JENKINS_VOL="my-cicd_jenkins_home"
|
||||||
|
# Try common volume name variants
|
||||||
|
for VOL_NAME in "my-cicd_jenkins_home" "my-cicd_jenkins-home" "my-cicd_jenkins-data" "jenkins_home"; do
|
||||||
|
if docker volume inspect "$VOL_NAME" &>/dev/null; then
|
||||||
|
echo -n " 📁 my-jenkins volume ($VOL_NAME) ... "
|
||||||
|
docker run --rm \
|
||||||
|
-v "${VOL_NAME}:/source:ro" \
|
||||||
|
-v "$(pwd)/cicd-configs:/backup" \
|
||||||
|
alpine tar czf "/backup/my-jenkins-volume.tar.gz" -C /source . \
|
||||||
|
&& echo "✅" || echo "⚠️ FAILED (continuing)"
|
||||||
|
break
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
# management-platform source (excluding pycache)
|
||||||
|
if [ -d /root/management-platform ]; then
|
||||||
|
tar czf cicd-configs/management-platform.tar.gz \
|
||||||
|
--exclude='*/__pycache__' \
|
||||||
|
--exclude='*.pyc' \
|
||||||
|
--exclude='*/venv' \
|
||||||
|
-C /root management-platform \
|
||||||
|
&& echo " ✅ management-platform source" \
|
||||||
|
|| echo " ⚠️ management-platform backup failed"
|
||||||
|
else
|
||||||
|
echo " ⏭️ /root/management-platform not found — skipping"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Traefik config folder
|
||||||
|
if [ -d /root/traefik ]; then
|
||||||
|
cp -r /root/traefik cicd-configs/traefik \
|
||||||
|
&& echo " ✅ /root/traefik config" \
|
||||||
|
|| echo " ⚠️ traefik copy failed"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# acme.json SSL cert backup (critical for domain SSL)
|
||||||
|
if [ -f /root/acme.json.working-backup ]; then
|
||||||
|
cp /root/acme.json.working-backup cicd-configs/acme.json.backup \
|
||||||
|
&& echo " ✅ acme.json (SSL certs)" \
|
||||||
|
|| echo " ⚠️ acme.json copy failed"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# K8s ingress rules (domain routing config — cloudops, nextcloud, odoo)
|
||||||
|
kubectl get ingress --all-namespaces -o yaml \
|
||||||
|
> cicd-configs/k8s/ingress-all.yaml 2>/dev/null \
|
||||||
|
&& echo " ✅ K8s ingress rules" \
|
||||||
|
|| echo " ⏭️ kubectl not available — skipping K8s exports"
|
||||||
|
|
||||||
|
# K8s services & endpoints (ExternalName services pointing to Docker containers)
|
||||||
|
kubectl get services --all-namespaces -o yaml \
|
||||||
|
> cicd-configs/k8s/services-all.yaml 2>/dev/null || true
|
||||||
|
kubectl get endpoints --all-namespaces -o yaml \
|
||||||
|
> cicd-configs/k8s/endpoints-all.yaml 2>/dev/null || true
|
||||||
|
|
||||||
|
# K8s TLS secrets (cert-manager issued certs for domains)
|
||||||
|
kubectl get secret --all-namespaces \
|
||||||
|
--field-selector type=kubernetes.io/tls -o yaml \
|
||||||
|
> cicd-configs/k8s/tls-secrets.yaml 2>/dev/null \
|
||||||
|
&& echo " ✅ K8s TLS secrets (domain certs)" || true
|
||||||
|
|
||||||
|
echo " ✅ CI/CD & Infrastructure backup done"
|
||||||
|
|
||||||
|
# --------------------------------------------------
|
||||||
|
# 7. Metadata + checksum
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "📝 [6/7] Writing backup metadata..."
|
echo "📝 [6/7] Writing backup metadata..."
|
||||||
@@ -179,6 +261,8 @@ Backup Name: $BACKUP_NAME
|
|||||||
Backup Date: $(date)
|
Backup Date: $(date)
|
||||||
Hostname: $(hostname)
|
Hostname: $(hostname)
|
||||||
Apps: Frappe, Nextcloud, Mautic, n8n, Odoo
|
Apps: Frappe, Nextcloud, Mautic, n8n, Odoo
|
||||||
|
CI/CD: my-jenkins, management-platform
|
||||||
|
Infra: K8s ingress, Traefik, SSL certs
|
||||||
Volumes: $VOLUME_COUNT volume(s) backed up
|
Volumes: $VOLUME_COUNT volume(s) backed up
|
||||||
Docker info: $(docker --version 2>/dev/null || echo 'N/A')
|
Docker info: $(docker --version 2>/dev/null || echo 'N/A')
|
||||||
Storage Tiers:
|
Storage Tiers:
|
||||||
@@ -198,7 +282,7 @@ done
|
|||||||
echo " ✅ Done"
|
echo " ✅ Done"
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# 7. Compress
|
# 8. Compress
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "🗜️ [7/7] Compressing backup..."
|
echo "🗜️ [7/7] Compressing backup..."
|
||||||
@@ -223,7 +307,7 @@ echo ""
|
|||||||
echo "✅ [TIER 1] Local backup complete: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"
|
echo "✅ [TIER 1] Local backup complete: $BACKUP_ARCHIVE ($COMPRESSED_SIZE)"
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# 8. Retention — keep only MAX_BACKUPS locally
|
# 9. Retention — keep only MAX_BACKUPS locally
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "🧹 [Retention] Keeping latest ${MAX_BACKUPS} local backups..."
|
echo "🧹 [Retention] Keeping latest ${MAX_BACKUPS} local backups..."
|
||||||
@@ -242,7 +326,7 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# 9. VM transfer [TIER 2] — non-fatal
|
# 10. VM transfer [TIER 2] — non-fatal
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "📤 [TIER 2] Sending backup to VM (${VM_HOST}:${VM_PORT})..."
|
echo "📤 [TIER 2] Sending backup to VM (${VM_HOST}:${VM_PORT})..."
|
||||||
@@ -266,7 +350,7 @@ else
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# 10. Cloudflare R2 [TIER 3] — non-fatal
|
# 11. Cloudflare R2 [TIER 3] — non-fatal
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "☁️ [TIER 3] Uploading to Cloudflare R2..."
|
echo "☁️ [TIER 3] Uploading to Cloudflare R2..."
|
||||||
@@ -376,7 +460,7 @@ PYEOF
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
# 11. Final summary + log
|
# 12. Final summary + log
|
||||||
# --------------------------------------------------
|
# --------------------------------------------------
|
||||||
echo ""
|
echo ""
|
||||||
echo "========================================="
|
echo "========================================="
|
||||||
|
|||||||
Reference in New Issue
Block a user