CloudOps/platform/app.py

# app.py
from flask import Flask, render_template, request, redirect, url_for, session, jsonify
import os
import re
import subprocess
import sys
import threading
import uuid
import time
from datetime import datetime, timezone

from config import (
    MAIN_SERVER_IP, RUNNING_ON_MAIN_SERVER,
    VM_HOST, VM_PORT, VM_KEY, VM_USER,
    MAIN_SERVER_KEY, MAIN_SERVER_USER,
    MAIN_SERVER_SSH_HOST, MAIN_SERVER_SSH_PORT,
)
from modules.auth import login_required
from modules.backups import (
    get_containers, get_all_root_containers,
    get_local_backups, get_vm_backups,
    get_local_backups_with_status, get_vm_backups_with_status,
    get_all_stats, get_system_info,
    get_rootless_user_containers_remote,
    container_action, get_container_status,
    audit_backup, delete_backup,
    get_backup_log_entries, get_backup_script_path,
    _ssh_main, _human_bytes, _run,
)
from modules.commands import run_command
from modules.users import (
    get_all_users, get_user_containers, get_all_users_containers,
    create_user, delete_user, get_user_disk_usage,
)
from modules.cloud_backup import (
    r2_test_connection, r2_list_backups, r2_get_bucket_stats,
    r2_delete_backup, r2_upload_async, get_upload_job,
    r2_is_configured, R2_BUCKET_NAME,
)
from modules.sites import get_sites_list, get_site_health, SITES, PLATFORM
from modules.kubernetes import get_cluster_overview

app = Flask(__name__)
app.secret_key = 'navitrends-secret-key-2025'
app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0

restore_jobs = {}
backup_jobs  = {}

# Whitelist of apps the restore script knows how to selectively restore.
# Anything not in this set is silently dropped before it ever reaches a shell string.
ALLOWED_RESTORE_APPS = {'frappe', 'odoo', 'nextcloud', 'mautic', 'n8n'}


def _stream_restore(job_id, cmd):
    restore_jobs[job_id] = {'status': 'running', 'log': [], 'started': time.time()}
    try:
        proc = subprocess.Popen(
            cmd, shell=True,
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
            text=True, bufsize=1
        )
        for line in proc.stdout:
            restore_jobs[job_id]['log'].append(line.rstrip())
        proc.wait()
        restore_jobs[job_id]['status'] = 'done' if proc.returncode == 0 else 'error'
        restore_jobs[job_id]['returncode'] = proc.returncode
    except Exception as e:
        restore_jobs[job_id]['log'].append(f"ERROR: {e}")
        restore_jobs[job_id]['status'] = 'error'


def _stream_backup(job_id, script_path):
    backup_jobs[job_id] = {'status': 'running', 'log': [], 'started': time.time()}
    try:
        proc = subprocess.Popen(
            ['bash', script_path],
            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
            text=True, bufsize=1
        )
        for line in proc.stdout:
            backup_jobs[job_id]['log'].append(line.rstrip())
        proc.wait()
        backup_jobs[job_id]['status'] = 'done' if proc.returncode == 0 else 'error'
        backup_jobs[job_id]['returncode'] = proc.returncode
    except Exception as e:
        backup_jobs[job_id]['log'].append(f"ERROR: {e}")
        backup_jobs[job_id]['status'] = 'error'


# ─────────────────────────────────────────────
# PAGES
# ─────────────────────────────────────────────

@app.route('/')
@login_required
def dashboard():
    backups    = get_local_backups()
    vm_backups = get_vm_backups()

    if RUNNING_ON_MAIN_SERVER:
        root_ctrs     = get_all_root_containers()
        user_ctrs     = get_rootless_user_containers_remote()
        all_ctrs      = root_ctrs + user_ctrs
        running_count = sum(1 for c in all_ctrs if c.get('running'))
        system        = get_system_info()
        users         = get_all_users()
        container_count = len(all_ctrs)
    else:
        running_count = 0
        system        = {}
        users         = []
        container_count = 0

    return render_template('pages/dashboard.html',
                           container_count=container_count,
                           running_count=running_count,
                           site_count=len(SITES),
                           backups=backups,
                           vm_backups=vm_backups,
                           main_server=MAIN_SERVER_IP,
                           system=system,
                           users=users,
                           running_on_main=RUNNING_ON_MAIN_SERVER,
                           active_page='dashboard',
                           page_title='Dashboard',
                           page_subtitle=MAIN_SERVER_IP)


@app.route('/sites')
@login_required
def sites_page():
    return render_template(
        'pages/sites.html',
        main_server=MAIN_SERVER_IP,
        site_count=len(SITES),
        platform_domain=PLATFORM['domain'],
        platform_url=f"{PLATFORM['domain_protocol']}://{PLATFORM['domain']}",
        platform_ssl=PLATFORM.get('ssl_configured', True),
        active_page='sites',
        page_title='Application Sites',
        page_subtitle='direct access · health · deployment info'
    )


@app.route('/containers')
@login_required
def containers_page():
    return render_template(
        'pages/containers.html',
        main_server=MAIN_SERVER_IP,
        active_page='containers',
        page_title='All Containers',
        page_subtitle='main server · all users'
    )


@app.route('/cluster')
@login_required
def cluster_page():
    return render_template(
        'pages/cluster.html',
        main_server=MAIN_SERVER_IP,
        active_page='cluster',
        page_title='Cluster',
        page_subtitle='kubernetes · 8 namespaces'
    )


@app.route('/backups')
@login_required
def backups_page():
    return render_template(
        'pages/backups.html',
        backups=get_local_backups_with_status(),
        vm_backups=get_vm_backups_with_status(),
        main_server=MAIN_SERVER_IP,
        active_page='backups',
        page_title='Backup Management',
        page_subtitle='local & VM'
    )


@app.route('/restore')
@login_required
def restore_page():
    prefill = {
        'source': request.args.get('source', '').strip(),
        'file':   request.args.get('file',   '').strip(),
    }
    return render_template(
        'pages/restore.html',
        backups=get_local_backups(),
        vm_backups=get_vm_backups(),
        restore_prefill=prefill,
        main_server=MAIN_SERVER_IP,
        running_on_main_server=RUNNING_ON_MAIN_SERVER,
        # Only meaningful when running as the standby: "restore on this
        # server" is nonsensical there (no local cluster/kubectl at all),
        # so pre-fill the remote-target fields with the tunnel back to the
        # real main server instead of leaving the user to find these values.
        tunnel_remote_ip=MAIN_SERVER_SSH_HOST,
        tunnel_remote_port=MAIN_SERVER_SSH_PORT,
        tunnel_ssh_key=MAIN_SERVER_KEY,
        active_page='restore',
        page_title='Restore',
        page_subtitle='backup → target'
    )


@app.route('/users')
@login_required
def users_page():
    users = get_all_users() if RUNNING_ON_MAIN_SERVER else []
    return render_template(
        'pages/users.html',
        users=users,
        main_server=MAIN_SERVER_IP,
        active_page='users',
        page_title='User Management',
        page_subtitle='linux + docker'
    )


@app.route('/settings')
@login_required
def settings_page():
    system = get_system_info() if RUNNING_ON_MAIN_SERVER else {}
    return render_template(
        'pages/settings.html',
        main_server=MAIN_SERVER_IP,
        platform_domain=PLATFORM['domain'],
        platform_url=f"{PLATFORM['domain_protocol']}://{PLATFORM['domain']}",
        platform_ssl=PLATFORM.get('ssl_configured', True),
        system=system,
        running_on_main=RUNNING_ON_MAIN_SERVER,
        active_page='settings',
        page_title='Settings',
        page_subtitle='platform config'
    )


@app.route('/cloud')
@login_required
def cloud_page():
    return render_template(
        'pages/cloud.html',
        local_backups=get_local_backups(),
        vm_backups=get_vm_backups(),
        main_server=MAIN_SERVER_IP,
        r2_bucket=R2_BUCKET_NAME,
        r2_configured=r2_is_configured(),
        active_page='cloud',
        page_title='Cloud Storage',
        page_subtitle='Cloudflare R2'
    )


# ─────────────────────────────────────────────
# API — system + stats
# ─────────────────────────────────────────────

@app.route('/api/system')
@login_required
def api_system():
    return jsonify(get_system_info())


@app.route('/api/stats')
@login_required
def api_stats():
    return jsonify(get_all_stats())


@app.route('/api/containers')
@login_required
def api_containers():
    containers    = get_all_root_containers()
    running_count = sum(1 for c in containers if 'Up' in c.get('status', ''))
    return jsonify({'containers': containers, 'running': running_count})


@app.route('/api/containers/all')
@login_required
def api_containers_all():
    root_ctrs = get_all_root_containers()
    user_ctrs = get_rootless_user_containers_remote()
    all_ctrs  = root_ctrs + user_ctrs
    running   = sum(1 for c in all_ctrs if 'Up' in c.get('status', ''))
    return jsonify({'containers': all_ctrs, 'running': running})


@app.route('/api/cluster')
@login_required
def api_cluster():
    return jsonify(get_cluster_overview())


@app.route('/api/sites')
@login_required
def api_sites():
    include_health = request.args.get('health', '0') == '1'
    return jsonify({'sites': get_sites_list(include_health=include_health)})


@app.route('/api/sites/<site_id>/health')
@login_required
def api_site_health(site_id):
    result = get_site_health(site_id)
    if result is None:
        return jsonify({'error': 'Site not found'}), 404
    return jsonify(result)


@app.route('/api/nav-summary')
@login_required
def api_nav_summary():
    root_ctrs = get_all_root_containers()
    user_ctrs = get_rootless_user_containers_remote()
    all_ctrs  = root_ctrs + user_ctrs
    users     = get_all_users()
    return jsonify({
        'container_count': len(all_ctrs),
        'user_count':      len(users),
    })


@app.route('/api/dashboard')
@login_required
def api_dashboard():
    system    = get_system_info()
    root_ctrs = get_all_root_containers()
    user_ctrs = get_rootless_user_containers_remote()
    all_ctrs  = root_ctrs + user_ctrs
    users     = get_all_users()
    running   = sum(1 for c in all_ctrs if c.get('running'))
    stopped   = len(all_ctrs) - running

    return jsonify({
        'system':          system,
        'container_count': len(all_ctrs),
        'running_count':   running,
        'stopped_count':   stopped,
        'sites_count':     len(SITES),
        'user_count':      len(users),
        'local_backups':   len(get_local_backups()),
        'vm_backups':      len(get_vm_backups()),
    })


# ─────────────────────────────────────────────
# API — container actions
# ─────────────────────────────────────────────

@app.route('/api/container/action', methods=['POST'])
@login_required
def api_container_action():
    data   = request.get_json() or {}
    name   = data.get('name', '').strip()
    action = data.get('action', '').strip()

    if not name or not action:
        return jsonify({'success': False, 'message': 'name and action required'}), 400

    success, output = container_action(name, action)
    time.sleep(1.5)
    status_info = get_container_status(name)

    return jsonify({
        'success':        success,
        'output':         output,
        'new_status':     status_info['status'],
        'new_status_raw': status_info['raw'],
    })


@app.route('/api/container/status/<name>')
@login_required
def api_container_status(name):
    return jsonify(get_container_status(name))


# ─────────────────────────────────────────────
# API — backups
# ─────────────────────────────────────────────

@app.route('/api/backups')
@login_required
def api_backups():
    return jsonify({'local': get_local_backups_with_status(), 'vm': get_vm_backups_with_status()})


@app.route('/api/backups/log')
@login_required
def api_backup_log():
    limit   = int(request.args.get('limit', 20))
    entries = get_backup_log_entries(limit)
    return jsonify({'entries': entries})


@app.route('/api/backups/audit', methods=['POST'])
@login_required
def api_backup_audit():
    data   = request.get_json() or {}
    bfile  = data.get('backup_file', '').strip()
    source = data.get('source', 'local').strip()

    if not bfile:
        return jsonify({'error': 'backup_file required'}), 400

    result = audit_backup(bfile, source)
    return jsonify(result)


@app.route('/api/backups/delete', methods=['POST'])
@login_required
def api_backup_delete():
    data   = request.get_json() or {}
    bfile  = data.get('backup_file', '').strip()
    source = data.get('source', 'local').strip()

    if not bfile:
        return jsonify({'success': False, 'message': 'backup_file required'}), 400

    success, message = delete_backup(bfile, source)
    return jsonify({'success': success, 'message': message})


@app.route('/api/backups/run', methods=['POST'])
@login_required
def api_backup_run():
    if not RUNNING_ON_MAIN_SERVER:
        return jsonify({
            'success': False,
            'message': 'Manual backup can only be triggered from the main server platform.'
        }), 400

    script = '/root/CloudOps/backup/backup-k8s-apps.sh'
    if not os.path.exists(script):
        return jsonify({
            'success': False,
            'message': f'Backup script not found at {script}'
        }), 500

    job_id = str(uuid.uuid4())
    t = threading.Thread(target=_stream_backup, args=(job_id, script), daemon=True)
    t.start()
    return jsonify({'success': True, 'job_id': job_id, 'status': 'started'})


@app.route('/api/backups/run/status/<job_id>')
@login_required
def api_backup_run_status(job_id):
    job = backup_jobs.get(job_id)
    if not job:
        return jsonify({'error': 'Job not found'}), 404
    return jsonify({
        'status':  job['status'],
        'log':     job['log'],
        'elapsed': round(time.time() - job.get('started', time.time()))
    })


@app.route('/api/backups/details', methods=['POST'])
@login_required
def api_backup_details():
    """
    Fast metadata for the Details popup — no gzip scan, just stat + sha sidecar.
    Body: { backup_file: "myapps-backup-YYYYMMDD_HHMMSS.tar.gz", source: "local"|"vm" }
    """
    data        = request.get_json() or {}
    backup_file = data.get('backup_file', '').strip()
    source      = data.get('source', 'local').strip()

    # Accepts both the legacy Docker-era lineage (myapps-backup-*) and the
    # k8s-native one (myapps-k8s-backup-*, from backup-k8s-apps.sh).
    if not re.match(r'^myapps-(k8s-)?backup-\d{8}_\d{6}\.tar\.gz$', backup_file):
        return jsonify({'error': 'Invalid filename'}), 400

    if source == 'local':
        archive_path = f'/root/backups/{backup_file}'
        sha_path     = archive_path + '.sha256'
    else:
        archive_path = f'/backups/main-server/{backup_file}'
        sha_path     = archive_path + '.sha256'

    result = {
        'backup_file': backup_file,
        'source':      source,
        'path':        archive_path,
        'size_bytes':  None,
        'size_human':  None,
        'created_at':  None,
        'age_days':    None,
        'sha256':      None,
        'on_cloud':    False,
    }

    # ── size + mtime ──────────────────────────────────────────────
    # Try direct stat first regardless of RUNNING_ON_MAIN_SERVER: the
    # management-platform pod has /root hostPath-mounted from the main
    # server, so /root/backups is already right here even though the
    # hostname check that flag relies on never matches inside a pod (it
    # used to always take the SSH branch, using a key/topology this
    # in-pod deployment doesn't have, and silently come back empty).
    if source == 'local':
        if os.path.exists(archive_path):
            # direct stat on this machine
            st = os.stat(archive_path)
            result['size_bytes'] = st.st_size
            result['size_human'] = _human_bytes(st.st_size)
            mtime = datetime.fromtimestamp(st.st_mtime, tz=timezone.utc)
            result['created_at'] = mtime.strftime('%Y-%m-%d %H:%M:%S UTC')
            result['age_days']   = (datetime.now(tz=timezone.utc) - mtime).days
            if os.path.exists(sha_path):
                try:
                    with open(sha_path) as f:
                        result['sha256'] = f.read().split()[0].strip()
                except Exception:
                    pass
        else:
            # SSH to main server
            stat_out, _ = _ssh_main(f"stat -c '%s %Y' {archive_path} 2>/dev/null")
            if stat_out:
                parts = stat_out.split()
                if len(parts) >= 2:
                    size_bytes = int(parts[0])
                    mtime      = datetime.fromtimestamp(int(parts[1]), tz=timezone.utc)
                    result['size_bytes'] = size_bytes
                    result['size_human'] = _human_bytes(size_bytes)
                    result['created_at'] = mtime.strftime('%Y-%m-%d %H:%M:%S UTC')
                    result['age_days']   = (datetime.now(tz=timezone.utc) - mtime).days
            sha_out, _ = _ssh_main(f"cat {sha_path} 2>/dev/null | awk '{{print $1}}'")
            if sha_out.strip():
                result['sha256'] = sha_out.strip()

    elif source == 'vm':
        if os.path.exists(archive_path):
            # direct stat when reachable (VM's own local disk, or a mount)
            st = os.stat(archive_path)
            result['size_bytes'] = st.st_size
            result['size_human'] = _human_bytes(st.st_size)
            mtime = datetime.fromtimestamp(st.st_mtime, tz=timezone.utc)
            result['created_at'] = mtime.strftime('%Y-%m-%d %H:%M:%S UTC')
            result['age_days']   = (datetime.now(tz=timezone.utc) - mtime).days
            if os.path.exists(sha_path):
                try:
                    with open(sha_path) as f:
                        result['sha256'] = f.read().split()[0].strip()
                except Exception:
                    pass
        else:
            # SSH to VM
            ssh_prefix = (
                f"ssh -i {VM_KEY} -p {VM_PORT} "
                f"-o StrictHostKeyChecking=no -o ConnectTimeout=10 -o BatchMode=yes "
                f"{VM_USER}@{VM_HOST}"
            )
            stat_out, _ = _run(
                f"{ssh_prefix} \"stat -c '%s %Y' {archive_path} 2>/dev/null\"",
                timeout=20
            )
            if stat_out:
                parts = stat_out.split()
                if len(parts) >= 2:
                    size_bytes = int(parts[0])
                    mtime      = datetime.fromtimestamp(int(parts[1]), tz=timezone.utc)
                    result['size_bytes'] = size_bytes
                    result['size_human'] = _human_bytes(size_bytes)
                    result['created_at'] = mtime.strftime('%Y-%m-%d %H:%M:%S UTC')
                    result['age_days']   = (datetime.now(tz=timezone.utc) - mtime).days
            sha_out, _ = _run(
                f"{ssh_prefix} \"cat {sha_path} 2>/dev/null | awk '{{print $1}}'\"",
                timeout=15
            )
            if sha_out.strip():
                result['sha256'] = sha_out.strip()

    # ── R2 presence check (best-effort) ──────────────────────────
    try:
        r2_list  = r2_list_backups()
        r2_names = {b.get('name') for b in r2_list}
        result['on_cloud'] = backup_file in r2_names
    except Exception:
        pass

    return jsonify(result)


# ─────────────────────────────────────────────
# API — users
# ─────────────────────────────────────────────

@app.route('/api/users')
@login_required
def api_users():
    return jsonify(get_all_users())


@app.route('/api/users/<username>/containers')
@login_required
def api_user_containers(username):
    return jsonify(get_user_containers(username))


@app.route('/api/users/<username>/disk')
@login_required
def api_user_disk(username):
    return jsonify(get_user_disk_usage(username))


@app.route('/api/users/create', methods=['POST'])
@login_required
def api_create_user():
    data         = request.get_json() or {}
    username     = data.get('username', '').strip()
    password     = data.get('password', '').strip()
    setup_docker = data.get('setup_docker', True)
    disk_quota_mb = data.get('disk_quota_mb')

    if not username:
        return jsonify({'success': False, 'message': 'Username required'}), 400

    success, message = create_user(
        username=username,
        password=password or None,
        setup_docker=setup_docker,
        disk_quota_mb=disk_quota_mb,
    )
    return jsonify({'success': success, 'message': message})


@app.route('/api/users/delete', methods=['POST'])
@login_required
def api_delete_user():
    data        = request.get_json() or {}
    username    = data.get('username', '').strip()
    remove_home = data.get('remove_home', False)

    if not username:
        return jsonify({'success': False, 'message': 'Username required'}), 400

    success, message = delete_user(username, remove_home=remove_home)
    return jsonify({'success': success, 'message': message})


# ─────────────────────────────────────────────
# API — Cloudflare R2 cloud storage
# ─────────────────────────────────────────────

@app.route('/api/cloud/r2/test')
@login_required
def api_r2_test():
    return jsonify(r2_test_connection())


@app.route('/api/cloud/r2/stats')
@login_required
def api_r2_stats():
    return jsonify(r2_get_bucket_stats())


@app.route('/api/cloud/r2/backups')
@login_required
def api_r2_list():
    return jsonify({'backups': r2_list_backups()})


@app.route('/api/cloud/r2/delete', methods=['POST'])
@login_required
def api_r2_delete():
    data = request.get_json() or {}
    key  = data.get('key', '').strip()
    if not key:
        return jsonify({'success': False, 'message': 'key required'}), 400
    success, message = r2_delete_backup(key)
    return jsonify({'success': success, 'message': message})


@app.route('/api/cloud/r2/audit', methods=['POST'])
@login_required
def api_r2_audit():
    from modules.cloud_backup import r2_audit_backup
    data = request.get_json() or {}
    key  = data.get('key', '').strip()
    if not key:
        return jsonify({'error': 'key required'}), 400
    return jsonify(r2_audit_backup(key))


@app.route('/api/cloud/r2/upload', methods=['POST'])
@login_required
def api_r2_upload():
    data        = request.get_json() or {}
    backup_file = data.get('backup_file', '').strip()
    source      = data.get('source', 'local').strip()

    if not backup_file:
        return jsonify({'success': False, 'message': 'backup_file required'}), 400

    if source == 'local':
        if RUNNING_ON_MAIN_SERVER:
            local_path = f"/root/backups/{backup_file}"
        else:
            local_path = f"/tmp/{backup_file}"
            if not os.path.exists(local_path):
                pull_cmd = (
                    f"scp -i {MAIN_SERVER_KEY} -P {MAIN_SERVER_SSH_PORT} "
                    f"-o StrictHostKeyChecking=no -o ConnectTimeout=15 "
                    f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST}:/root/backups/{backup_file} "
                    f"{local_path}"
                )
                res = subprocess.run(pull_cmd, shell=True, capture_output=True, text=True)
                if res.returncode != 0:
                    return jsonify({'success': False, 'message': f'Failed to pull from main server: {res.stderr}'}), 500
    else:
        if RUNNING_ON_MAIN_SERVER:
            local_path = f"/tmp/{backup_file}"
            if not os.path.exists(local_path):
                pull_cmd = (
                    f"scp -i {VM_KEY} -P {VM_PORT} "
                    f"-o StrictHostKeyChecking=no -o ConnectTimeout=15 "
                    f"{VM_USER}@{VM_HOST}:/backups/main-server/{backup_file} "
                    f"{local_path}"
                )
                res = subprocess.run(pull_cmd, shell=True, capture_output=True, text=True)
                if res.returncode != 0:
                    return jsonify({'success': False, 'message': f'Failed to pull from VM: {res.stderr}'}), 500
        else:
            local_path = f"/backups/main-server/{backup_file}"

    if not os.path.exists(local_path):
        return jsonify({'success': False, 'message': f'File not found: {local_path}'}), 400

    job_id = str(uuid.uuid4())
    t = threading.Thread(target=r2_upload_async, args=(local_path, job_id), daemon=True)
    t.start()
    return jsonify({'success': True, 'job_id': job_id})


@app.route('/api/cloud/r2/upload/status/<job_id>')
@login_required
def api_r2_upload_status(job_id):
    job = get_upload_job(job_id)
    if not job:
        return jsonify({'error': 'Job not found'}), 404
    return jsonify(job)


# ─────────────────────────────────────────────
# RESTORE
# ─────────────────────────────────────────────

@app.route('/restore/start', methods=['POST'])
@login_required
def restore_start():
    data = request.get_json()
    if not data:
        return jsonify({'error': 'No JSON body'}), 400

    backup_source = data.get('backup_source', 'local')
    backup_file   = data.get('backup_file', '').strip()
    cloud_key     = data.get('cloud_key', '').strip()
    target        = data.get('target', 'local')
    remote_ip     = data.get('remote_ip', '').strip()
    remote_port   = str(data.get('remote_port', '22')).strip() or '22'
    remote_user   = data.get('remote_user', 'root').strip() or 'root'
    auth_method   = data.get('auth_method', 'key')
    ssh_key_path  = data.get('ssh_key_path', VM_KEY).strip()
    ssh_password  = data.get('ssh_password', '').strip()

    if not backup_file:
        return jsonify({'error': 'No backup file specified'}), 400

    # ── Selective app restore ────────────────────────────────────
    # `apps` is optional. Empty/missing => full restore (all apps).
    # Anything not in ALLOWED_RESTORE_APPS is dropped before it can
    # reach a shell string.
    raw_apps = data.get('apps', [])
    if isinstance(raw_apps, str):
        raw_apps = [a.strip() for a in raw_apps.split(',') if a.strip()]
    selected_apps = sorted({
        a.strip().lower() for a in raw_apps
        if isinstance(a, str) and a.strip().lower() in ALLOWED_RESTORE_APPS
    })
    apps_flag = f" --apps {','.join(selected_apps)}" if selected_apps else ""
    apps_label = ','.join(selected_apps) if selected_apps else 'ALL'

    # ── Resolve backup path by source ────────────────────────────
    if backup_source == 'cloud':
        backup_path = f"/tmp/{backup_file}"
        if not os.path.exists(backup_path):
            from modules.cloud_backup import _get_r2_client, _get_r2_config
            try:
                cfg    = _get_r2_config()
                bucket = cfg["bucket_name"]
                key    = cloud_key or f"backups/{backup_file}"
                client = _get_r2_client()
                client.download_file(bucket, key, backup_path)
            except Exception as e:
                return jsonify({'error': f'Failed to download from R2: {e}'}), 500

    elif backup_source == 'local':
        if RUNNING_ON_MAIN_SERVER:
            backup_path = f"/root/backups/{backup_file}"
            if not os.path.exists(backup_path):
                return jsonify({'error': f'Not found: {backup_path}'}), 400
        else:
            backup_path = f"/tmp/{backup_file}"
            if not os.path.exists(backup_path):
                pull_cmd = (
                    f"scp -i {MAIN_SERVER_KEY} -P {MAIN_SERVER_SSH_PORT} "
                    f"-o StrictHostKeyChecking=no -o ConnectTimeout=15 "
                    f"{MAIN_SERVER_USER}@{MAIN_SERVER_SSH_HOST}:/root/backups/{backup_file} "
                    f"{backup_path}"
                )
                res = subprocess.run(pull_cmd, shell=True, capture_output=True, text=True)
                if res.returncode != 0:
                    return jsonify({'error': f'Failed to pull from main server: {res.stderr}'}), 500
    else:  # vm
        if RUNNING_ON_MAIN_SERVER:
            backup_path = f"/tmp/{backup_file}"
            if not os.path.exists(backup_path):
                pull_cmd = (
                    f"scp -i {VM_KEY} -P {VM_PORT} "
                    f"-o StrictHostKeyChecking=no -o ConnectTimeout=15 "
                    f"{VM_USER}@{VM_HOST}:/backups/main-server/{backup_file} "
                    f"{backup_path}"
                )
                res = subprocess.run(pull_cmd, shell=True, capture_output=True, text=True)
                if res.returncode != 0:
                    return jsonify({'error': f'Failed to pull from VM: {res.stderr}'}), 500
        else:
            backup_path = f"/backups/main-server/{backup_file}"
            if not os.path.exists(backup_path):
                return jsonify({'error': f'Not found: {backup_path}'}), 400

    # Absolute /root path (not image-relative like the old restore-myapps.sh
    # was) — backup/restore-k8s-apps.sh lives in the CloudOps repo's backup/
    # folder, a sibling of platform/, so it's never baked into the Docker
    # image; it's reached the same way api_backup_run's script is, via the
    # pod's existing rw hostPath mount of /root.
    restore_script_local = '/root/CloudOps/backup/restore-k8s-apps.sh'
    if not os.path.exists(restore_script_local):
        return jsonify({'error': f'restore-k8s-apps.sh not found at {restore_script_local}'}), 500

    if target == 'local':
        if not RUNNING_ON_MAIN_SERVER:
            return jsonify({
                'error': 'This is the standby — there is no local cluster here to restore '
                         'into. Use "External Machine" to restore onto the real main server '
                         'over the tunnel instead.'
            }), 400
        hostname    = os.uname().nodename
        os.makedirs('/root/tmp', exist_ok=True)
        session_dir = f"/root/tmp/restore-session-{uuid.uuid4().hex[:8]}"
        cmd = (
            f"set -e && "
            f"echo 'Restoring on this server ({hostname})...' && "
            f"echo 'Apps: {apps_label}' && "
            f"mkdir -p {session_dir} && "
            f"echo 'Extracting backup...' && "
            f"tar -xzf {backup_path} -C {session_dir} --strip-components=1 && "
            f"cp {restore_script_local} {session_dir}/restore-k8s-apps.sh && "
            f"chmod +x {session_dir}/restore-k8s-apps.sh && "
            f"cd {session_dir} && bash restore-k8s-apps.sh{apps_flag} ; "
            f"EXIT=$? ; rm -rf {session_dir} ; exit $EXIT"
        )
    else:
        if not remote_ip:
            return jsonify({'error': 'remote_ip required'}), 400

        # Unified path: every "External Machine" restore goes through the
        # same Ansible playbook used for from-scratch DR (ansible/dr-
        # bootstrap.yml) rather than a bare scp+ssh+restore-k8s-apps.sh
        # dance. It's safe for BOTH cases, not just the empty-server one:
        # the playbook's own steps (create-namespace-if-missing, `kubectl
        # apply` for the sanitized PVC/Secret/manifests) are no-ops against
        # a target that already has this app running with a matching spec
        # — apply only reconciles differences, and a backup's own captured
        # manifests are by definition identical to what's already live —
        # so "restore onto an existing cluster" and "restore onto nothing"
        # are the same command now; the playbook's own checks-then-acts
        # steps decide how much of it actually needs to do anything.
        ansible_dir = '/root/CloudOps/ansible'
        playbook_path = f'{ansible_dir}/dr-bootstrap.yml'
        if not os.path.exists(playbook_path):
            return jsonify({'error': f'Ansible playbook not found at {playbook_path}'}), 500

        # ANSIBLE_HOST_KEY_CHECKING=False (env var, below) rather than
        # -e ansible_ssh_common_args='-o StrictHostKeyChecking=no ...':
        # passing that as an extra-var hits what looks like a real bug in
        # this ansible-core version's SSH connection plugin (its own
        # internal tty-detection re-parses the string with a strict
        # argparse and throws "argument -o: expected one argument" even
        # for a single well-formed -o KEY=VALUE) — reproduced directly on
        # the CLI, not a quoting artifact from this code. The env var
        # achieves the same effect (disables host-key checking) through
        # ansible's own dedicated option instead, bypassing that path
        # entirely.
        common_evars = [
            f"ansible_user={remote_user}",
            f"ansible_port={remote_port}",
        ]
        if auth_method == 'key':
            if not ssh_key_path:
                return jsonify({'error': 'ssh_key_path required'}), 400
            common_evars.append(f"ansible_ssh_private_key_file={ssh_key_path}")
        else:
            if not ssh_password:
                return jsonify({'error': 'ssh_password required'}), 400
            common_evars.append(f"ansible_password='{ssh_password}'")

        evars_str = " ".join(f"-e {v}" for v in common_evars)
        # ansible-playbook is a pip console-script installed alongside
        # whichever python is actually running this process — the system
        # one in the main pod's Docker image (no venv there), or this
        # server's own venv/bin on the standby. A bare "ansible-playbook"
        # in the command string only resolves on the main pod (system PATH
        # already includes it); on the standby, running a venv's python
        # directly (not through `activate`) never puts venv/bin on PATH,
        # so it'd fail with "command not found" there. Resolving it
        # relative to sys.executable works in both.
        ansible_playbook_bin = os.path.join(os.path.dirname(sys.executable), 'ansible-playbook')

        # dr-bootstrap.yml restores one app per run (it reads that app's own
        # manifests.yaml to work out the namespace) — loop for a multi-app
        # selection, each run against the same already-staged backup_path.
        apps_for_ansible = selected_apps if selected_apps else sorted(ALLOWED_RESTORE_APPS)
        ansible_env = "ANSIBLE_HOST_KEY_CHECKING=False ANSIBLE_TIMEOUT=15"
        per_app_cmds = [
            f"echo '=== [{a}] provisioning (if needed) + restore via Ansible ===' && "
            f"{ansible_env} {ansible_playbook_bin} -i '{remote_ip},' {evars_str} "
            f"-e dr_app={a} -e dr_backup_archive={backup_path} {playbook_path}"
            for a in apps_for_ansible
        ]
        cmd = (
            f"echo 'Target: {remote_user}@{remote_ip}:{remote_port}' && "
            f"echo 'Apps: {apps_label}' && "
            + " && ".join(per_app_cmds)
        )

    job_id = str(uuid.uuid4())
    t = threading.Thread(target=_stream_restore, args=(job_id, cmd), daemon=True)
    t.start()
    return jsonify({'job_id': job_id, 'status': 'started'})


@app.route('/restore/status/<job_id>')
@login_required
def restore_status_poll(job_id):
    job = restore_jobs.get(job_id)
    if not job:
        return jsonify({'error': 'Job not found'}), 404
    return jsonify({
        'status':  job['status'],
        'log':     job['log'],
        'elapsed': round(time.time() - job.get('started', time.time()))
    })


# ─────────────────────────────────────────────
# SERVER STATUS
# ─────────────────────────────────────────────

@app.route('/server/status')
@login_required
def server_status():
    stdout, stderr = run_command("uptime")
    if stderr or not stdout:
        return jsonify({'status': 'offline', 'error': stderr or 'Failed'})
    return jsonify({'status': 'online', 'info': stdout.strip()})


# ─────────────────────────────────────────────
# AUTH
# ─────────────────────────────────────────────

@app.route('/login', methods=['GET', 'POST'])
def login():
    error = ''
    if request.method == 'POST':
        if request.form.get('password') == 'admin123':
            session['logged_in'] = True
            return redirect(url_for('dashboard'))
        error = 'Wrong password'
    return render_template('login.html', error=error)


@app.route('/logout')
def logout():
    session.pop('logged_in', None)
    return redirect(url_for('login'))


if __name__ == '__main__':
    app.run(host='0.0.0.0', port=5000, debug=False)