#!/bin/bash # ============================================= # restore-myapps.sh — Smart Restore Script # ============================================= set -uo pipefail # docker compose v2 compat command -v docker-compose &>/dev/null || docker-compose() { docker compose "$@"; } # -------------------------------------------------- # Parse arguments # -------------------------------------------------- REMOTE_MODE=false REMOTE_IP="" REMOTE_USER="root" SSH_KEY="" USE_PASSWORD=false APPS_FILTER="all" # "all" or comma-separated: frappe,odoo,nextcloud,mautic,n8n VALID_APPS="frappe odoo nextcloud mautic n8n" while [[ $# -gt 0 ]]; do case "$1" in --remote) REMOTE_MODE=true REMOTE_IP="$2" REMOTE_USER="${3:-root}" shift 3 ;; --key) SSH_KEY="$2" shift 2 ;; --password) USE_PASSWORD=true shift ;; --apps) APPS_FILTER="$2" shift 2 ;; *) shift ;; esac done # -------------------------------------------------- # Normalize + validate --apps # -------------------------------------------------- if [ "$APPS_FILTER" != "all" ]; then APPS_FILTER=$(echo "$APPS_FILTER" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]') IFS=',' read -ra _APP_CHECK <<< "$APPS_FILTER" for _a in "${_APP_CHECK[@]}"; do if [[ " $VALID_APPS " != *" $_a "* ]]; then echo "❌ Unknown app '$_a' in --apps. Valid options: $VALID_APPS" exit 1 fi done fi # Returns 0 (true) if $1 should be restored given APPS_FILTER app_selected() { local app="$1" [ "$APPS_FILTER" = "all" ] && return 0 [[ ",$APPS_FILTER," == *",${app},"* ]] } # -------------------------------------------------- # Remote mode # -------------------------------------------------- if [ "$REMOTE_MODE" = true ]; then if [ -z "$REMOTE_IP" ]; then echo "❌ --remote requires an IP address." exit 1 fi SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REMOTE_DEST="/tmp/restore-session-$(date +%s)" SSH_OPTS="-o StrictHostKeyChecking=no -o ConnectTimeout=15" if [ -n "$SSH_KEY" ]; then SSH_OPTS="$SSH_OPTS -i $SSH_KEY" elif [ "$USE_PASSWORD" = true ]; then if ! command -v sshpass &>/dev/null; then echo "❌ sshpass not installed. Install it with: apt install sshpass" exit 1 fi read -s -p "SSH password for ${REMOTE_USER}@${REMOTE_IP}: " SSH_PASS echo "" SSH_CMD="sshpass -p '$SSH_PASS' ssh $SSH_OPTS ${REMOTE_USER}@${REMOTE_IP}" SCP_CMD="sshpass -p '$SSH_PASS' scp $SSH_OPTS" fi SSH_CMD="ssh $SSH_OPTS ${REMOTE_USER}@${REMOTE_IP}" SCP_CMD="scp $SSH_OPTS" if [ -n "$SSH_KEY" ]; then SCP_CMD="scp -i $SSH_KEY $SSH_OPTS" fi echo "=========================================" echo "📡 REMOTE RESTORE MODE" echo " Target: ${REMOTE_USER}@${REMOTE_IP}" if [ "$APPS_FILTER" = "all" ]; then echo " Apps: ALL" else echo " Apps: $APPS_FILTER" fi echo "=========================================" REMOTE_ARGS="" if [ "$APPS_FILTER" != "all" ]; then REMOTE_ARGS="--apps $APPS_FILTER" fi $SSH_CMD "mkdir -p $REMOTE_DEST" $SCP_CMD -r "$SCRIPT_DIR/." "${REMOTE_USER}@${REMOTE_IP}:${REMOTE_DEST}/" $SSH_CMD "chmod +x $REMOTE_DEST/restore-myapps.sh && cd $REMOTE_DEST && bash restore-myapps.sh $REMOTE_ARGS" $SSH_CMD "rm -rf $REMOTE_DEST" echo "✅ Remote restore complete on $REMOTE_IP" exit 0 fi # =================================================== # LOCAL RESTORE # =================================================== SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" cd "$SCRIPT_DIR" # Detect IP — works inside containers and on host VM_IP=$(hostname -I 2>/dev/null | awk '{print $1}') [ -z "$VM_IP" ] && VM_IP=$(ip -4 addr show 2>/dev/null | grep -oP '(?<=inet\s)\d+(\.\d+){3}' | grep -v 127.0.0.1 | head -1) [ -z "$VM_IP" ] && VM_IP="173.249.20.244" echo "=========================================" echo "🔄 Smart Restore — LOCAL MODE" echo " Machine IP: $VM_IP" echo " Backup dir: $SCRIPT_DIR" if [ "$APPS_FILTER" = "all" ]; then echo " Apps: ALL" else echo " Apps: $APPS_FILTER (forced — ignoring health checks)" fi echo "=========================================" container_is_healthy() { local name="$1" local status status=$(docker inspect --format='{{.State.Status}}' "$name" 2>/dev/null || echo "missing") local health health=$(docker inspect --format='{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$name" 2>/dev/null || echo "none") if [ "$status" = "running" ] && { [ "$health" = "healthy" ] || [ "$health" = "none" ]; }; then return 0 fi return 1 } # -------------------------------------------------- # NEW: Detect a crash-looping Postgres container and # attempt automatic recovery via pg_resetwal if the # logs show WAL/checkpoint corruption. This is a safety # net — the real fix is that backups now stop DB # containers before tar'ing their volumes — but this # protects against older/already-corrupt backups too. # -------------------------------------------------- repair_postgres_if_crashlooping() { local ctr="$1" local vol="$2" [ -z "$ctr" ] || [ -z "$vol" ] && return 0 docker ps -a --format '{{.Names}}' | grep -qx "$ctr" || return 0 sleep 8 # give it a moment to attempt startup local restart_count status restart_count=$(docker inspect --format='{{.RestartCount}}' "$ctr" 2>/dev/null || echo 0) status=$(docker inspect --format='{{.State.Status}}' "$ctr" 2>/dev/null || echo "missing") if [ "$status" = "restarting" ] || [ "$restart_count" -gt 2 ]; then echo " ⚠️ $ctr is crash-looping (status=$status, restarts=$restart_count)" local last_log last_log=$(docker logs "$ctr" --tail 30 2>&1) if echo "$last_log" | grep -qi "could not locate a valid checkpoint record\|invalid primary checkpoint\|unexpected pageaddr"; then echo " 🔧 Detected WAL corruption — attempting pg_resetwal recovery on volume '$vol'..." docker stop "$ctr" &>/dev/null || true docker run --rm -v "${vol}:/var/lib/postgresql/data" postgres:15 \ bash -c "chown -R postgres:postgres /var/lib/postgresql/data && \ su postgres -c 'pg_resetwal -f /var/lib/postgresql/data'" \ && echo " ✅ pg_resetwal completed" \ || echo " ❌ pg_resetwal failed — manual intervention needed" docker start "$ctr" &>/dev/null || true sleep 8 status=$(docker inspect --format='{{.State.Status}}' "$ctr" 2>/dev/null || echo "missing") if [ "$status" = "running" ]; then echo " ✅ $ctr recovered and running (note: pg_resetwal can drop the most recent transactions — verify your data)" else echo " ❌ $ctr still not healthy after pg_resetwal — check 'docker logs $ctr' manually. DATA MAY BE LOST." fi else echo " ❌ $ctr crash-looping for a reason other than WAL corruption — check 'docker logs $ctr' manually." fi fi } # -------------------------------------------------- # STEP 1: Restore Volumes # -------------------------------------------------- echo "" echo "=========================================" echo "📦 STEP 1 — Restoring Volumes" echo "=========================================" declare -A VOLUME_OWNERS=( ["frappe-setup_frappe-sites"]="frappe-erpnext" ["frappe-setup_mariadb-data"]="frappe-mariadb" ["nextcloud-setup_nextcloud-data"]="nextcloud-app" ["nextcloud-setup_nextcloud-db-data"]="nextcloud-postgres" ["mautic-setup_mautic-data"]="mautic-app" ["mautic-setup_mautic-db-data"]="mautic-mariadb" ["n8n-setup_n8n-data"]="n8n-app" ["n8n-setup_n8n-db-data"]="n8n-postgres" ["odoo-clean_db-data"]="odoo-clean-db-1" ["odoo-clean_odoo-etc"]="odoo-clean-odoo-1" ) declare -A VOLUME_APP=( ["frappe-setup_frappe-sites"]="frappe" ["frappe-setup_mariadb-data"]="frappe" ["nextcloud-setup_nextcloud-data"]="nextcloud" ["nextcloud-setup_nextcloud-db-data"]="nextcloud" ["mautic-setup_mautic-data"]="mautic" ["mautic-setup_mautic-db-data"]="mautic" ["n8n-setup_n8n-data"]="n8n" ["n8n-setup_n8n-db-data"]="n8n" ["odoo-clean_db-data"]="odoo" ["odoo-clean_odoo-etc"]="odoo" ) if [ -d "$SCRIPT_DIR/volumes" ]; then cd "$SCRIPT_DIR/volumes" for backup in *.tar.gz; do [ -f "$backup" ] || continue volume=$(basename "$backup" .tar.gz) owner="${VOLUME_OWNERS[$volume]:-}" vol_app="${VOLUME_APP[$volume]:-}" if [ -n "$vol_app" ] && ! app_selected "$vol_app"; then echo " ⏭️ $volume — app '$vol_app' not selected, skipping" continue fi # Health-based protection only applies on full ("all") runs. # An explicit --apps selection means the user wants this restored # regardless of what the health check currently reports. if [ "$APPS_FILTER" = "all" ] && [ -n "$owner" ] && container_is_healthy "$owner"; then echo " ⏭️ $volume — container '$owner' is healthy, skipping" continue fi echo -n " 📁 Restoring $volume ... " docker volume create "$volume" &>/dev/null || true docker run --rm \ -v "${volume}:/target" \ -v "$(pwd):/backup:ro" \ alpine \ sh -c "cd /target && tar xzf /backup/${backup}" \ && echo "✅" || echo "⚠️ FAILED" done cd "$SCRIPT_DIR" else echo " ⚠️ No volumes/ directory found — skipping" fi # -------------------------------------------------- # STEP 2: Start Containers # -------------------------------------------------- echo "" echo "=========================================" echo "🚀 STEP 2 — Starting Containers" echo "=========================================" declare -A APP_DIRS=( ["Frappe"]="frappe-setup" ["Odoo"]="odoo-clean" ["Nextcloud"]="nextcloud-setup" ["Mautic"]="mautic-setup" ["n8n"]="n8n-setup" ) declare -A APP_MAIN_CONTAINER=( ["Frappe"]="frappe-erpnext" ["Odoo"]="odoo-clean-odoo-1" ["Nextcloud"]="nextcloud-app" ["Mautic"]="mautic-app" ["n8n"]="n8n-app" ) declare -A APP_KEY=( ["Frappe"]="frappe" ["Odoo"]="odoo" ["Nextcloud"]="nextcloud" ["Mautic"]="mautic" ["n8n"]="n8n" ) # Postgres-backed apps: map app key -> "container:volume" for the # crash-loop auto-repair check run right after each app starts. declare -A APP_POSTGRES=( ["n8n"]="n8n-postgres:n8n-setup_n8n-db-data" ["nextcloud"]="nextcloud-postgres:nextcloud-setup_nextcloud-db-data" ["odoo"]="odoo-clean-db-1:odoo-clean_db-data" ) if [ -d "$SCRIPT_DIR/compose-files" ]; then cd "$SCRIPT_DIR/compose-files" for app in Frappe Odoo Nextcloud Mautic n8n; do app_key="${APP_KEY[$app]}" dir="${APP_DIRS[$app]}" main_ctr="${APP_MAIN_CONTAINER[$app]}" if ! app_selected "$app_key"; then echo " ⏭️ $app — not selected, skipping" continue fi if [ ! -d "$dir" ]; then echo " ⏭️ $app compose dir '$dir' not found, skipping" continue fi if [ "$APPS_FILTER" = "all" ] && container_is_healthy "$main_ctr"; then echo " ⏭️ $app — already running and healthy, skipping" continue fi cd "$dir" if [ "$APPS_FILTER" != "all" ] && container_is_healthy "$main_ctr"; then # Already running — up -d alone won't reload the freshly restored # volume, so force a recreate. echo " 🔁 $app — explicitly selected, recreating to load restored data..." docker-compose down 2>&1 | tail -3 else echo " 🚀 Starting $app..." fi docker-compose up -d 2>&1 | tail -3 cd .. # If this app has a Postgres DB, check it didn't come up crash-looping. pg_spec="${APP_POSTGRES[$app_key]:-}" if [ -n "$pg_spec" ]; then pg_ctr="${pg_spec%%:*}" pg_vol="${pg_spec##*:}" repair_postgres_if_crashlooping "$pg_ctr" "$pg_vol" fi done cd "$SCRIPT_DIR" else echo " ⚠️ No compose-files/ directory — skipping" fi echo "" echo "⏳ Waiting 60s for containers to initialize..." sleep 60 # -------------------------------------------------- # STEP 3: Post-Restore Fixes # -------------------------------------------------- echo "" echo "=========================================" echo "🔧 STEP 3 — Applying Post-Restore Fixes" echo "=========================================" # ---- NEXTCLOUD ---- echo "" echo "📌 Nextcloud — Trusted domains..." if ! app_selected nextcloud; then echo " ⏭️ Nextcloud not selected, skipping" elif [ "$APPS_FILTER" = "all" ] && container_is_healthy nextcloud-app; then echo " ⏭️ Nextcloud is healthy, skipping fix" else if docker ps | grep -q nextcloud-app; then docker exec nextcloud-app php /var/www/html/occ config:system:delete trusted_domains 1 2>/dev/null || true docker exec nextcloud-app php /var/www/html/occ config:system:delete trusted_domains 2 2>/dev/null || true docker exec nextcloud-app php /var/www/html/occ config:system:set trusted_domains 0 --value="localhost" docker exec nextcloud-app php /var/www/html/occ config:system:set trusted_domains 1 --value="$VM_IP" docker exec nextcloud-app php /var/www/html/occ config:system:set trusted_domains 2 --value="${VM_IP}:8082" docker exec nextcloud-app php /var/www/html/occ config:system:set trusted_domains 3 --value="localhost:8082" docker exec nextcloud-postgres psql -U nextcloud -d nextcloud -c " GRANT ALL PRIVILEGES ON ALL TABLES IN SCHEMA public TO oc_admin; GRANT ALL PRIVILEGES ON ALL SEQUENCES IN SCHEMA public TO oc_admin; " 2>/dev/null || true docker restart nextcloud-app echo " ✅ Nextcloud fixed" else echo " ⚠️ nextcloud-app not running" fi fi # ---- MAUTIC ---- echo "" echo "📌 Mautic — Config + admin password..." if ! app_selected mautic; then echo " ⏭️ Mautic not selected, skipping" elif [ "$APPS_FILTER" = "all" ] && container_is_healthy mautic-app; then echo " ⏭️ Mautic is healthy, skipping fix" else if docker ps | grep -q mautic-app; then cat > /tmp/mautic-local.php << EOF 'pdo_mysql', 'db_host' => 'mautic-mariadb', 'db_port' => '3306', 'db_name' => 'mautic', 'db_user' => 'mautic', 'db_password' => 'mautic123', 'db_table_prefix' => '', 'site_url' => 'http://${VM_IP}:8081' ); EOF docker cp /tmp/mautic-local.php mautic-app:/var/www/html/config/local.php docker exec mautic-app touch /var/www/html/var/.installed 2>/dev/null || true docker exec mautic-app chown -R www-data:www-data /var/www/html/var /var/www/html/config 2>/dev/null || true docker restart mautic-app sleep 10 HASH=$(docker exec mautic-app php -r "echo password_hash('Admin!Password123', PASSWORD_BCRYPT);" 2>/dev/null || true) if [ -n "$HASH" ]; then docker exec mautic-mariadb mysql -uroot -pmautic_root_password \ -e "USE mautic; UPDATE users SET password = '$HASH' WHERE username = 'admin';" 2>/dev/null || true echo " ✅ Admin password → Admin!Password123" fi docker restart mautic-app echo " ✅ Mautic fixed → http://${VM_IP}:8081/s/login" else echo " ⚠️ mautic-app not running" fi fi # ---- ODOO ---- echo "" echo "📌 Odoo — Assets + DB + Config..." if ! app_selected odoo; then echo " ⏭️ Odoo not selected, skipping" elif [ "$APPS_FILTER" = "all" ] && container_is_healthy odoo-clean-odoo-1; then echo " ⏭️ Odoo is healthy, skipping fix" else if docker ps | grep -q odoo-clean-db-1; then # Fix odoo.conf with proper settings docker run --rm -v odoo-clean_odoo-etc:/etc/odoo alpine sh -c 'cat > /etc/odoo/odoo.conf << CONF [options] addons_path = /mnt/extra-addons data_dir = /var/lib/odoo db_host = db db_user = odoo db_password = odoo db_name = odoo workers = 4 max_cron_threads = 1 proxy_mode = True assets_bundle_enabled = True limit_memory_hard = 2684354560 limit_memory_soft = 2147483648 CONF' 2>/dev/null || true docker exec odoo-clean-db-1 psql -U odoo -d odoo \ -c "DELETE FROM ir_attachment WHERE url LIKE '/web/assets/%';" 2>/dev/null || true docker exec odoo-clean-db-1 psql -U odoo -d odoo \ -c "UPDATE ir_config_parameter SET value='https://odooo.nav.ovh' WHERE key='web.base.url';" 2>/dev/null || true docker exec odoo-clean-db-1 psql -U odoo \ -c "ALTER USER odoo WITH PASSWORD 'odoo';" 2>/dev/null || true docker restart odoo-clean-odoo-1 echo " ✅ Odoo fixed → http://${VM_IP}:8069/web" else echo " ⚠️ odoo-clean-db-1 not running" fi fi # ---- FRAPPE ---- echo "" echo "📌 Frappe — DB permissions + cache + URL..." if ! app_selected frappe; then echo " ⏭️ Frappe not selected, skipping" elif [ "$APPS_FILTER" = "all" ] && container_is_healthy frappe-erpnext; then echo " ⏭️ Frappe is healthy, skipping fix" else if docker ps | grep -q frappe-erpnext && docker ps | grep -q frappe-mariadb; then SITE_CONFIG="/home/frappe/frappe-bench/sites/erpnext.navitrends.ovh/site_config.json" DB_NAME=$(docker exec frappe-erpnext cat "$SITE_CONFIG" 2>/dev/null | grep -o '"db_name": *"[^"]*"' | cut -d'"' -f4) DB_PASS=$(docker exec frappe-erpnext cat "$SITE_CONFIG" 2>/dev/null | grep -o '"db_password": *"[^"]*"' | cut -d'"' -f4) if [ -n "$DB_NAME" ] && [ -n "$DB_PASS" ]; then docker exec frappe-mariadb mysql -uroot -p123 -e " GRANT ALL PRIVILEGES ON *.* TO '${DB_NAME}'@'%' IDENTIFIED BY '${DB_PASS}' WITH GRANT OPTION; GRANT ALL PRIVILEGES ON *.* TO '${DB_NAME}'@'172.%' IDENTIFIED BY '${DB_PASS}' WITH GRANT OPTION; FLUSH PRIVILEGES; " 2>/dev/null && echo " ✅ DB permissions fixed" || echo " ⚠️ DB grant failed" docker exec frappe-erpnext bash -c " cd /home/frappe/frappe-bench bench --site erpnext.navitrends.ovh set-config redis_cache 'redis://frappe-redis:6379' bench --site erpnext.navitrends.ovh set-config redis_queue 'redis://frappe-redis:6379' bench --site erpnext.navitrends.ovh set-config enable_scheduler 1 bench --site erpnext.navitrends.ovh set-config site_url 'http://${VM_IP}:8080' bench --site erpnext.navitrends.ovh migrate bench --site erpnext.navitrends.ovh clear-cache " 2>/dev/null && echo " ✅ Frappe configured" || echo " ⚠️ Frappe config step had errors" docker restart frappe-erpnext echo " ✅ Frappe fixed → http://${VM_IP}:8080" else echo " ⚠️ Could not read Frappe DB credentials" fi else echo " ⚠️ Frappe containers not running" fi fi # ---- N8N ---- echo "" echo "📌 n8n — Network check..." if ! app_selected n8n; then echo " ⏭️ n8n not selected, skipping" elif [ "$APPS_FILTER" = "all" ] && container_is_healthy n8n-app; then echo " ⏭️ n8n is healthy, skipping fix" else docker network inspect integration-network &>/dev/null \ || { docker network create integration-network && echo " ✅ Created integration-network"; } if ! docker ps | grep -q n8n-app; then if [ -d "$SCRIPT_DIR/compose-files/n8n-setup" ]; then cd "$SCRIPT_DIR/compose-files/n8n-setup" docker-compose up -d 2>&1 | tail -3 cd "$SCRIPT_DIR" else docker start n8n-app 2>/dev/null && echo " ✅ Started n8n-app" || echo " ⚠️ Could not start n8n" fi fi # Re-check postgres after this section's potential restart too. repair_postgres_if_crashlooping "n8n-postgres" "n8n-setup_n8n-db-data" if container_is_healthy n8n-postgres; then echo " ✅ n8n → http://${VM_IP}:5678" else echo " ⚠️ n8n-app is up but n8n-postgres is NOT healthy — check: docker logs n8n-postgres" fi fi # -------------------------------------------------- # Summary # -------------------------------------------------- echo "" echo "=========================================" echo "✅ RESTORE COMPLETE" echo "=========================================" summarize_app() { local label="$1" url="$2" db_ctr="$3" if [ -n "$db_ctr" ] && ! container_is_healthy "$db_ctr"; then echo " ${label} → ⚠️ DB NOT healthy ($db_ctr) — check: docker logs $db_ctr" else echo " ${label} → ${url}" fi } if app_selected nextcloud; then summarize_app "Nextcloud" "http://${VM_IP}:8082" "nextcloud-postgres"; fi if app_selected mautic; then summarize_app "Mautic " "http://${VM_IP}:8081/s/login (admin / Admin!Password123)" "mautic-mariadb"; fi if app_selected odoo; then summarize_app "Odoo " "https://odooo.nav.ovh/web" "odoo-clean-db-1"; fi if app_selected n8n; then summarize_app "n8n " "http://${VM_IP}:5678" "n8n-postgres"; fi if app_selected frappe; then summarize_app "Frappe " "http://${VM_IP}:8080" "frappe-mariadb"; fi echo "========================================="