Daniel Bedeleanu
fc149184e9
- Created OPERATIONAL_RUNBOOK.md: comprehensive step-by-step procedures for both Docker and Standalone deployment modes covering deployment, daily ops, troubleshooting, backup/restore, disaster recovery, scaling, and updates - Created HEALTH_MONITORING_CHECKLIST.md: daily/weekly/monthly health check procedures with alert thresholds and quick troubleshooting reference - Created DISASTER_RECOVERY_PLAN.md: detailed procedures for 6 failure scenarios (database corruption, hardware failure, data center failure, app crash, disk full, network isolation) with RTO/RPO targets - Created CONFIGURATION_REFERENCE.md: complete documentation of all inventory.env parameters for both deployment modes with common scenarios and troubleshooting - Created EMERGENCY_PROCEDURES.md: quick-reference incident response playbook with 7 critical scenarios, decision tree, escalation path, and printable cheat sheet - Created scripts/backup.sh: automated backup script supporting both Docker and Standalone with integrity verification and retention management - Created scripts/restore.sh: restore script with triple confirmation, safety backups, and validation tests for both deployment modes - Created config/backup-cron.sh: installer for daily/weekly automated backup cron jobs (2 AM daily, 3 AM Sunday) All documentation covers dual-deployment modes with shared configuration files. Documentation is operator-ready with copy-paste commands and clear expected outputs.
129 lines
3.6 KiB
Bash
129 lines
3.6 KiB
Bash
#!/bin/bash
|
|
set -euo pipefail
|
|
|
|
# Phase 6, Plan 02, Task 1: Automated Backup Script
|
|
# Supports both Docker and Standalone deployment modes
|
|
# Usage: ./backup.sh [daily|weekly|manual] [retention_days]
|
|
|
|
BACKUP_TYPE="${1:-manual}"
|
|
RETENTION_DAYS="${2:-30}"
|
|
BACKUP_DIR="./backups"
|
|
DATA_DIR="./data"
|
|
TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S)
|
|
BACKUP_FILE="$BACKUP_DIR/inventory-$TIMESTAMP.tar.gz"
|
|
|
|
# Colors
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
NC='\033[0m'
|
|
|
|
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
|
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
|
log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
|
|
|
|
# Create backup directory
|
|
mkdir -p "$BACKUP_DIR"
|
|
|
|
# Determine deployment mode
|
|
if command -v docker-compose &> /dev/null; then
|
|
log_info "Docker deployment mode detected"
|
|
IS_DOCKER=true
|
|
|
|
# Verify Docker daemon is running
|
|
docker ps > /dev/null 2>&1 || log_error "Docker daemon not running"
|
|
|
|
# 1. Verify services are running
|
|
log_info "Checking services..."
|
|
if ! docker-compose ps | grep -q "Up"; then
|
|
log_warn "Not all services running; attempting to start..."
|
|
docker-compose up -d
|
|
fi
|
|
|
|
# 2. Stop backend to ensure DB consistency
|
|
log_info "Stopping backend service (DB consistency)..."
|
|
docker-compose stop backend
|
|
else
|
|
log_info "Standalone deployment mode detected"
|
|
IS_DOCKER=false
|
|
|
|
# For standalone mode, we'll just backup while running
|
|
# (Process is single-threaded, so minimal locking risk)
|
|
log_warn "Standalone mode: backing up with minimal service pause"
|
|
fi
|
|
|
|
# Wait for graceful shutdown (Docker only)
|
|
[[ "$IS_DOCKER" == "true" ]] && sleep 2
|
|
|
|
# 3. Create backup tarball
|
|
log_info "Creating backup: $BACKUP_FILE"
|
|
|
|
tar --exclude="$DATA_DIR/caddy_*" \
|
|
--exclude="$BACKUP_DIR" \
|
|
--exclude=".git" \
|
|
--exclude="node_modules" \
|
|
--exclude=".next" \
|
|
--exclude=".venv" \
|
|
--exclude="__pycache__" \
|
|
-czf "$BACKUP_FILE" \
|
|
"$DATA_DIR/inventory.db" \
|
|
"$DATA_DIR/inventory.db-wal" \
|
|
"$DATA_DIR/inventory.db-shm" \
|
|
"config/" \
|
|
"inventory.env" \
|
|
2>/dev/null || log_error "Backup creation failed"
|
|
|
|
# Restart backend (Docker only)
|
|
if [[ "$IS_DOCKER" == "true" ]]; then
|
|
log_info "Restarting backend service..."
|
|
docker-compose start backend
|
|
|
|
# Wait for backend to be ready
|
|
sleep 5
|
|
|
|
# Verify backend is healthy
|
|
if curl -sf "http://localhost:8000/health" > /dev/null; then
|
|
log_info "Backend restored and healthy"
|
|
else
|
|
log_warn "Backend not responding yet; check logs with 'docker-compose logs -f backend'"
|
|
fi
|
|
fi
|
|
|
|
# 4. Verify backup integrity
|
|
log_info "Verifying backup integrity..."
|
|
if tar -tzf "$BACKUP_FILE" > /dev/null 2>&1; then
|
|
log_info "✓ Backup verified"
|
|
else
|
|
log_error "Backup corrupted"
|
|
fi
|
|
|
|
# 5. Log backup metadata
|
|
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
|
|
BACKUP_VERSION=$(cat VERSION.json 2>/dev/null | grep version | cut -d'"' -f4 || echo "unknown")
|
|
log_info "Backup size: $BACKUP_SIZE, Version: $BACKUP_VERSION"
|
|
|
|
# 6. Cleanup old backups
|
|
log_info "Cleaning up backups older than $RETENTION_DAYS days..."
|
|
find "$BACKUP_DIR" -name "inventory-*.tar.gz" -type f -mtime +$RETENTION_DAYS -delete
|
|
|
|
# 7. Create backup manifest
|
|
cat > "$BACKUP_DIR/MANIFEST.txt" << EOF
|
|
Backup Metadata
|
|
===============
|
|
Timestamp: $TIMESTAMP
|
|
Type: $BACKUP_TYPE
|
|
Retention: $RETENTION_DAYS days
|
|
Size: $BACKUP_SIZE
|
|
Version: $BACKUP_VERSION
|
|
Deployment Mode: $([ "$IS_DOCKER" == "true" ] && echo "Docker" || echo "Standalone")
|
|
Contents: DB, config, certificates (excluding certs)
|
|
Restored: Not yet
|
|
|
|
Command to restore:
|
|
./restore.sh $BACKUP_FILE
|
|
EOF
|
|
|
|
log_info "Backup complete: $BACKUP_FILE"
|
|
log_info "Retention policy: Delete after $RETENTION_DAYS days"
|
|
log_info "Next backup: $(date -d '+1 day' '+%Y-%m-%d')"
|