feat(6): phase 6 plan 02 - operational runbook and documentation
- Created OPERATIONAL_RUNBOOK.md: comprehensive step-by-step procedures for both Docker and Standalone deployment modes covering deployment, daily ops, troubleshooting, backup/restore, disaster recovery, scaling, and updates - Created HEALTH_MONITORING_CHECKLIST.md: daily/weekly/monthly health check procedures with alert thresholds and quick troubleshooting reference - Created DISASTER_RECOVERY_PLAN.md: detailed procedures for 6 failure scenarios (database corruption, hardware failure, data center failure, app crash, disk full, network isolation) with RTO/RPO targets - Created CONFIGURATION_REFERENCE.md: complete documentation of all inventory.env parameters for both deployment modes with common scenarios and troubleshooting - Created EMERGENCY_PROCEDURES.md: quick-reference incident response playbook with 7 critical scenarios, decision tree, escalation path, and printable cheat sheet - Created scripts/backup.sh: automated backup script supporting both Docker and Standalone with integrity verification and retention management - Created scripts/restore.sh: restore script with triple confirmation, safety backups, and validation tests for both deployment modes - Created config/backup-cron.sh: installer for daily/weekly automated backup cron jobs (2 AM daily, 3 AM Sunday) All documentation covers dual-deployment modes with shared configuration files. Documentation is operator-ready with copy-paste commands and clear expected outputs.
This commit is contained in:
128
scripts/backup.sh
Normal file
128
scripts/backup.sh
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Phase 6, Plan 02, Task 1: Automated Backup Script
|
||||
# Supports both Docker and Standalone deployment modes
|
||||
# Usage: ./backup.sh [daily|weekly|manual] [retention_days]
|
||||
|
||||
BACKUP_TYPE="${1:-manual}"
|
||||
RETENTION_DAYS="${2:-30}"
|
||||
BACKUP_DIR="./backups"
|
||||
DATA_DIR="./data"
|
||||
TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S)
|
||||
BACKUP_FILE="$BACKUP_DIR/inventory-$TIMESTAMP.tar.gz"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
|
||||
|
||||
# Create backup directory
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Determine deployment mode
|
||||
if command -v docker-compose &> /dev/null; then
|
||||
log_info "Docker deployment mode detected"
|
||||
IS_DOCKER=true
|
||||
|
||||
# Verify Docker daemon is running
|
||||
docker ps > /dev/null 2>&1 || log_error "Docker daemon not running"
|
||||
|
||||
# 1. Verify services are running
|
||||
log_info "Checking services..."
|
||||
if ! docker-compose ps | grep -q "Up"; then
|
||||
log_warn "Not all services running; attempting to start..."
|
||||
docker-compose up -d
|
||||
fi
|
||||
|
||||
# 2. Stop backend to ensure DB consistency
|
||||
log_info "Stopping backend service (DB consistency)..."
|
||||
docker-compose stop backend
|
||||
else
|
||||
log_info "Standalone deployment mode detected"
|
||||
IS_DOCKER=false
|
||||
|
||||
# For standalone mode, we'll just backup while running
|
||||
# (Process is single-threaded, so minimal locking risk)
|
||||
log_warn "Standalone mode: backing up with minimal service pause"
|
||||
fi
|
||||
|
||||
# Wait for graceful shutdown (Docker only)
|
||||
[[ "$IS_DOCKER" == "true" ]] && sleep 2
|
||||
|
||||
# 3. Create backup tarball
|
||||
log_info "Creating backup: $BACKUP_FILE"
|
||||
|
||||
tar --exclude="$DATA_DIR/caddy_*" \
|
||||
--exclude="$BACKUP_DIR" \
|
||||
--exclude=".git" \
|
||||
--exclude="node_modules" \
|
||||
--exclude=".next" \
|
||||
--exclude=".venv" \
|
||||
--exclude="__pycache__" \
|
||||
-czf "$BACKUP_FILE" \
|
||||
"$DATA_DIR/inventory.db" \
|
||||
"$DATA_DIR/inventory.db-wal" \
|
||||
"$DATA_DIR/inventory.db-shm" \
|
||||
"config/" \
|
||||
"inventory.env" \
|
||||
2>/dev/null || log_error "Backup creation failed"
|
||||
|
||||
# Restart backend (Docker only)
|
||||
if [[ "$IS_DOCKER" == "true" ]]; then
|
||||
log_info "Restarting backend service..."
|
||||
docker-compose start backend
|
||||
|
||||
# Wait for backend to be ready
|
||||
sleep 5
|
||||
|
||||
# Verify backend is healthy
|
||||
if curl -sf "http://localhost:8000/health" > /dev/null; then
|
||||
log_info "Backend restored and healthy"
|
||||
else
|
||||
log_warn "Backend not responding yet; check logs with 'docker-compose logs -f backend'"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. Verify backup integrity
|
||||
log_info "Verifying backup integrity..."
|
||||
if tar -tzf "$BACKUP_FILE" > /dev/null 2>&1; then
|
||||
log_info "✓ Backup verified"
|
||||
else
|
||||
log_error "Backup corrupted"
|
||||
fi
|
||||
|
||||
# 5. Log backup metadata
|
||||
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
|
||||
BACKUP_VERSION=$(cat VERSION.json 2>/dev/null | grep version | cut -d'"' -f4 || echo "unknown")
|
||||
log_info "Backup size: $BACKUP_SIZE, Version: $BACKUP_VERSION"
|
||||
|
||||
# 6. Cleanup old backups
|
||||
log_info "Cleaning up backups older than $RETENTION_DAYS days..."
|
||||
find "$BACKUP_DIR" -name "inventory-*.tar.gz" -type f -mtime +$RETENTION_DAYS -delete
|
||||
|
||||
# 7. Create backup manifest
|
||||
cat > "$BACKUP_DIR/MANIFEST.txt" << EOF
|
||||
Backup Metadata
|
||||
===============
|
||||
Timestamp: $TIMESTAMP
|
||||
Type: $BACKUP_TYPE
|
||||
Retention: $RETENTION_DAYS days
|
||||
Size: $BACKUP_SIZE
|
||||
Version: $BACKUP_VERSION
|
||||
Deployment Mode: $([ "$IS_DOCKER" == "true" ] && echo "Docker" || echo "Standalone")
|
||||
Contents: DB, config, certificates (excluding certs)
|
||||
Restored: Not yet
|
||||
|
||||
Command to restore:
|
||||
./restore.sh $BACKUP_FILE
|
||||
EOF
|
||||
|
||||
log_info "Backup complete: $BACKUP_FILE"
|
||||
log_info "Retention policy: Delete after $RETENTION_DAYS days"
|
||||
log_info "Next backup: $(date -d '+1 day' '+%Y-%m-%d')"
|
||||
140
scripts/restore.sh
Normal file
140
scripts/restore.sh
Normal file
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Phase 6, Plan 02, Task 2: Restore from Backup
|
||||
# Supports both Docker and Standalone deployment modes
|
||||
# Usage: ./restore.sh <backup_file> [--validate]
|
||||
|
||||
BACKUP_FILE="${1:-}"
|
||||
VALIDATE="${2:---validate}"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
|
||||
|
||||
# Validate input
|
||||
[[ -z "$BACKUP_FILE" ]] && log_error "Usage: ./restore.sh <backup_file>"
|
||||
[[ ! -f "$BACKUP_FILE" ]] && log_error "Backup file not found: $BACKUP_FILE"
|
||||
|
||||
log_info "Restoring from: $BACKUP_FILE"
|
||||
|
||||
# Triple confirmation (security requirement)
|
||||
echo "⚠️ WARNING: This will overwrite current data!"
|
||||
read -p "Type 'RESTORE' to confirm (1/3): " confirm1
|
||||
[[ "$confirm1" != "RESTORE" ]] && log_error "Restore cancelled"
|
||||
|
||||
read -p "Type 'RESTORE' again to confirm (2/3): " confirm2
|
||||
[[ "$confirm2" != "RESTORE" ]] && log_error "Restore cancelled"
|
||||
|
||||
read -p "Type 'RESTORE' one more time to confirm (3/3): " confirm3
|
||||
[[ "$confirm3" != "RESTORE" ]] && log_error "Restore cancelled"
|
||||
|
||||
log_warn "Proceeding with restore..."
|
||||
|
||||
# Determine deployment mode
|
||||
if command -v docker-compose &> /dev/null; then
|
||||
log_info "Docker deployment mode detected"
|
||||
IS_DOCKER=true
|
||||
|
||||
# 1. Stop services
|
||||
log_info "Stopping services..."
|
||||
docker-compose down
|
||||
|
||||
# 2. Backup current data (safety copy)
|
||||
log_info "Creating safety backup of current data..."
|
||||
mkdir -p data/backups_before_restore
|
||||
tar -czf "data/backups_before_restore/backup-before-restore-$(date +%s).tar.gz" \
|
||||
data/inventory.db data/inventory.db-* 2>/dev/null || true
|
||||
|
||||
# 3. Extract backup
|
||||
log_info "Extracting backup..."
|
||||
tar -xzf "$BACKUP_FILE" -C . || log_error "Backup extraction failed"
|
||||
|
||||
# 4. Verify essential files
|
||||
log_info "Verifying restored files..."
|
||||
[[ -f "data/inventory.db" ]] || log_error "Database not found in backup"
|
||||
[[ -f "inventory.env" ]] || log_error "inventory.env not found in backup"
|
||||
|
||||
# 5. Restart services
|
||||
log_info "Restarting services..."
|
||||
docker-compose up -d
|
||||
|
||||
# 6. Wait for health
|
||||
log_info "Waiting for services to be healthy..."
|
||||
max_attempts=30
|
||||
attempt=0
|
||||
while [[ $attempt -lt $max_attempts ]]; do
|
||||
if curl -sf "http://localhost:8000/health" > /dev/null 2>&1; then
|
||||
log_info "Services healthy!"
|
||||
break
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [[ $attempt -eq $max_attempts ]]; then
|
||||
log_warn "Services did not become healthy within 60 seconds"
|
||||
docker-compose logs backend
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 7. Validation tests (if requested)
|
||||
if [[ "$VALIDATE" == "--validate" ]]; then
|
||||
log_info "Running validation tests..."
|
||||
|
||||
# Test 1: Database accessible
|
||||
DB_ITEMS=$(docker-compose exec backend sqlite3 /app/data/inventory.db \
|
||||
"SELECT COUNT(*) FROM items" 2>/dev/null || echo "0")
|
||||
log_info "✓ Database has $DB_ITEMS items"
|
||||
|
||||
# Test 2: API responsive
|
||||
if curl -sf "http://localhost:8000/health" > /dev/null; then
|
||||
log_info "✓ API health check passed"
|
||||
else
|
||||
log_error "API health check failed"
|
||||
fi
|
||||
|
||||
# Test 3: Frontend loads
|
||||
if curl -sf "http://localhost:3000" > /dev/null 2>&1; then
|
||||
log_info "✓ Frontend loads"
|
||||
else
|
||||
log_warn "⚠ Frontend check failed (normal on first startup)"
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
log_info "Standalone deployment mode detected"
|
||||
IS_DOCKER=false
|
||||
|
||||
# 1. Kill any running servers
|
||||
log_info "Stopping running services..."
|
||||
pkill -f "next start" || true
|
||||
pkill -f "uvicorn" || true
|
||||
sleep 2
|
||||
|
||||
# 2. Backup current data
|
||||
log_info "Creating safety backup of current data..."
|
||||
mkdir -p data/backups_before_restore
|
||||
tar -czf "data/backups_before_restore/backup-before-restore-$(date +%s).tar.gz" \
|
||||
data/inventory.db data/inventory.db-* 2>/dev/null || true
|
||||
|
||||
# 3. Extract backup
|
||||
log_info "Extracting backup..."
|
||||
tar -xzf "$BACKUP_FILE" -C . || log_error "Backup extraction failed"
|
||||
|
||||
# 4. Verify essential files
|
||||
log_info "Verifying restored files..."
|
||||
[[ -f "data/inventory.db" ]] || log_error "Database not found in backup"
|
||||
[[ -f "inventory.env" ]] || log_error "inventory.env not found in backup"
|
||||
|
||||
log_info "✓ Restore complete"
|
||||
log_warn "Remember to start services: ./start_server.sh"
|
||||
fi
|
||||
|
||||
log_info "Restore complete!"
|
||||
log_warn "Remember to verify data in the application before returning to production"
|
||||
Reference in New Issue
Block a user