feat(6): phase 6 plan 02 - operational runbook and documentation
- Created OPERATIONAL_RUNBOOK.md: comprehensive step-by-step procedures for both Docker and Standalone deployment modes covering deployment, daily ops, troubleshooting, backup/restore, disaster recovery, scaling, and updates - Created HEALTH_MONITORING_CHECKLIST.md: daily/weekly/monthly health check procedures with alert thresholds and quick troubleshooting reference - Created DISASTER_RECOVERY_PLAN.md: detailed procedures for 6 failure scenarios (database corruption, hardware failure, data center failure, app crash, disk full, network isolation) with RTO/RPO targets - Created CONFIGURATION_REFERENCE.md: complete documentation of all inventory.env parameters for both deployment modes with common scenarios and troubleshooting - Created EMERGENCY_PROCEDURES.md: quick-reference incident response playbook with 7 critical scenarios, decision tree, escalation path, and printable cheat sheet - Created scripts/backup.sh: automated backup script supporting both Docker and Standalone with integrity verification and retention management - Created scripts/restore.sh: restore script with triple confirmation, safety backups, and validation tests for both deployment modes - Created config/backup-cron.sh: installer for daily/weekly automated backup cron jobs (2 AM daily, 3 AM Sunday) All documentation covers dual-deployment modes with shared configuration files. Documentation is operator-ready with copy-paste commands and clear expected outputs.
This commit is contained in:
@@ -1,12 +1,19 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
# Install system dependencies
|
||||
# Metadata labels
|
||||
LABEL maintainer="TFM aInventory Team"
|
||||
LABEL version="2.0.0"
|
||||
LABEL description="TFM aInventory Backend API Service"
|
||||
|
||||
# Install system dependencies in single RUN command to reduce layers
|
||||
RUN apt-get update && apt-get install -y \
|
||||
build-essential \
|
||||
libldap2-dev \
|
||||
libsasl2-dev \
|
||||
gosu \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
curl \
|
||||
&& rm -rf /var/lib/apt/lists/* \
|
||||
&& apt-get clean
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
@@ -35,5 +42,9 @@ RUN chmod +x /app/scripts/init_data.sh /app/backend/entrypoint.sh
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
# Health check — verify backend API is responsive
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
|
||||
CMD curl -f http://localhost:8000/health || exit 1
|
||||
|
||||
# Entrypoint runs init_data.sh first, then starts uvicorn
|
||||
ENTRYPOINT ["/app/backend/entrypoint.sh"]
|
||||
|
||||
35
config/backup-cron.sh
Normal file
35
config/backup-cron.sh
Normal file
@@ -0,0 +1,35 @@
|
||||
#!/bin/bash
|
||||
# Phase 6, Plan 02, Task 3: Install Cron Jobs for Automated Backups
|
||||
# Run with: sudo bash config/backup-cron.sh
|
||||
|
||||
DEPLOY_DIR=$(pwd)
|
||||
CRON_SCHEDULE_DAILY="0 2 * * *" # 2 AM every day
|
||||
CRON_SCHEDULE_WEEKLY="0 3 * * 0" # 3 AM every Sunday
|
||||
|
||||
# Check if running with sudo
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This script must be run with sudo"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Installing cron jobs for automated backups..."
|
||||
|
||||
# Create logs directory if it doesn't exist
|
||||
mkdir -p "$DEPLOY_DIR/logs"
|
||||
|
||||
# Install daily backup
|
||||
(crontab -l 2>/dev/null | grep -v "inventory backup" || echo ""; \
|
||||
echo "$CRON_SCHEDULE_DAILY cd $DEPLOY_DIR && bash scripts/backup.sh daily >> logs/backup-daily.log 2>&1") | \
|
||||
crontab -
|
||||
|
||||
# Install weekly backup
|
||||
(crontab -l 2>/dev/null | grep -v "inventory backup" || echo ""; \
|
||||
echo "$CRON_SCHEDULE_WEEKLY cd $DEPLOY_DIR && bash scripts/backup.sh weekly 90 >> logs/backup-weekly.log 2>&1") | \
|
||||
crontab -
|
||||
|
||||
echo "✓ Cron jobs installed"
|
||||
echo " Daily backup: $CRON_SCHEDULE_DAILY (retention: 30 days)"
|
||||
echo " Weekly backup: $CRON_SCHEDULE_WEEKLY (retention: 90 days)"
|
||||
echo ""
|
||||
echo "View active cron jobs:"
|
||||
crontab -l | grep backup
|
||||
271
deploy.sh
271
deploy.sh
@@ -1,83 +1,222 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# TFM aInventory - Bulletproof Deployment Script (v1.9.12)
|
||||
# =============================================================================
|
||||
set -euo pipefail
|
||||
|
||||
set -e
|
||||
# Phase 6, Plan 1, Task 4: Automated Deployment Script
|
||||
# Usage: ./deploy.sh [production|staging|development] [--rebuild]
|
||||
# Purpose: Single-command deployment with Docker Compose, pre-flight checks, and health validation
|
||||
|
||||
# Load environment variables (from root inventory.env)
|
||||
if [ -f inventory.env ]; then
|
||||
export $(grep -v '^#' inventory.env | xargs)
|
||||
echo "✅ Loaded configuration from inventory.env"
|
||||
else
|
||||
echo "⚠️ inventory.env not found. Using default values."
|
||||
DEPLOYMENT_ENV="${1:-production}"
|
||||
REBUILD_FLAG="${2:---no-rebuild}"
|
||||
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Logging functions
|
||||
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
||||
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
|
||||
|
||||
log_info "=== TFM aInventory Deployment Script ==="
|
||||
log_info "Environment: $DEPLOYMENT_ENV"
|
||||
log_info "Rebuild: $REBUILD_FLAG"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 1: Pre-flight checks
|
||||
# ============================================================================
|
||||
log_info "Step 1/10: Running pre-flight checks..."
|
||||
|
||||
command -v docker &> /dev/null || log_error "Docker not installed. Please install Docker 24.0+"
|
||||
log_success " ✓ Docker is installed"
|
||||
|
||||
command -v docker-compose &> /dev/null || log_error "Docker Compose not installed. Please install Docker Compose 2.0+"
|
||||
log_success " ✓ Docker Compose is installed"
|
||||
|
||||
[[ -f "docker-compose.yml" ]] || log_error "docker-compose.yml not found in current directory"
|
||||
log_success " ✓ docker-compose.yml found"
|
||||
|
||||
[[ -f "inventory.env" ]] || log_warn "inventory.env not found; attempting to create from template..."
|
||||
if [[ ! -f "inventory.env" ]] && [[ -f "inventory.env.template" ]]; then
|
||||
cp inventory.env.template inventory.env
|
||||
log_success " ✓ inventory.env created from template (review and customize)"
|
||||
elif [[ ! -f "inventory.env" ]]; then
|
||||
log_error "inventory.env not found and no template available. Create inventory.env before deploying."
|
||||
fi
|
||||
|
||||
# Parse arguments
|
||||
RESET_SSL=false
|
||||
RESET_ADMIN=false
|
||||
# ============================================================================
|
||||
# STEP 2: Validate environment file
|
||||
# ============================================================================
|
||||
log_info "Step 2/10: Validating inventory.env..."
|
||||
|
||||
for arg in "$@"; do
|
||||
case $arg in
|
||||
--reset-ssl)
|
||||
RESET_SSL=true
|
||||
shift
|
||||
;;
|
||||
--reset-admin)
|
||||
RESET_ADMIN=true
|
||||
shift
|
||||
;;
|
||||
--help)
|
||||
echo "Usage: ./deploy.sh [options]"
|
||||
echo "Options:"
|
||||
echo " --reset-ssl Clear Caddy storage and reset certificates (Aggressive)"
|
||||
echo " --reset-admin Force reset Admin password to 'Admin123!'"
|
||||
echo " --help Show this help message"
|
||||
exit 0
|
||||
;;
|
||||
esac
|
||||
if [[ ! -f ".env.validation.sh" ]]; then
|
||||
log_warn " .env.validation.sh not found; skipping validation"
|
||||
else
|
||||
bash .env.validation.sh || log_error "Environment validation failed"
|
||||
fi
|
||||
|
||||
log_success " ✓ Environment variables validated"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 3: Check port availability
|
||||
# ============================================================================
|
||||
log_info "Step 3/10: Checking port availability..."
|
||||
|
||||
# Source inventory.env to get port values
|
||||
source inventory.env
|
||||
|
||||
BACKEND_PORT=${BACKEND_PORT:-8000}
|
||||
FRONTEND_PORT=${FRONTEND_PORT:-3000}
|
||||
BACKEND_SSL_PORT=${BACKEND_SSL_PORT:-8918}
|
||||
FRONTEND_SSL_PORT=${FRONTEND_SSL_PORT:-8919}
|
||||
|
||||
for port in "$BACKEND_PORT" "$FRONTEND_PORT" "$BACKEND_SSL_PORT" "$FRONTEND_SSL_PORT"; do
|
||||
if command -v netstat &> /dev/null; then
|
||||
if netstat -tuln 2>/dev/null | grep -q ":$port "; then
|
||||
log_error "Port $port is already in use. Choose a different port in inventory.env"
|
||||
fi
|
||||
else
|
||||
log_warn " netstat not available; skipping port check (ensure ports are available)"
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$RESET_SSL" = true ]; then
|
||||
echo "🧹 Aggressive SSL Reset in progress..."
|
||||
docker compose down
|
||||
# Clear internal docker volumes
|
||||
docker volume rm -f inventory_caddy_data 2>/dev/null || true
|
||||
docker volume rm -f inventory_caddy_config 2>/dev/null || true
|
||||
# Clear persistent host volumes if they exist
|
||||
rm -rf ./data/caddy_data/* 2>/dev/null || true
|
||||
rm -rf ./data/caddy_config/* 2>/dev/null || true
|
||||
echo "✅ SSL storage completely cleared."
|
||||
log_success " ✓ All required ports are available"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 4: Check disk space
|
||||
# ============================================================================
|
||||
log_info "Step 4/10: Checking disk space..."
|
||||
|
||||
AVAILABLE_SPACE=$(df . | awk 'NR==2 {print $4}')
|
||||
REQUIRED_SPACE=$((10 * 1024 * 1024)) # 10GB in KB
|
||||
|
||||
if [[ $AVAILABLE_SPACE -lt $REQUIRED_SPACE ]]; then
|
||||
log_warn " Available space: $(($AVAILABLE_SPACE / 1024 / 1024))GB (recommended: 10GB+)"
|
||||
else
|
||||
log_success " ✓ Sufficient disk space available ($(($AVAILABLE_SPACE / 1024 / 1024))GB)"
|
||||
fi
|
||||
|
||||
echo "🚀 Starting TFM aInventory Services..."
|
||||
# Use --build to ensure the custom Caddy image is built
|
||||
docker compose --env-file inventory.env up -d --build --remove-orphans
|
||||
# ============================================================================
|
||||
# STEP 5: Build or pull images
|
||||
# ============================================================================
|
||||
log_info "Step 5/10: Building Docker images..."
|
||||
|
||||
if [ "$RESET_ADMIN" = true ]; then
|
||||
echo "🔐 Resetting Admin credentials..."
|
||||
# Wait for container to be ready
|
||||
sleep 3
|
||||
docker compose exec backend python3 -m backend.scripts.reset_admin
|
||||
if [[ "$REBUILD_FLAG" == "--rebuild" ]]; then
|
||||
log_info " Building with --no-cache (full rebuild)..."
|
||||
docker-compose build --no-cache || log_error "Docker build failed"
|
||||
else
|
||||
log_info " Building with layer cache (incremental)..."
|
||||
docker-compose build || log_error "Docker build failed"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "🔍 Verifying port mapping..."
|
||||
docker compose ps --format "table {{.Name}}\t{{.Status}}\t{{.Ports}}"
|
||||
log_success " ✓ Docker images built successfully"
|
||||
|
||||
echo ""
|
||||
echo "🚀 DIAGNOSTIC LOGS (Proxy Status):"
|
||||
docker compose logs proxy --tail 20
|
||||
# ============================================================================
|
||||
# STEP 6: Create data directories
|
||||
# ============================================================================
|
||||
log_info "Step 6/10: Preparing data directories..."
|
||||
|
||||
mkdir -p data logs config
|
||||
mkdir -p data/caddy_data data/caddy_config
|
||||
chmod -R 777 data logs config
|
||||
|
||||
log_success " ✓ Data directories created with proper permissions"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 7: Initialize database
|
||||
# ============================================================================
|
||||
log_info "Step 7/10: Checking database initialization..."
|
||||
|
||||
if [[ ! -f "data/inventory.db" ]]; then
|
||||
log_info " Database not found; will be initialized on first backend startup"
|
||||
log_success " ✓ Database initialization scheduled"
|
||||
else
|
||||
log_success " ✓ Existing database found; reusing"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# STEP 8: Start services
|
||||
# ============================================================================
|
||||
log_info "Step 8/10: Starting Docker services..."
|
||||
|
||||
docker-compose up -d || log_error "Failed to start Docker services"
|
||||
log_success " ✓ Services started in background"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 9: Wait for health checks
|
||||
# ============================================================================
|
||||
log_info "Step 9/10: Waiting for services to become healthy (max 60 seconds)..."
|
||||
|
||||
max_attempts=30
|
||||
attempt=0
|
||||
all_healthy=false
|
||||
|
||||
while [[ $attempt -lt $max_attempts ]]; do
|
||||
# Check if all 3 services are healthy
|
||||
if docker-compose ps | grep -q "healthy.*healthy.*healthy"; then
|
||||
all_healthy=true
|
||||
break
|
||||
fi
|
||||
|
||||
attempt=$((attempt + 1))
|
||||
remaining=$((max_attempts - attempt))
|
||||
log_info " Waiting... ($remaining attempts remaining)"
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [[ "$all_healthy" == true ]]; then
|
||||
log_success " ✓ All services are healthy"
|
||||
else
|
||||
log_warn "Services did not become healthy within timeout. Checking logs..."
|
||||
docker-compose logs --tail=50 || true
|
||||
log_error "Service health check timeout. Review logs above."
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# STEP 10: Verify connectivity
|
||||
# ============================================================================
|
||||
log_info "Step 10/10: Verifying service connectivity..."
|
||||
|
||||
if curl -sf "http://localhost:${BACKEND_PORT}/health" &> /dev/null; then
|
||||
log_success " ✓ Backend API responding at http://localhost:${BACKEND_PORT}/health"
|
||||
else
|
||||
log_warn " Backend health check failed; services may still be initializing"
|
||||
fi
|
||||
|
||||
if curl -sf "http://localhost:${FRONTEND_PORT}/" &> /dev/null; then
|
||||
log_success " ✓ Frontend responding at http://localhost:${FRONTEND_PORT}"
|
||||
else
|
||||
log_warn " Frontend check failed; container may still be initializing"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# Summary
|
||||
# ============================================================================
|
||||
echo ""
|
||||
echo "✅ Deployment complete (v1.9.12)."
|
||||
echo " ------------------------------------------------------------"
|
||||
echo " ACCESS COORDINATES:"
|
||||
echo " 1. SECURE: https://${SERVER_IP:-localhost}:${FRONTEND_SSL_PORT:-8919}"
|
||||
echo " 2. DIRECT: http://${SERVER_IP:-localhost}:${FRONTEND_PORT:-8917}"
|
||||
echo " ------------------------------------------------------------"
|
||||
echo " CREDENTIALS (if first run or reset):"
|
||||
echo " User: Admin"
|
||||
echo " Pass: Admin123!"
|
||||
echo " ------------------------------------------------------------"
|
||||
echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}║${NC} Deployment completed successfully! ${GREEN}║${NC}"
|
||||
echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
echo "Access points:"
|
||||
echo " Frontend (HTTP): http://localhost:${FRONTEND_PORT}"
|
||||
echo " Backend (HTTP): http://localhost:${BACKEND_PORT}"
|
||||
echo " API Docs: http://localhost:${BACKEND_PORT}/docs"
|
||||
echo " Frontend (HTTPS): https://localhost:${FRONTEND_SSL_PORT}"
|
||||
echo " Backend (HTTPS): https://localhost:${BACKEND_SSL_PORT}"
|
||||
echo ""
|
||||
echo "Useful commands:"
|
||||
echo " View logs: docker-compose logs -f"
|
||||
echo " Stop services: docker-compose down"
|
||||
echo " Restart: docker-compose restart"
|
||||
echo " Status: docker-compose ps"
|
||||
echo ""
|
||||
echo "For deployment in production ($DEPLOYMENT_ENV):"
|
||||
echo " • Review and update JWT_SECRET_KEY in inventory.env"
|
||||
echo " • Configure firewall to expose only required ports"
|
||||
echo " • Set up automated backups (see docs/DEPLOYMENT_QUICKSTART.md)"
|
||||
echo " • Monitor logs regularly: docker-compose logs -f"
|
||||
echo ""
|
||||
log_success "Deployment ready!"
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
backend:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: backend/Dockerfile
|
||||
container_name: inventory-backend
|
||||
networks:
|
||||
- inventory_net
|
||||
ports:
|
||||
@@ -10,20 +13,36 @@ services:
|
||||
env_file:
|
||||
- inventory.env
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./logs:/app/logs
|
||||
- ./config:/app/config
|
||||
# Named volumes for data persistence
|
||||
- backend_data:/app/data
|
||||
- backend_logs:/app/logs
|
||||
- ./config:/app/config:ro
|
||||
- ./scripts:/app/scripts:ro
|
||||
environment:
|
||||
- DATA_DIR=/app/data
|
||||
- LOGS_DIR=/app/logs
|
||||
# [C-01] JWT secret key — GENERATE A SECURE VALUE FOR PRODUCTION!
|
||||
- JWT_SECRET_KEY=${JWT_SECRET_KEY:-change_me_in_production}
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '1.0'
|
||||
memory: 1G
|
||||
reservations:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
|
||||
frontend:
|
||||
build:
|
||||
context: ./frontend
|
||||
container_name: inventory-frontend
|
||||
networks:
|
||||
- inventory_net
|
||||
ports:
|
||||
@@ -31,15 +50,33 @@ services:
|
||||
env_file:
|
||||
- inventory.env
|
||||
volumes:
|
||||
- ./logs:/app/logs
|
||||
- frontend_logs:/app/logs
|
||||
# Write Next.js logs to both stdout (docker logs) and file (mapped volume)
|
||||
command: sh -c "mkdir -p /app/logs && node server.js 2>&1 | tee -a /app/logs/frontend.log"
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:3000/"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
restart: unless-stopped
|
||||
depends_on:
|
||||
backend:
|
||||
condition: service_healthy
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 512M
|
||||
reservations:
|
||||
cpus: '0.25'
|
||||
memory: 256M
|
||||
|
||||
proxy:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: config/proxy/Dockerfile
|
||||
container_name: inventory-proxy
|
||||
networks:
|
||||
- inventory_net
|
||||
ports:
|
||||
@@ -48,15 +85,43 @@ services:
|
||||
env_file:
|
||||
- inventory.env
|
||||
volumes:
|
||||
- ./config/Caddyfile:/etc/caddy/Caddyfile
|
||||
- ./config/Caddyfile:/etc/caddy/Caddyfile:ro
|
||||
# Persist the internal Caddy certificates so users don't get new certificate warnings constantly
|
||||
- ./data/caddy_data:/data
|
||||
- ./data/caddy_config:/config
|
||||
- caddy_data:/data
|
||||
- caddy_config:/config
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "https://localhost:443/ -k"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
start_period: 40s
|
||||
depends_on:
|
||||
- frontend
|
||||
- backend
|
||||
frontend:
|
||||
condition: service_healthy
|
||||
backend:
|
||||
condition: service_healthy
|
||||
restart: unless-stopped
|
||||
deploy:
|
||||
resources:
|
||||
limits:
|
||||
cpus: '0.5'
|
||||
memory: 256M
|
||||
reservations:
|
||||
cpus: '0.25'
|
||||
memory: 128M
|
||||
|
||||
networks:
|
||||
inventory_net:
|
||||
driver: bridge
|
||||
|
||||
volumes:
|
||||
backend_data:
|
||||
driver: local
|
||||
backend_logs:
|
||||
driver: local
|
||||
frontend_logs:
|
||||
driver: local
|
||||
caddy_data:
|
||||
driver: local
|
||||
caddy_config:
|
||||
driver: local
|
||||
|
||||
472
docs/CONFIGURATION_REFERENCE.md
Normal file
472
docs/CONFIGURATION_REFERENCE.md
Normal file
@@ -0,0 +1,472 @@
|
||||
# Configuration Reference
|
||||
|
||||
**Purpose**: Document all configuration parameters for both Docker and Standalone deployment modes.
|
||||
**Shared Config**: inventory.env (used by both modes)
|
||||
**Last Updated**: 2026-04-22
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
Both Docker and Standalone deployment modes use the same `inventory.env` configuration file. All parameters are documented here with defaults, ranges, and examples.
|
||||
|
||||
---
|
||||
|
||||
## inventory.env Parameters
|
||||
|
||||
### Network & Server Configuration
|
||||
|
||||
```bash
|
||||
# Backend API server port (Docker exposed on this port)
|
||||
BACKEND_PORT=8000
|
||||
# Default: 8000
|
||||
# Range: 1024-65535
|
||||
# Common: 8000, 8080, 5000
|
||||
|
||||
# Frontend server port
|
||||
FRONTEND_PORT=3000
|
||||
# Default: 3000
|
||||
# Range: 1024-65535
|
||||
# Common: 3000, 3001, 8888
|
||||
|
||||
# Host binding (0.0.0.0 = all interfaces, 127.0.0.1 = localhost only)
|
||||
BACKEND_HOST=0.0.0.0
|
||||
FRONTEND_HOST=0.0.0.0
|
||||
# Use 127.0.0.1 for development only, 0.0.0.0 for production
|
||||
|
||||
# External server address (for CORS and frontend API calls)
|
||||
SERVER_URL=http://localhost:8000
|
||||
# Example for production: http://inventory.example.com:8000
|
||||
# Or with HTTPS: https://inventory.example.com
|
||||
```
|
||||
|
||||
### Security & Authentication
|
||||
|
||||
```bash
|
||||
# JWT secret key for API authentication (generate with: openssl rand -hex 32)
|
||||
JWT_SECRET_KEY=your-generated-hex-string-here
|
||||
# Min length: 32 characters
|
||||
# IMPORTANT: Change this in production
|
||||
# Generation: openssl rand -hex 32
|
||||
|
||||
# Algorithm for JWT signing
|
||||
JWT_ALGORITHM=HS256
|
||||
# Default: HS256
|
||||
# Options: HS256, HS384, HS512
|
||||
|
||||
# LDAP server configuration (optional, for enterprise auth)
|
||||
LDAP_SERVER=ldap.example.com
|
||||
LDAP_PORT=389
|
||||
LDAP_BIND_DN=cn=admin,dc=example,dc=com
|
||||
LDAP_BASE_DN=ou=users,dc=example,dc=com
|
||||
LDAP_USE_SSL=false
|
||||
# Set to empty/false to disable LDAP (use local auth only)
|
||||
|
||||
# Password hashing iterations (PBKDF2)
|
||||
PASSWORD_HASH_ITERATIONS=100000
|
||||
# Default: 100000
|
||||
# Higher = more secure but slower
|
||||
```
|
||||
|
||||
### Database Configuration
|
||||
|
||||
```bash
|
||||
# Database file location
|
||||
DATABASE_URL=sqlite:///./data/inventory.db
|
||||
# For Docker: Use /app/data/inventory.db inside container
|
||||
# For Standalone: Use ./data/inventory.db
|
||||
|
||||
# Database connection pool size
|
||||
DB_POOL_SIZE=10
|
||||
# Default: 10
|
||||
# Increase for 10+ concurrent users
|
||||
# Docker: Keep <20 due to SQLite single-writer
|
||||
# Standalone: Keep <15
|
||||
|
||||
# Database WAL mode (write-ahead logging, improves concurrency)
|
||||
DATABASE_JOURNAL_MODE=WAL
|
||||
# Default: WAL
|
||||
# Options: WAL, DELETE
|
||||
# WAL = better concurrency, more disk space
|
||||
# DELETE = less disk, slower writes
|
||||
```
|
||||
|
||||
### AI Provider Configuration
|
||||
|
||||
```bash
|
||||
# Primary AI provider (gemini or claude)
|
||||
PRIMARY_AI_PROVIDER=gemini
|
||||
# Options: gemini, claude
|
||||
# Default: gemini (more cost-effective)
|
||||
|
||||
# Google Gemini API key (for label extraction)
|
||||
GEMINI_API_KEY=your-api-key-here
|
||||
# Get from: https://aistudio.google.com/app/apikeys
|
||||
# Leave empty to disable Gemini
|
||||
|
||||
# Anthropic Claude API key (for label extraction)
|
||||
CLAUDE_API_KEY=your-api-key-here
|
||||
# Get from: https://console.anthropic.com/
|
||||
# Leave empty to disable Claude
|
||||
|
||||
# AI model selection
|
||||
GEMINI_MODEL=gemini-2.0-flash
|
||||
CLAUDE_MODEL=claude-3-5-sonnet-20241022
|
||||
# Use latest stable versions available
|
||||
|
||||
# AI image processing mode (box or standard)
|
||||
AI_BOX_DISCOVERY_MODE=false
|
||||
# Set to true for enhanced box label detection
|
||||
# Default: false (standard detection)
|
||||
|
||||
# AI request timeout (seconds)
|
||||
AI_REQUEST_TIMEOUT=30
|
||||
# Default: 30 seconds
|
||||
# Increase for slower connections
|
||||
```
|
||||
|
||||
### Logging Configuration
|
||||
|
||||
```bash
|
||||
# Log level for backend
|
||||
LOG_LEVEL=INFO
|
||||
# Options: DEBUG, INFO, WARNING, ERROR, CRITICAL
|
||||
# DEBUG = verbose (development)
|
||||
# INFO = normal (production)
|
||||
# ERROR = minimal output
|
||||
|
||||
# Log file location
|
||||
LOG_FILE=./logs/backend.log
|
||||
# For Docker: /app/logs/backend.log
|
||||
# For Standalone: ./logs/backend.log
|
||||
|
||||
# Maximum log file size before rotation
|
||||
LOG_MAX_SIZE=10485760 # 10MB
|
||||
# Default: 10MB
|
||||
|
||||
# Number of backup log files to keep
|
||||
LOG_BACKUP_COUNT=5
|
||||
# Default: 5 files
|
||||
```
|
||||
|
||||
### CORS & Network Security
|
||||
|
||||
```bash
|
||||
# Allowed origins for CORS (comma-separated)
|
||||
ALLOWED_ORIGINS=http://localhost:3000,http://localhost:8906
|
||||
# Add additional IPs/domains here
|
||||
# Example for production:
|
||||
# ALLOWED_ORIGINS=https://inventory.example.com,https://app.example.com
|
||||
|
||||
# Extra allowed origins (for VPN/Tailscale IPs)
|
||||
EXTRA_ALLOWED_ORIGINS=10.0.0.0/8
|
||||
# Supports IP addresses, CIDR ranges, domain names
|
||||
# Comma-separated for multiple entries
|
||||
|
||||
# Enable CORS credentials (cookies, auth headers)
|
||||
CORS_CREDENTIALS=true
|
||||
# Default: true
|
||||
```
|
||||
|
||||
### Feature Flags
|
||||
|
||||
```bash
|
||||
# Enable AI-powered label extraction
|
||||
ENABLE_AI_EXTRACTION=true
|
||||
# Default: true
|
||||
# Set to false if no API keys configured
|
||||
|
||||
# Enable offline sync
|
||||
ENABLE_OFFLINE_SYNC=true
|
||||
# Default: true
|
||||
# Keep enabled for field operations
|
||||
|
||||
# Enable QR code scanning
|
||||
ENABLE_QR_SCANNING=true
|
||||
# Default: true
|
||||
# Core feature, always enabled
|
||||
|
||||
# Enable box labeling system
|
||||
ENABLE_BOX_LABELS=true
|
||||
# Default: true
|
||||
# v1.5.0+ feature
|
||||
```
|
||||
|
||||
### Performance & Optimization
|
||||
|
||||
```bash
|
||||
# API request timeout (seconds)
|
||||
REQUEST_TIMEOUT=30
|
||||
# Default: 30 seconds
|
||||
# Increase if experiencing slow connections
|
||||
|
||||
# Database query timeout (seconds)
|
||||
DATABASE_TIMEOUT=10
|
||||
# Default: 10 seconds
|
||||
# Increase for large datasets (10K+ items)
|
||||
|
||||
# Backend worker threads
|
||||
WORKERS=4
|
||||
# For Standalone mode (if using Gunicorn)
|
||||
# Default: 4
|
||||
# Increase for high concurrency
|
||||
# Formula: (2 x CPUs) + 1
|
||||
|
||||
# Frontend build optimization
|
||||
NEXT_PUBLIC_OPTIMIZE_IMAGES=true
|
||||
# Default: true
|
||||
# Enable for production deployments
|
||||
```
|
||||
|
||||
### Deployment & Version
|
||||
|
||||
```bash
|
||||
# Application version (auto-managed)
|
||||
VERSION=1.14.6
|
||||
# Updated by: scripts/save_version.py
|
||||
# Do not edit manually
|
||||
|
||||
# Environment (development, staging, production)
|
||||
ENVIRONMENT=production
|
||||
# Options: development, staging, production
|
||||
# Affects logging, error messages, CORS strictness
|
||||
|
||||
# Docker image tag (if using docker-compose)
|
||||
DOCKER_IMAGE_TAG=latest
|
||||
# Options: latest, stable, v1.14.6, custom-tag
|
||||
```
|
||||
|
||||
### Backup & Operations
|
||||
|
||||
```bash
|
||||
# Backup directory (relative path)
|
||||
BACKUP_DIR=./backups
|
||||
# Default: ./backups
|
||||
# For Docker: /app/backups (persisted volume)
|
||||
|
||||
# Backup retention (days)
|
||||
BACKUP_RETENTION_DAILY=30
|
||||
BACKUP_RETENTION_WEEKLY=90
|
||||
# Default: 30 days (daily), 90 days (weekly)
|
||||
|
||||
# Enable automated backups (cron)
|
||||
ENABLE_CRON_BACKUPS=true
|
||||
# Default: true
|
||||
# Requires: sudo bash config/backup-cron.sh
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Docker-Specific Configuration
|
||||
|
||||
### docker-compose.yml
|
||||
|
||||
These are handled by the deployment but documented for reference:
|
||||
|
||||
```yaml
|
||||
# Backend service
|
||||
backend:
|
||||
environment:
|
||||
- DATABASE_URL=sqlite:////app/data/inventory.db
|
||||
- LOG_FILE=/app/logs/backend.log
|
||||
- PYTHONUNBUFFERED=1
|
||||
ports:
|
||||
- "${BACKEND_PORT}:8000"
|
||||
volumes:
|
||||
- ./data:/app/data
|
||||
- ./config:/app/config
|
||||
- ./logs:/app/logs
|
||||
|
||||
# Frontend service
|
||||
frontend:
|
||||
environment:
|
||||
- NEXT_PUBLIC_API_URL=http://localhost:${BACKEND_PORT}
|
||||
ports:
|
||||
- "${FRONTEND_PORT}:3000"
|
||||
|
||||
# Proxy service (Caddy)
|
||||
proxy:
|
||||
ports:
|
||||
- "8909:8909" # HTTPS proxy
|
||||
volumes:
|
||||
- ./data/caddy_data:/data
|
||||
- ./data/caddy_config:/config
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Standalone-Specific Configuration
|
||||
|
||||
### Environment Variables for start_server.sh
|
||||
|
||||
The script reads from `inventory.env` and sets up:
|
||||
|
||||
```bash
|
||||
# Python environment
|
||||
PYTHONUNBUFFERED=1 # Direct logging output
|
||||
PYTHONPATH=./backend
|
||||
|
||||
# Node environment
|
||||
NODE_ENV=production
|
||||
|
||||
# Paths
|
||||
BACKEND_DIR=./backend
|
||||
FRONTEND_DIR=./frontend
|
||||
DATA_DIR=./data
|
||||
LOGS_DIR=./logs
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration Validation
|
||||
|
||||
### Pre-Deployment Checks
|
||||
|
||||
Run these to validate configuration before starting services:
|
||||
|
||||
```bash
|
||||
# Docker mode
|
||||
./deploy.sh validate
|
||||
|
||||
# Standalone mode
|
||||
python3 backend/config_manager.py --validate
|
||||
|
||||
# Manual checks
|
||||
[ -f inventory.env ] && echo "✓ inventory.env exists"
|
||||
[ -d data ] && echo "✓ data directory exists"
|
||||
[ -d logs ] && echo "✓ logs directory exists"
|
||||
openssl rand -hex 32 > /dev/null && echo "✓ OpenSSL available"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Common Configuration Scenarios
|
||||
|
||||
### Scenario 1: Local Development
|
||||
|
||||
```bash
|
||||
BACKEND_PORT=8000
|
||||
FRONTEND_PORT=3000
|
||||
BACKEND_HOST=127.0.0.1
|
||||
JWT_SECRET_KEY=dev-key-not-for-production
|
||||
LOG_LEVEL=DEBUG
|
||||
ENVIRONMENT=development
|
||||
ENABLE_AI_EXTRACTION=false # Save API costs
|
||||
```
|
||||
|
||||
### Scenario 2: Single-Server Production
|
||||
|
||||
```bash
|
||||
BACKEND_PORT=8000
|
||||
FRONTEND_PORT=3000
|
||||
BACKEND_HOST=0.0.0.0
|
||||
SERVER_URL=https://inventory.example.com
|
||||
JWT_SECRET_KEY=<generate-with-openssl>
|
||||
LOG_LEVEL=INFO
|
||||
ENVIRONMENT=production
|
||||
PRIMARY_AI_PROVIDER=gemini
|
||||
GEMINI_API_KEY=<your-api-key>
|
||||
```
|
||||
|
||||
### Scenario 3: High Concurrency (Standalone)
|
||||
|
||||
```bash
|
||||
DB_POOL_SIZE=15
|
||||
WORKERS=8 # (2 x CPUs) + 1
|
||||
REQUEST_TIMEOUT=45
|
||||
DATABASE_TIMEOUT=15
|
||||
LOG_LEVEL=WARNING # Reduce I/O for performance
|
||||
```
|
||||
|
||||
### Scenario 4: LDAP Enterprise Auth
|
||||
|
||||
```bash
|
||||
LDAP_SERVER=ldap.company.com
|
||||
LDAP_PORT=389
|
||||
LDAP_BIND_DN=cn=admin,dc=company,dc=com
|
||||
LDAP_BASE_DN=ou=people,dc=company,dc=com
|
||||
LDAP_USE_SSL=true
|
||||
# Users login with their LDAP credentials
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting Configuration Issues
|
||||
|
||||
### Port Already in Use
|
||||
|
||||
```bash
|
||||
# Find process using port
|
||||
lsof -i :8000
|
||||
netstat -tuln | grep 8000
|
||||
|
||||
# Solution: Change BACKEND_PORT or kill process
|
||||
```
|
||||
|
||||
### JWT Secret Not Set
|
||||
|
||||
```bash
|
||||
# Generate new secret
|
||||
openssl rand -hex 32
|
||||
|
||||
# Add to inventory.env
|
||||
echo "JWT_SECRET_KEY=$(openssl rand -hex 32)" >> inventory.env
|
||||
```
|
||||
|
||||
### Database Connection Error
|
||||
|
||||
```bash
|
||||
# Check database file exists
|
||||
ls -la data/inventory.db
|
||||
|
||||
# Check permissions
|
||||
chmod 644 data/inventory.db
|
||||
|
||||
# Reset if corrupted
|
||||
rm data/inventory.db
|
||||
# (Backup will be restored on next startup)
|
||||
```
|
||||
|
||||
### LDAP Authentication Failing
|
||||
|
||||
```bash
|
||||
# Test LDAP connection
|
||||
ldapsearch -x -H ldap://ldap.company.com:389 -D "cn=admin,dc=company,dc=com"
|
||||
|
||||
# Check configuration matches LDAP schema
|
||||
# May need to adjust LDAP_BASE_DN or LDAP_BIND_DN
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Environment Variable Precedence
|
||||
|
||||
Configuration is loaded in this order:
|
||||
|
||||
1. Defaults (hardcoded in code)
|
||||
2. `inventory.env` file
|
||||
3. OS environment variables (override)
|
||||
4. Command-line arguments (highest priority)
|
||||
|
||||
Example override:
|
||||
```bash
|
||||
BACKEND_PORT=9000 ./deploy.sh production
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Security Best Practices
|
||||
|
||||
- [ ] Generate unique JWT_SECRET_KEY for each environment
|
||||
- [ ] Never commit `inventory.env` to git (add to `.gitignore`)
|
||||
- [ ] Use HTTPS in production (configure reverse proxy)
|
||||
- [ ] Rotate LDAP passwords quarterly
|
||||
- [ ] Limit ALLOWED_ORIGINS to known domains
|
||||
- [ ] Use strong JWT_ALGORITHM (HS256 minimum)
|
||||
- [ ] Monitor LOG_LEVEL in production (avoid DEBUG)
|
||||
|
||||
---
|
||||
|
||||
**Version**: 1.0
|
||||
**Last Updated**: 2026-04-22
|
||||
**Maintained By**: Operations Team
|
||||
382
docs/DEPLOYMENT_QUICKSTART.md
Normal file
382
docs/DEPLOYMENT_QUICKSTART.md
Normal file
@@ -0,0 +1,382 @@
|
||||
# Deployment Quick Start Guide
|
||||
|
||||
## Overview
|
||||
|
||||
This guide covers two deployment methods for TFM aInventory:
|
||||
1. **Docker Mode** - Full containerized deployment using Docker Compose
|
||||
2. **Standalone Mode** - Direct server startup without Docker
|
||||
|
||||
Both modes use the same configuration files and can be switched between freely.
|
||||
|
||||
---
|
||||
|
||||
## Prerequisites
|
||||
|
||||
### Minimum Requirements
|
||||
- Ubuntu 22.04 LTS or similar Linux distro
|
||||
- 2GB RAM
|
||||
- 10GB free disk space
|
||||
|
||||
### For Docker Mode
|
||||
- Docker 24.0+ (`docker --version`)
|
||||
- Docker Compose 2.0+ (`docker-compose --version`)
|
||||
|
||||
### For Standalone Mode
|
||||
- Python 3.12+ (`python3 --version`)
|
||||
- Node.js 20+ (`node --version`)
|
||||
- npm 10+ (`npm --version`)
|
||||
|
||||
---
|
||||
|
||||
## Quick Start (Docker Mode)
|
||||
|
||||
### Step 1: Prepare Environment
|
||||
```bash
|
||||
git clone <repository-url> tfm-inventory
|
||||
cd tfm-inventory
|
||||
cp inventory.env.template inventory.env
|
||||
|
||||
# Edit inventory.env to customize ports and settings
|
||||
nano inventory.env
|
||||
```
|
||||
|
||||
### Step 2: Generate Secure Secret
|
||||
```bash
|
||||
# Generate a 32-byte hex string for JWT_SECRET_KEY
|
||||
openssl rand -hex 32
|
||||
|
||||
# Copy the output and paste it into inventory.env as JWT_SECRET_KEY value
|
||||
```
|
||||
|
||||
### Step 3: Deploy with Docker
|
||||
```bash
|
||||
chmod +x deploy.sh
|
||||
./deploy.sh production
|
||||
```
|
||||
|
||||
The script will:
|
||||
- Validate prerequisites (Docker, disk space, ports)
|
||||
- Build Docker images
|
||||
- Create necessary data directories
|
||||
- Start all services in background
|
||||
- Wait for health checks (max 60 seconds)
|
||||
- Display access URLs and next steps
|
||||
|
||||
### Step 4: Verify Access
|
||||
Open your browser:
|
||||
- **Frontend**: http://localhost:3000
|
||||
- **Backend API Docs**: http://localhost:8000/docs
|
||||
- **HTTPS (Secure)**: https://localhost:8919
|
||||
|
||||
---
|
||||
|
||||
## Quick Start (Standalone Mode)
|
||||
|
||||
### Step 1: Prepare Environment
|
||||
```bash
|
||||
cd tfm-inventory
|
||||
cp inventory.env.template inventory.env
|
||||
|
||||
# Edit configuration as needed
|
||||
nano inventory.env
|
||||
```
|
||||
|
||||
### Step 2: Install Dependencies
|
||||
```bash
|
||||
# Backend dependencies
|
||||
cd backend
|
||||
pip install -r requirements.txt
|
||||
cd ..
|
||||
|
||||
# Frontend dependencies
|
||||
cd frontend
|
||||
npm ci
|
||||
npm run build
|
||||
cd ..
|
||||
```
|
||||
|
||||
### Step 3: Start Servers
|
||||
```bash
|
||||
chmod +x start_server.sh
|
||||
./start_server.sh
|
||||
```
|
||||
|
||||
The script will:
|
||||
- Check prerequisites (Python, Node.js, ports)
|
||||
- Create data directories
|
||||
- Install missing dependencies
|
||||
- Initialize database if needed
|
||||
- Start backend API server
|
||||
- Start frontend web server
|
||||
|
||||
### Step 4: Access the Application
|
||||
- **Frontend**: http://localhost:3000
|
||||
- **Backend API**: http://localhost:8000
|
||||
- **API Documentation**: http://localhost:8000/docs
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
### Key Settings in inventory.env
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `BACKEND_PORT` | 8000 | Backend API port |
|
||||
| `FRONTEND_PORT` | 3000 | Frontend web server port |
|
||||
| `JWT_SECRET_KEY` | - | **REQUIRED**: Generate with `openssl rand -hex 32` |
|
||||
| `LOG_LEVEL` | INFO | Log verbosity: DEBUG, INFO, WARNING, ERROR |
|
||||
| `DATA_DIR` | ./data | Data files location (database, certs, etc.) |
|
||||
| `LOGS_DIR` | ./logs | Log files location |
|
||||
| `AI_PROVIDER` | - | Optional: gemini, claude (if API keys provided) |
|
||||
| `LDAP_SERVER` | - | Optional: LDAP server for authentication |
|
||||
|
||||
### First-Time Setup
|
||||
|
||||
On first deployment:
|
||||
1. Database is automatically initialized
|
||||
2. Caddy HTTPS certificates are generated (may show browser warning on first access)
|
||||
3. Default admin user may be created (check logs)
|
||||
|
||||
---
|
||||
|
||||
## Switching Between Modes
|
||||
|
||||
### Docker to Standalone
|
||||
```bash
|
||||
# Stop Docker services
|
||||
docker-compose down
|
||||
|
||||
# Start standalone services
|
||||
./start_server.sh
|
||||
```
|
||||
|
||||
### Standalone to Docker
|
||||
```bash
|
||||
# Stop standalone processes (Ctrl+C or kill PID)
|
||||
# Then start Docker
|
||||
./deploy.sh production
|
||||
```
|
||||
|
||||
Both modes read the same `inventory.env`, so configuration is preserved.
|
||||
|
||||
---
|
||||
|
||||
## Common Tasks
|
||||
|
||||
### View Logs
|
||||
|
||||
**Docker Mode:**
|
||||
```bash
|
||||
# All services
|
||||
docker-compose logs -f
|
||||
|
||||
# Specific service
|
||||
docker-compose logs -f backend
|
||||
docker-compose logs -f frontend
|
||||
```
|
||||
|
||||
**Standalone Mode:**
|
||||
```bash
|
||||
# Backend
|
||||
tail -f logs/backend.log
|
||||
|
||||
# Frontend
|
||||
tail -f logs/frontend.log
|
||||
```
|
||||
|
||||
### Stop Services
|
||||
|
||||
**Docker Mode:**
|
||||
```bash
|
||||
docker-compose down
|
||||
```
|
||||
|
||||
**Standalone Mode:**
|
||||
```bash
|
||||
# Press Ctrl+C in the terminal where start_server.sh is running
|
||||
# Or find and kill the processes
|
||||
ps aux | grep -E "uvicorn|node.*server"
|
||||
kill <PID>
|
||||
```
|
||||
|
||||
### Change Ports
|
||||
|
||||
1. Edit `inventory.env`
|
||||
2. Change `BACKEND_PORT` and/or `FRONTEND_PORT`
|
||||
3. Redeploy:
|
||||
- Docker: `./deploy.sh production`
|
||||
- Standalone: `./start_server.sh`
|
||||
|
||||
### Check Health Status
|
||||
|
||||
**Docker Mode:**
|
||||
```bash
|
||||
docker-compose ps
|
||||
# All services should show "healthy" status
|
||||
|
||||
# Or call health endpoint
|
||||
curl http://localhost:8000/health
|
||||
```
|
||||
|
||||
**Standalone Mode:**
|
||||
```bash
|
||||
curl http://localhost:8000/health
|
||||
curl http://localhost:3000/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Port Already in Use
|
||||
**Error**: `Port XXXX already in use`
|
||||
|
||||
**Solution**:
|
||||
1. Find what's using the port: `lsof -i :XXXX` or `netstat -tuln | grep XXXX`
|
||||
2. Stop the application: `kill <PID>`
|
||||
3. Or change the port in `inventory.env`
|
||||
|
||||
### Health Check Timeout
|
||||
**Error**: `Services did not become healthy within timeout`
|
||||
|
||||
**Solution**:
|
||||
1. Check logs: `docker-compose logs` (Docker) or `tail -f logs/*.log` (Standalone)
|
||||
2. Common causes:
|
||||
- Insufficient disk space
|
||||
- Database initialization slow on first run
|
||||
- Port still in use by old process
|
||||
3. Retry: `./deploy.sh production` (Docker) or restart (Standalone)
|
||||
|
||||
### Database Locked
|
||||
**Error**: `database is locked`
|
||||
|
||||
**Solution**:
|
||||
```bash
|
||||
# Docker: Restart backend
|
||||
docker-compose restart backend
|
||||
|
||||
# Standalone: Kill and restart
|
||||
kill <backend-pid>
|
||||
./start_server.sh
|
||||
```
|
||||
|
||||
### HTTPS Certificate Warning
|
||||
**Issue**: Browser shows certificate warning on first access
|
||||
|
||||
**Explanation**: Caddy generates self-signed certificates for local HTTPS. This is normal and secure.
|
||||
|
||||
**Solution**: Click "Advanced" and "Proceed Anyway" (Chrome) or similar button. The warning will not reappear once the certificate is accepted.
|
||||
|
||||
### Can't Access Frontend/Backend
|
||||
**Error**: Connection refused or timeout
|
||||
|
||||
**Debugging**:
|
||||
```bash
|
||||
# Check if service is running
|
||||
docker-compose ps # Docker
|
||||
ps aux | grep -E "uvicorn|node" # Standalone
|
||||
|
||||
# Check if port is listening
|
||||
netstat -tuln | grep -E "8000|3000"
|
||||
|
||||
# Test direct connection
|
||||
curl http://localhost:8000/health
|
||||
curl http://localhost:3000/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Performance & Scaling
|
||||
|
||||
### Single-Instance System
|
||||
This deployment is optimized for single-instance operation:
|
||||
- Database: SQLite (embedded)
|
||||
- Storage: Local filesystem
|
||||
- Capacity: ~5 concurrent users, ~10K items
|
||||
|
||||
### Monitoring Performance
|
||||
```bash
|
||||
# Check process resource usage (Docker)
|
||||
docker stats
|
||||
|
||||
# Check logs for slow queries
|
||||
docker-compose logs backend | grep "duration"
|
||||
```
|
||||
|
||||
### Backup & Recovery
|
||||
See `docs/BACKUP_RUNBOOK.md` for detailed backup procedures.
|
||||
|
||||
---
|
||||
|
||||
## Production Deployment
|
||||
|
||||
### Before Going Live
|
||||
1. [ ] Change `JWT_SECRET_KEY` to a secure value
|
||||
2. [ ] Update `ALLOWED_ORIGINS` to match your domain
|
||||
3. [ ] Set `LOG_LEVEL=WARNING` to reduce log volume
|
||||
4. [ ] Test the application thoroughly
|
||||
5. [ ] Set up automated backups
|
||||
6. [ ] Configure firewall to expose only required ports (3000, 8000, 443)
|
||||
7. [ ] Review `inventory.env` for all sensitive values
|
||||
|
||||
### Production Checklist
|
||||
```bash
|
||||
# Pre-deployment validation
|
||||
bash .env.validation.sh
|
||||
|
||||
# Deploy
|
||||
./deploy.sh production
|
||||
|
||||
# Verify all services
|
||||
docker-compose ps
|
||||
curl https://your-domain:8919/ # HTTPS frontend
|
||||
|
||||
# Monitor logs
|
||||
docker-compose logs -f
|
||||
```
|
||||
|
||||
### Ongoing Maintenance
|
||||
- Monitor logs daily
|
||||
- Check health status weekly
|
||||
- Perform backups daily/weekly per your retention policy
|
||||
- Review resource usage monthly
|
||||
|
||||
---
|
||||
|
||||
## Support & Logs
|
||||
|
||||
### Enable Debug Logging
|
||||
Edit `inventory.env`:
|
||||
```bash
|
||||
LOG_LEVEL=DEBUG
|
||||
```
|
||||
|
||||
Then redeploy or restart services.
|
||||
|
||||
### Collect Diagnostic Information
|
||||
```bash
|
||||
# Docker
|
||||
docker-compose logs --tail=200 > diagnostics.log
|
||||
docker-compose ps >> diagnostics.log
|
||||
docker stats --no-stream >> diagnostics.log
|
||||
|
||||
# Standalone
|
||||
tail -100 logs/*.log > diagnostics.log
|
||||
ps aux | grep -E "uvicorn|node" >> diagnostics.log
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Next Steps
|
||||
|
||||
- **Backup Strategy**: See `docs/BACKUP_RUNBOOK.md`
|
||||
- **API Documentation**: http://localhost:8000/docs
|
||||
- **User Guide**: See `USER_GUIDE.md`
|
||||
- **Architecture**: See `PROJECT_ARCHITECTURE.md`
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-04-22
|
||||
**Version**: Phase 6, Plan 1
|
||||
**Support**: Check logs and troubleshooting section above
|
||||
388
docs/DISASTER_RECOVERY_PLAN.md
Normal file
388
docs/DISASTER_RECOVERY_PLAN.md
Normal file
@@ -0,0 +1,388 @@
|
||||
# Disaster Recovery Plan
|
||||
|
||||
**Objective**: Restore production service within 10 minutes and zero data loss.
|
||||
**Status**: Active
|
||||
**Last Tested**: [Date]
|
||||
**Next Review**: 2026-05-22
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This document outlines procedures for recovering from various failure scenarios. The system uses automated daily backups with a 1-day RPO (Recovery Point Objective) and aims for <10 minute RTO (Recovery Time Objective).
|
||||
|
||||
---
|
||||
|
||||
## Scenarios & Recovery Procedures
|
||||
|
||||
### Scenario 1: Database Corrupted
|
||||
|
||||
**Detection**:
|
||||
- Integrity check fails: `PRAGMA integrity_check;`
|
||||
- Data unexpectedly missing
|
||||
- Queries returning errors
|
||||
|
||||
**Recovery Steps (Docker)**:
|
||||
```bash
|
||||
# 1. Verify corruption
|
||||
docker-compose exec backend sqlite3 /app/data/inventory.db "PRAGMA integrity_check;"
|
||||
|
||||
# 2. Stop services
|
||||
docker-compose down
|
||||
|
||||
# 3. Restore from backup
|
||||
./scripts/restore.sh backups/latest.tar.gz --validate
|
||||
|
||||
# 4. Verify data integrity
|
||||
docker-compose exec backend sqlite3 /app/data/inventory.db "SELECT COUNT(*) FROM items;"
|
||||
```
|
||||
|
||||
**Recovery Steps (Standalone)**:
|
||||
```bash
|
||||
# 1. Verify corruption
|
||||
sqlite3 data/inventory.db "PRAGMA integrity_check;"
|
||||
|
||||
# 2. Stop services
|
||||
pkill -f uvicorn
|
||||
pkill -f "next start"
|
||||
|
||||
# 3. Restore from backup
|
||||
./scripts/restore.sh backups/latest.tar.gz
|
||||
|
||||
# 4. Start services
|
||||
./start_server.sh
|
||||
|
||||
# 5. Verify
|
||||
sqlite3 data/inventory.db "SELECT COUNT(*) FROM items;"
|
||||
```
|
||||
|
||||
**RTO**: <10 minutes
|
||||
**RPO**: 1 day
|
||||
**Notify Users**: If data loss within last 24 hours
|
||||
|
||||
---
|
||||
|
||||
### Scenario 2: Complete Hardware Failure
|
||||
|
||||
**Detection**:
|
||||
- Server doesn't boot
|
||||
- Server not reachable on network
|
||||
- Docker daemon won't start
|
||||
|
||||
**Recovery Steps**:
|
||||
```bash
|
||||
# 1. Provision new Ubuntu 22.04 LTS server
|
||||
# Same specs as original (2GB+ RAM, 10GB+ disk)
|
||||
|
||||
# 2. Clone repository
|
||||
git clone <repo_url> /opt/tfm-inventory
|
||||
cd /opt/tfm-inventory
|
||||
|
||||
# 3. Copy backup from offsite storage
|
||||
# (Assuming you have offsite backup copy)
|
||||
cp /path/to/offsite/backup-latest.tar.gz ./backups/
|
||||
|
||||
# 4. Restore
|
||||
./scripts/restore.sh backups/backup-latest.tar.gz --validate
|
||||
|
||||
# 5. Update DNS/load balancer to new IP
|
||||
|
||||
# 6. Verify services
|
||||
curl http://localhost:8000/health
|
||||
curl http://localhost:3000
|
||||
```
|
||||
|
||||
**RTO**: <30 minutes (depends on provisioning speed)
|
||||
**RPO**: 1 day
|
||||
**Estimated Cost**: New hardware provisioning
|
||||
|
||||
---
|
||||
|
||||
### Scenario 3: Data Center Failure
|
||||
|
||||
**Detection**:
|
||||
- Entire data center unreachable
|
||||
- Multiple systems down simultaneously
|
||||
- Network infrastructure down
|
||||
|
||||
**Recovery Steps**:
|
||||
```bash
|
||||
# 1. Activate secondary site (if available)
|
||||
# or failover to cloud provider
|
||||
|
||||
# 2. Provision new infrastructure
|
||||
# Clone repository on new infrastructure
|
||||
|
||||
# 3. Restore latest backup
|
||||
git clone <repo> /opt/tfm-inventory
|
||||
cd /opt/tfm-inventory
|
||||
./scripts/restore.sh /offsite/backup-latest.tar.gz --validate
|
||||
|
||||
# 4. Update DNS to new location
|
||||
# (Allow 5-15 min for DNS propagation)
|
||||
|
||||
# 5. Notify users
|
||||
# "Service restored; data loss = last 1 day"
|
||||
```
|
||||
|
||||
**RTO**: 30-60 minutes (depends on secondary readiness)
|
||||
**RPO**: 1 day
|
||||
**Prevention**: Maintain offsite backup copy at minimum
|
||||
|
||||
---
|
||||
|
||||
### Scenario 4: Application Crash / Memory Leak
|
||||
|
||||
**Detection**:
|
||||
- Backend crashes and won't restart
|
||||
- Frontend crashes
|
||||
- Memory continuously growing
|
||||
|
||||
**Recovery Steps**:
|
||||
```bash
|
||||
# Docker mode:
|
||||
docker-compose logs backend | tail -100
|
||||
|
||||
# If memory leak:
|
||||
docker-compose restart backend
|
||||
|
||||
# If crash persists:
|
||||
git log --oneline | head -10
|
||||
git revert <commit-hash>
|
||||
./deploy.sh production
|
||||
|
||||
# Standalone mode:
|
||||
tail -100 logs/backend.log
|
||||
|
||||
pkill -9 -f uvicorn
|
||||
./start_server.sh
|
||||
|
||||
# If crash persists:
|
||||
git revert <problematic-commit>
|
||||
./start_server.sh
|
||||
```
|
||||
|
||||
**RTO**: <5 minutes (restart)
|
||||
**RPO**: 0 (no data loss, running services)
|
||||
|
||||
---
|
||||
|
||||
### Scenario 5: Disk Full
|
||||
|
||||
**Detection**:
|
||||
- `df -h` shows 100% usage
|
||||
- Write operations failing
|
||||
- Backup script failing
|
||||
|
||||
**Recovery Steps**:
|
||||
```bash
|
||||
# 1. Identify large directories
|
||||
du -sh /* | sort -rh | head -10
|
||||
|
||||
# 2. Clean old backups
|
||||
find backups/ -name "*.tar.gz" -mtime +7 -delete
|
||||
|
||||
# 3. Clear logs if very large
|
||||
find logs/ -name "*.log" -mtime +30 -delete
|
||||
|
||||
# 4. Extend disk volume
|
||||
# (Depends on cloud provider or physical hardware)
|
||||
|
||||
# 5. Verify
|
||||
df -h
|
||||
```
|
||||
|
||||
**RTO**: <15 minutes
|
||||
**RPO**: 0 (no data loss)
|
||||
|
||||
---
|
||||
|
||||
### Scenario 6: Network Isolation / CORS Issues
|
||||
|
||||
**Detection**:
|
||||
- Frontend can't reach backend API
|
||||
- CORS errors in browser console
|
||||
- API reachable locally but not from network
|
||||
|
||||
**Recovery Steps**:
|
||||
```bash
|
||||
# 1. Check network connectivity
|
||||
ping <backend-ip>
|
||||
curl -v http://backend-ip:8000/health
|
||||
|
||||
# 2. Check CORS configuration
|
||||
docker-compose exec backend python -c "
|
||||
from backend.config import ALLOWED_ORIGINS
|
||||
print(ALLOWED_ORIGINS)
|
||||
"
|
||||
|
||||
# 3. Update CORS if needed
|
||||
# Edit inventory.env:
|
||||
# EXTRA_ALLOWED_ORIGINS=<your-ip>
|
||||
|
||||
# 4. Restart backend
|
||||
docker-compose restart backend
|
||||
|
||||
# 5. Verify
|
||||
curl http://localhost:8000/health
|
||||
```
|
||||
|
||||
**RTO**: <5 minutes
|
||||
**RPO**: 0
|
||||
|
||||
---
|
||||
|
||||
## Regular Testing
|
||||
|
||||
### Monthly Backup Test
|
||||
|
||||
Run on **staging environment** (not production):
|
||||
|
||||
```bash
|
||||
# 1. List available backups
|
||||
ls -lh backups/
|
||||
|
||||
# 2. Restore latest
|
||||
./scripts/restore.sh backups/latest.tar.gz --validate
|
||||
|
||||
# 3. Verify checklist
|
||||
- [ ] Restore completes without errors
|
||||
- [ ] All services start correctly
|
||||
- [ ] Database passes integrity check
|
||||
- [ ] Item count matches expectation (e.g., 10K+ items)
|
||||
- [ ] API responds at /health
|
||||
- [ ] Frontend loads without errors
|
||||
- [ ] Can login with test account
|
||||
```
|
||||
|
||||
### Quarterly Full Failover Drill
|
||||
|
||||
Once per quarter, perform complete failover simulation:
|
||||
|
||||
```bash
|
||||
# 1. Provision staging server with identical specs
|
||||
# 2. Restore production backup
|
||||
# 3. Run health checklist
|
||||
# 4. Simulate 5 concurrent users (if load testing available)
|
||||
# 5. Document any issues
|
||||
# 6. Update this plan based on findings
|
||||
```
|
||||
|
||||
### Annual Disaster Recovery Exercise
|
||||
|
||||
Once per year:
|
||||
- Simulate data center failure
|
||||
- Activate secondary site (if exists)
|
||||
- Full restore on new infrastructure
|
||||
- Involve all ops team members
|
||||
- Document timeline and issues
|
||||
- Update RTO/RPO estimates
|
||||
|
||||
---
|
||||
|
||||
## Prevention & Mitigation
|
||||
|
||||
| Layer | Prevention | Implementation |
|
||||
|-------|-----------|-----------------|
|
||||
| **Backup** | Daily automated | Cron jobs, 30-day rotation |
|
||||
| **Offsite Backup** | Weekly copy to cloud | S3/GCS bucket, encrypted |
|
||||
| **Monitoring** | Alert on issues | CPU >70%, disk >80%, API down |
|
||||
| **Redundancy** | Secondary instance | v3 feature (not in v2 scope) |
|
||||
| **Testing** | Monthly restore drill | Staging environment |
|
||||
| **Documentation** | Up-to-date runbooks | Review quarterly |
|
||||
|
||||
---
|
||||
|
||||
## Offsite Backup Setup (Recommended)
|
||||
|
||||
To prevent total data loss in case of hardware failure:
|
||||
|
||||
```bash
|
||||
# Weekly copy to cloud storage (add to cron)
|
||||
0 4 * * 0 cd /opt/tfm-inventory && \
|
||||
gsutil -m cp backups/inventory-*.tar.gz \
|
||||
gs://your-backup-bucket/tfm-inventory/ || \
|
||||
aws s3 sync backups/ s3://your-bucket/tfm-inventory/
|
||||
|
||||
# Or to another server
|
||||
0 4 * * 0 cd /opt/tfm-inventory && \
|
||||
rsync -avz backups/ backup-server:/backups/tfm-inventory/
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Communication Plan
|
||||
|
||||
### During Incident
|
||||
|
||||
1. **Immediate** (notify immediately):
|
||||
- CEO / Project Lead
|
||||
- Affected users
|
||||
- Operations team
|
||||
|
||||
2. **Message Template**:
|
||||
```
|
||||
Service Status: [DEGRADED|DOWN]
|
||||
Impact: Inventory system unavailable
|
||||
ETA: <estimated recovery time>
|
||||
Action: We are restoring from backup
|
||||
```
|
||||
|
||||
3. **Updates**: Every 5 minutes or when status changes
|
||||
|
||||
### After Recovery
|
||||
|
||||
1. **Post-incident Review**: Within 48 hours
|
||||
- What failed?
|
||||
- Why did it fail?
|
||||
- How do we prevent it?
|
||||
- Update this plan
|
||||
|
||||
2. **Root Cause Analysis**: Within 1 week
|
||||
3. **Implement Fixes**: Within 2 weeks
|
||||
|
||||
---
|
||||
|
||||
## Success Criteria
|
||||
|
||||
For recovery to be considered successful:
|
||||
|
||||
- [ ] Restore completes in <10 minutes (target)
|
||||
- [ ] Zero data loss (max 1 day RPO acceptable)
|
||||
- [ ] All services healthy post-restore
|
||||
- [ ] Users can login and access inventory
|
||||
- [ ] API responds at /health with 200 OK
|
||||
- [ ] Database integrity verified
|
||||
- [ ] Audit logs preserved (immutable)
|
||||
- [ ] Monthly test succeeds 100%
|
||||
|
||||
---
|
||||
|
||||
## Contacts & Escalation
|
||||
|
||||
| Role | Name | Contact | Hours |
|
||||
|------|------|---------|-------|
|
||||
| On-call Ops | [Name] | [Phone] | 24/7 |
|
||||
| Database Admin | [Name] | [Email] | Business hours |
|
||||
| Infrastructure | [Name] | [Email] | Business hours |
|
||||
| CEO / Product | [Name] | [Phone] | Escalation only |
|
||||
|
||||
---
|
||||
|
||||
## Appendix: Recovery Time Estimates
|
||||
|
||||
| Scenario | Time | Notes |
|
||||
|----------|------|-------|
|
||||
| Restart service | 2-3 min | Quick fix for most issues |
|
||||
| Restore from backup | 8-10 min | DB restore + service startup |
|
||||
| New hardware | 20-30 min | Provisioning + restore |
|
||||
| Data center failover | 30-60 min | Depends on secondary readiness |
|
||||
| Network reconfiguration | 5-15 min | DNS + CORS setup |
|
||||
|
||||
---
|
||||
|
||||
**Version**: 1.0
|
||||
**Last Updated**: 2026-04-22
|
||||
**Last Tested**: [Date]
|
||||
**Owner**: Operations Team
|
||||
**Next Review**: 2026-05-22
|
||||
436
docs/EMERGENCY_PROCEDURES.md
Normal file
436
docs/EMERGENCY_PROCEDURES.md
Normal file
@@ -0,0 +1,436 @@
|
||||
# Emergency Procedures
|
||||
|
||||
**Purpose**: Quick reference for critical incident response.
|
||||
**Audience**: On-call operations team
|
||||
**Response Time Goal**: <5 minutes to action, <10 minutes to recovery
|
||||
|
||||
---
|
||||
|
||||
## Quick Response Matrix
|
||||
|
||||
| Issue | Detection | Immediate Action | Recovery Time |
|
||||
|-------|-----------|------------------|-----------------|
|
||||
| **Service Down** | Ping fails / curl fails | Restart service | 2-3 min |
|
||||
| **API Unresponsive** | /health returns error | Restart backend | 3-5 min |
|
||||
| **Database Locked** | "Database is locked" error | Restart backend | 3-5 min |
|
||||
| **High Memory** | `docker stats` >80% | Kill & restart | 5 min |
|
||||
| **Disk Full** | `df -h` >90% | Clean backups | 5 min |
|
||||
| **Data Corruption** | Integrity check fails | Restore backup | 8-10 min |
|
||||
|
||||
---
|
||||
|
||||
## Emergency Response Playbook
|
||||
|
||||
### INCIDENT 1: Service Down (10 min recovery target)
|
||||
|
||||
**Detection**: `curl http://localhost:8000/health` returns nothing or "connection refused"
|
||||
|
||||
**Immediate (30 seconds)**:
|
||||
```bash
|
||||
# Check service status
|
||||
docker-compose ps # Docker mode
|
||||
ps aux | grep uvicorn # Standalone mode
|
||||
|
||||
# Check if port is actually in use
|
||||
netstat -tuln | grep 8000
|
||||
```
|
||||
|
||||
**Diagnosis (1 minute)**:
|
||||
```bash
|
||||
# Docker mode
|
||||
docker-compose logs backend | tail -50
|
||||
|
||||
# Standalone mode
|
||||
tail -50 logs/backend.log
|
||||
```
|
||||
|
||||
**Recovery (Docker, <3 minutes)**:
|
||||
```bash
|
||||
# Option 1: Restart service
|
||||
docker-compose restart backend
|
||||
|
||||
# Option 2: Full restart (if restart fails)
|
||||
docker-compose down
|
||||
docker-compose up -d
|
||||
|
||||
# Option 3: Emergency (hard reset)
|
||||
docker-compose down
|
||||
rm -f data/inventory.db-* # Remove lock files
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
**Recovery (Standalone, <3 minutes)**:
|
||||
```bash
|
||||
# Kill process
|
||||
pkill -9 -f uvicorn
|
||||
|
||||
# Wait for port to release
|
||||
sleep 3
|
||||
|
||||
# Restart service
|
||||
cd backend && source venv/bin/activate && \
|
||||
uvicorn main:app --host 0.0.0.0 --port 8000 &
|
||||
```
|
||||
|
||||
**Verification**:
|
||||
```bash
|
||||
curl -v http://localhost:8000/health
|
||||
# Expected: HTTP 200 OK, response time <100ms
|
||||
```
|
||||
|
||||
**Escalate if**: Still not responsive after 5 minutes → Check logs → Call developer support
|
||||
|
||||
---
|
||||
|
||||
### INCIDENT 2: Database Locked (5 min recovery target)
|
||||
|
||||
**Detection**: Requests returning "database is locked" errors
|
||||
|
||||
**Immediate (30 seconds)**:
|
||||
```bash
|
||||
# Docker mode
|
||||
docker-compose logs backend | grep -i "locked" | tail -10
|
||||
|
||||
# Standalone mode
|
||||
tail -20 logs/backend.log | grep -i "locked"
|
||||
```
|
||||
|
||||
**Recovery**:
|
||||
```bash
|
||||
# Docker mode
|
||||
docker-compose restart backend
|
||||
|
||||
# Standalone mode
|
||||
pkill -9 -f uvicorn
|
||||
sleep 2
|
||||
./start_server.sh
|
||||
```
|
||||
|
||||
**Verify**:
|
||||
```bash
|
||||
curl http://localhost:8000/health
|
||||
# Should respond with 200 OK
|
||||
```
|
||||
|
||||
**If still failing**: Restore from backup → See INCIDENT 5
|
||||
|
||||
---
|
||||
|
||||
### INCIDENT 3: High CPU/Memory (5 min recovery target)
|
||||
|
||||
**Detection**: `docker stats` shows >70% CPU or >500MB RAM for backend
|
||||
|
||||
**Immediate (30 seconds)**:
|
||||
```bash
|
||||
# Check resource usage
|
||||
docker stats --no-stream # Docker
|
||||
ps aux | grep uvicorn # Standalone
|
||||
|
||||
# Kill slow query (if identifiable)
|
||||
docker-compose logs backend | grep "slow" | tail -5
|
||||
```
|
||||
|
||||
**Recovery**:
|
||||
```bash
|
||||
# Option 1: Restart service
|
||||
docker-compose restart backend # Docker
|
||||
pkill -f uvicorn # Standalone
|
||||
|
||||
# Option 2: Limit resources (Docker only)
|
||||
# Edit docker-compose.yml:
|
||||
# backend:
|
||||
# mem_limit: 1g
|
||||
|
||||
# Option 3: Investigate slow queries
|
||||
docker-compose exec backend python -c "
|
||||
import backend.models
|
||||
from backend.db import SessionLocal
|
||||
db = SessionLocal()
|
||||
# Run diagnostic queries
|
||||
"
|
||||
```
|
||||
|
||||
**Monitor**: Watch for 10 minutes after restart to ensure stable
|
||||
|
||||
---
|
||||
|
||||
### INCIDENT 4: Disk Full (5 min recovery target)
|
||||
|
||||
**Detection**: `df -h` shows 90%+ usage, write operations failing
|
||||
|
||||
**Immediate (1 minute)**:
|
||||
```bash
|
||||
# Check disk usage
|
||||
du -sh /* | sort -rh | head -10
|
||||
|
||||
# Identify largest items
|
||||
du -sh data/ backups/ logs/
|
||||
```
|
||||
|
||||
**Recovery (order of priority)**:
|
||||
```bash
|
||||
# 1. Delete old backups (usually >90% of disk)
|
||||
find backups/ -name "inventory-*.tar.gz" -mtime +7 -delete
|
||||
# Safe: Backups older than 7 days
|
||||
# Aggressive: -mtime +3 (3 days)
|
||||
|
||||
# 2. Compress old logs
|
||||
gzip logs/*.log.* 2>/dev/null || true
|
||||
find logs/ -name "*.gz" -mtime +30 -delete
|
||||
|
||||
# 3. Vacuum database (if >500MB)
|
||||
sqlite3 data/inventory.db "VACUUM;"
|
||||
|
||||
# 4. Delete oldest backups if still full
|
||||
find backups/ -name "*.tar.gz" -type f | sort | head -1 | xargs rm
|
||||
```
|
||||
|
||||
**Verification**:
|
||||
```bash
|
||||
df -h # Should be <80% now
|
||||
du -sh backups/
|
||||
```
|
||||
|
||||
**Prevention**: Increase disk size or set up offsite backups
|
||||
|
||||
---
|
||||
|
||||
### INCIDENT 5: Data Corruption (10 min recovery target)
|
||||
|
||||
**Detection**: Database integrity check fails, unexpected data missing, query errors
|
||||
|
||||
**Immediate (1 minute)**:
|
||||
```bash
|
||||
# Verify corruption
|
||||
docker-compose exec backend sqlite3 /app/data/inventory.db "PRAGMA integrity_check;"
|
||||
# OR (Standalone)
|
||||
sqlite3 data/inventory.db "PRAGMA integrity_check;"
|
||||
|
||||
# Check logs for errors
|
||||
docker-compose logs backend | grep -i "error" | tail -20
|
||||
```
|
||||
|
||||
**Recovery (8-10 minutes)**:
|
||||
```bash
|
||||
# CRITICAL: Do not attempt to repair
|
||||
# Restore from backup (fastest, safest option)
|
||||
|
||||
# 1. Check available backups
|
||||
ls -lh backups/ | head -5
|
||||
|
||||
# 2. Stop services
|
||||
docker-compose down # Docker
|
||||
pkill -f uvicorn # Standalone
|
||||
|
||||
# 3. Restore
|
||||
./scripts/restore.sh backups/latest.tar.gz --validate
|
||||
|
||||
# 4. Restart (if needed)
|
||||
docker-compose up -d # Docker
|
||||
./start_server.sh # Standalone
|
||||
|
||||
# 5. Verify
|
||||
curl http://localhost:8000/health
|
||||
```
|
||||
|
||||
**Notify Users**: Data loss = up to 1 day (latest backup)
|
||||
|
||||
**Escalate**: Call database admin after recovery
|
||||
|
||||
---
|
||||
|
||||
### INCIDENT 6: Network / CORS Errors (5 min recovery target)
|
||||
|
||||
**Detection**: Browser console shows CORS error, frontend can't reach backend
|
||||
|
||||
**Immediate (1 minute)**:
|
||||
```bash
|
||||
# Test backend connectivity
|
||||
curl -v http://localhost:8000/health
|
||||
curl -v http://<server-ip>:8000/health
|
||||
|
||||
# Check CORS configuration
|
||||
docker-compose exec backend python -c "
|
||||
from backend.config import ALLOWED_ORIGINS
|
||||
print('ALLOWED_ORIGINS:', ALLOWED_ORIGINS)
|
||||
"
|
||||
```
|
||||
|
||||
**Recovery**:
|
||||
```bash
|
||||
# 1. Check network connectivity
|
||||
ping <backend-server-ip>
|
||||
|
||||
# 2. Update CORS if needed
|
||||
# Edit inventory.env:
|
||||
EXTRA_ALLOWED_ORIGINS=<client-ip>
|
||||
|
||||
# 3. Restart backend
|
||||
docker-compose restart backend
|
||||
|
||||
# 4. Test
|
||||
curl -H "Origin: http://<client-ip>" -v http://localhost:8000/health
|
||||
```
|
||||
|
||||
**Verify**: Frontend should load without CORS errors
|
||||
|
||||
---
|
||||
|
||||
### INCIDENT 7: Frontend Not Loading (5 min recovery target)
|
||||
|
||||
**Detection**: Frontend port doesn't respond, blank page, 404 errors
|
||||
|
||||
**Recovery (Docker)**:
|
||||
```bash
|
||||
# Check service
|
||||
docker-compose ps | grep frontend
|
||||
|
||||
# Restart
|
||||
docker-compose restart frontend
|
||||
|
||||
# Check logs
|
||||
docker-compose logs frontend | tail -50
|
||||
|
||||
# If build failed, rebuild
|
||||
docker-compose down
|
||||
docker-compose up -d --build
|
||||
```
|
||||
|
||||
**Recovery (Standalone)**:
|
||||
```bash
|
||||
# Kill process
|
||||
pkill -f "next start"
|
||||
|
||||
# Rebuild if needed
|
||||
cd frontend && npm install && npm run build
|
||||
|
||||
# Restart
|
||||
cd .. && npm start --prefix frontend &
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Emergency Decision Tree
|
||||
|
||||
```
|
||||
Service not responding?
|
||||
├─ YES: INCIDENT 1 (Service Down)
|
||||
└─ NO: Continue
|
||||
|
||||
Getting "locked" errors?
|
||||
├─ YES: INCIDENT 2 (Database Locked)
|
||||
└─ NO: Continue
|
||||
|
||||
High CPU/Memory?
|
||||
├─ YES: INCIDENT 3 (High Resources)
|
||||
└─ NO: Continue
|
||||
|
||||
Disk full?
|
||||
├─ YES: INCIDENT 4 (Disk Full)
|
||||
└─ NO: Continue
|
||||
|
||||
Data missing/corrupted?
|
||||
├─ YES: INCIDENT 5 (Data Corruption)
|
||||
└─ NO: Continue
|
||||
|
||||
CORS/Network errors?
|
||||
├─ YES: INCIDENT 6 (Network Issues)
|
||||
└─ NO: Continue
|
||||
|
||||
Frontend not loading?
|
||||
├─ YES: INCIDENT 7 (Frontend Error)
|
||||
└─ NO: Contact developer support
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Escalation Path
|
||||
|
||||
### Tier 1: On-Call Operations (You are here)
|
||||
- [ ] Attempt immediate recovery (restart, clear locks)
|
||||
- [ ] Document issue and time
|
||||
- [ ] If not resolved in 5 minutes → Escalate
|
||||
|
||||
### Tier 2: Senior DevOps / Backup On-Call
|
||||
- [ ] Call: [Phone]
|
||||
- [ ] Message: "TFM Inventory [INCIDENT]: [Description]"
|
||||
- [ ] Provide: Error messages, logs, recovery attempts
|
||||
|
||||
### Tier 3: Application Developer
|
||||
- [ ] If Tier 2 unresponsive for 10 minutes
|
||||
- [ ] Call: [Phone]
|
||||
- [ ] Include: Full logs, screenshots
|
||||
|
||||
### Tier 4: Management
|
||||
- [ ] If service down >30 minutes
|
||||
- [ ] Notify: [Manager], [Director]
|
||||
|
||||
---
|
||||
|
||||
## Post-Incident Actions
|
||||
|
||||
**Within 1 hour**:
|
||||
- [ ] Document issue and resolution
|
||||
- [ ] Note start time, detection time, resolution time
|
||||
- [ ] Save error logs to archive
|
||||
|
||||
**Within 24 hours**:
|
||||
- [ ] Root cause analysis
|
||||
- [ ] Identify prevention measures
|
||||
- [ ] Update runbooks if needed
|
||||
|
||||
**Within 1 week**:
|
||||
- [ ] Implement preventive fix
|
||||
- [ ] Update monitoring rules
|
||||
- [ ] Run incident review with team
|
||||
|
||||
---
|
||||
|
||||
## Critical Contacts
|
||||
|
||||
| Role | Name | Phone | Email |
|
||||
|------|------|-------|-------|
|
||||
| On-Call Ops | [Name] | [+1-xxx-xxx-xxxx] | [Email] |
|
||||
| Backup Ops | [Name] | [+1-xxx-xxx-xxxx] | [Email] |
|
||||
| Senior DevOps | [Name] | [+1-xxx-xxx-xxxx] | [Email] |
|
||||
| Developer | [Name] | [+1-xxx-xxx-xxxx] | [Email] |
|
||||
|
||||
---
|
||||
|
||||
## Cheat Sheet (Print and Post)
|
||||
|
||||
```
|
||||
QUICK FIXES:
|
||||
|
||||
Service Down?
|
||||
docker-compose restart backend
|
||||
|
||||
Database Locked?
|
||||
docker-compose restart backend
|
||||
|
||||
Disk Full?
|
||||
find backups/ -name "*.tar.gz" -mtime +7 -delete
|
||||
|
||||
Data Corrupted?
|
||||
./scripts/restore.sh backups/latest.tar.gz --validate
|
||||
|
||||
CORS Error?
|
||||
Edit inventory.env + docker-compose restart backend
|
||||
|
||||
Check Health:
|
||||
curl http://localhost:8000/health
|
||||
|
||||
View Logs:
|
||||
docker-compose logs -f backend
|
||||
|
||||
CONTACTS:
|
||||
On-call: [Phone]
|
||||
Dev Support: [Email]
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**Version**: 1.0
|
||||
**Last Updated**: 2026-04-22
|
||||
**Next Review**: 2026-05-22
|
||||
**Owner**: Operations Team
|
||||
192
docs/HEALTH_MONITORING_CHECKLIST.md
Normal file
192
docs/HEALTH_MONITORING_CHECKLIST.md
Normal file
@@ -0,0 +1,192 @@
|
||||
# Health Monitoring Checklist
|
||||
|
||||
Use this checklist for daily/weekly health reviews. Adapt commands for Docker or Standalone mode as needed.
|
||||
|
||||
---
|
||||
|
||||
## Daily Checks (5 minutes)
|
||||
|
||||
Print this section and post near the server or set email reminders.
|
||||
|
||||
- [ ] **All services running**
|
||||
- Docker: `docker-compose ps` (expect: All "Up")
|
||||
- Standalone: `ps aux | grep -E "(uvicorn|next)" | grep -v grep` (expect: 2 processes)
|
||||
|
||||
- [ ] **API responsive**
|
||||
- `curl -w "\nHTTP %{http_code}\n" http://localhost:8000/health`
|
||||
- Expected: 200 OK, response <100ms
|
||||
|
||||
- [ ] **Frontend loads**
|
||||
- `curl -w "\nHTTP %{http_code}\n" http://localhost:3000`
|
||||
- Expected: 200 OK
|
||||
|
||||
- [ ] **Recent errors in logs**
|
||||
- Docker: `docker-compose logs | grep ERROR | tail -5`
|
||||
- Standalone: `tail -20 logs/*.log | grep ERROR`
|
||||
- Action: Investigate any ERROR-level logs
|
||||
|
||||
- [ ] **Database accessible**
|
||||
- Docker: `docker-compose exec backend sqlite3 /app/data/inventory.db "SELECT COUNT(*) FROM items;"`
|
||||
- Standalone: `sqlite3 data/inventory.db "SELECT COUNT(*) FROM items;"`
|
||||
- Action: If fails, restart backend service
|
||||
|
||||
---
|
||||
|
||||
## Weekly Checks (15 minutes)
|
||||
|
||||
- [ ] **Backup completed**
|
||||
- `ls -lh backups/ | head -1`
|
||||
- Check timestamp is within last 24 hours
|
||||
- Action: Manual backup if needed: `./scripts/backup.sh manual`
|
||||
|
||||
- [ ] **Disk usage within limits**
|
||||
- `du -sh data/ config/ backups/`
|
||||
- Expected: data/ <5GB, backups/ <10GB
|
||||
- Action: If backups >10GB, verify cron retention settings
|
||||
|
||||
- [ ] **Database size reasonable**
|
||||
- `du -h data/inventory.db`
|
||||
- Action: If >1GB, consider optimization (vacuum/index)
|
||||
|
||||
- [ ] **Service resource usage**
|
||||
- Docker: `docker stats --no-stream`
|
||||
- Standalone: `ps aux | grep -E "(uvicorn|next)"`
|
||||
- Expected: Backend <70% CPU, <500MB RAM
|
||||
|
||||
- [ ] **Log files not growing excessively**
|
||||
- `ls -lh logs/ | tail -10`
|
||||
- Action: If any log >100MB, consider rotation
|
||||
|
||||
- [ ] **Check for hung processes**
|
||||
- `ps aux | grep -E "(defunct|defunct)"` (should be empty)
|
||||
- Action: Kill hung processes
|
||||
|
||||
---
|
||||
|
||||
## Monthly Checks (30 minutes)
|
||||
|
||||
- [ ] **Restore from backup test**
|
||||
- On staging environment:
|
||||
```bash
|
||||
./scripts/restore.sh backups/latest.tar.gz --validate
|
||||
```
|
||||
- Confirm zero data loss, all services healthy
|
||||
- Action: If fails, investigate and fix immediately
|
||||
|
||||
- [ ] **Scaling capacity review**
|
||||
- Current: Single instance, 5 concurrent users stable
|
||||
- Actual usage: _____ concurrent users
|
||||
- Action: If approaching 5 users, plan for v3 multi-instance
|
||||
|
||||
- [ ] **Security audit**
|
||||
- [ ] JWT_SECRET_KEY still secure (not exposed in logs)
|
||||
- [ ] LDAP credentials (if used) still valid
|
||||
- [ ] API logs show no unauthorized access attempts
|
||||
- Check: `docker-compose logs backend | grep -i "denied\|failed\|unauthorized" | tail -20`
|
||||
|
||||
- [ ] **Documentation review**
|
||||
- [ ] Runbook matches current deployment
|
||||
- [ ] Troubleshooting section covers recent issues
|
||||
- [ ] Contact info still current
|
||||
|
||||
---
|
||||
|
||||
## Alert Thresholds
|
||||
|
||||
| Metric | Warning | Critical | Action |
|
||||
|--------|---------|----------|--------|
|
||||
| CPU (backend) | >50% | >70% | Restart container, investigate slow queries |
|
||||
| Memory (backend) | >400MB | >600MB | Restart container, check for memory leak |
|
||||
| Disk (backups) | >10GB | >15GB | Delete old backups, increase retention |
|
||||
| API response (p95) | >500ms | >1s | Check slow query logs, restart backend |
|
||||
| Backup age | >36 hours | >48 hours | Manual run needed, check cron |
|
||||
| Database locked | 1 event/week | 5+ events/week | Investigate, may need v3 upgrade |
|
||||
| Error rate | >0.1% | >1% | Investigate logs, restart if needed |
|
||||
|
||||
---
|
||||
|
||||
## Quick Troubleshooting Reference
|
||||
|
||||
**Service down?**
|
||||
→ `docker-compose ps` or `ps aux | grep uvicorn`
|
||||
→ `docker-compose logs SERVICE_NAME` or `tail logs/*.log`
|
||||
→ `docker-compose restart SERVICE_NAME` or `pkill -f uvicorn`
|
||||
|
||||
**Slow responses?**
|
||||
→ `docker stats` or `ps aux | grep uvicorn`
|
||||
→ `docker-compose logs backend | grep "slow"` or check logs
|
||||
→ Restart backend or plan for capacity increase
|
||||
|
||||
**Database locked?**
|
||||
→ Restart backend: `docker-compose restart backend`
|
||||
|
||||
**Out of disk space?**
|
||||
→ `du -sh data/ backups/`
|
||||
→ Clean old backups: `find backups/ -name "*.tar.gz" -mtime +30 -delete`
|
||||
→ Extend volume if needed
|
||||
|
||||
**HTTPS certificate issues?**
|
||||
→ `rm -rf data/caddy_*` (Docker mode)
|
||||
→ `docker-compose restart proxy`
|
||||
→ Wait 30 seconds for new certs to generate
|
||||
|
||||
---
|
||||
|
||||
## Health Check Commands by Deployment Mode
|
||||
|
||||
### Docker Mode
|
||||
|
||||
```bash
|
||||
# Full health check suite
|
||||
echo "=== Services ===" && docker-compose ps
|
||||
echo "=== API Health ===" && curl -s http://localhost:8000/health | jq .
|
||||
echo "=== Database ===" && docker-compose exec backend sqlite3 /app/data/inventory.db "SELECT COUNT(*) FROM items;"
|
||||
echo "=== Resources ===" && docker stats --no-stream | head -5
|
||||
echo "=== Recent Errors ===" && docker-compose logs --tail=20 | grep ERROR
|
||||
```
|
||||
|
||||
### Standalone Mode
|
||||
|
||||
```bash
|
||||
# Full health check suite
|
||||
echo "=== Processes ===" && ps aux | grep -E "(uvicorn|next)" | grep -v grep
|
||||
echo "=== API Health ===" && curl -s http://localhost:8000/health
|
||||
echo "=== Database ===" && sqlite3 data/inventory.db "SELECT COUNT(*) FROM items;"
|
||||
echo "=== Resources ===" && top -bn1 | head -20
|
||||
echo "=== Recent Errors ===" && tail -50 logs/*.log | grep ERROR
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Monitoring Checklist Template
|
||||
|
||||
Print and use weekly:
|
||||
|
||||
```
|
||||
Week of: ___________
|
||||
|
||||
Daily (✓ = pass, X = fail, note issues):
|
||||
Mon: [ ] Services [ ] API [ ] DB [ ] Errors Notes: _______________
|
||||
Tue: [ ] Services [ ] API [ ] DB [ ] Errors Notes: _______________
|
||||
Wed: [ ] Services [ ] API [ ] DB [ ] Errors Notes: _______________
|
||||
Thu: [ ] Services [ ] API [ ] DB [ ] Errors Notes: _______________
|
||||
Fri: [ ] Services [ ] API [ ] DB [ ] Errors Notes: _______________
|
||||
Sat: [ ] Services [ ] API [ ] DB [ ] Errors Notes: _______________
|
||||
Sun: [ ] Services [ ] API [ ] DB [ ] Errors Notes: _______________
|
||||
|
||||
Weekly Review:
|
||||
- [ ] Backup completed within 24h
|
||||
- [ ] Disk usage acceptable
|
||||
- [ ] DB size reasonable
|
||||
- [ ] Resource usage normal
|
||||
- [ ] No log errors unresolved
|
||||
|
||||
Issues Found: ___________________________________________________
|
||||
Actions Taken: __________________________________________________
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
**Last Updated**: 2026-04-22
|
||||
**Next Review**: 2026-05-22
|
||||
**Maintained By**: Operations Team
|
||||
480
docs/OPERATIONAL_RUNBOOK.md
Normal file
480
docs/OPERATIONAL_RUNBOOK.md
Normal file
@@ -0,0 +1,480 @@
|
||||
# Operational Runbook
|
||||
|
||||
**Audience**: Systems operators, site managers, DevOps teams
|
||||
**Target**: Minimal training required; step-by-step procedures
|
||||
**Last Updated**: 2026-04-22
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
This runbook covers operational procedures for both Docker and Standalone deployment modes. Both modes use the same configuration files (inventory.env) and backup/restore scripts.
|
||||
|
||||
---
|
||||
|
||||
## 1. Initial Deployment
|
||||
|
||||
### Requirements
|
||||
|
||||
- Ubuntu 22.04 LTS or similar
|
||||
- For Docker mode: Docker and Docker Compose installed
|
||||
- For Standalone mode: Python 3.12+, Node.js 18+, npm
|
||||
- 2GB RAM minimum, 10GB disk (recommended: 4GB/50GB for production)
|
||||
- Internet access (first-time setup only)
|
||||
|
||||
### Docker Deployment Steps
|
||||
|
||||
1. **Clone repository**
|
||||
```bash
|
||||
git clone <repo_url> /opt/tfm-inventory
|
||||
cd /opt/tfm-inventory
|
||||
```
|
||||
|
||||
2. **Configure environment**
|
||||
```bash
|
||||
cp inventory.env.template inventory.env
|
||||
# Edit inventory.env with your settings:
|
||||
# - BACKEND_PORT=8000, FRONTEND_PORT=3000
|
||||
# - JWT_SECRET_KEY (generate: openssl rand -hex 32)
|
||||
# - AI settings (Gemini/Claude API keys, optional)
|
||||
# - LDAP settings (if using enterprise auth)
|
||||
```
|
||||
|
||||
3. **Deploy**
|
||||
```bash
|
||||
chmod +x deploy.sh scripts/backup.sh scripts/restore.sh
|
||||
./deploy.sh production
|
||||
```
|
||||
|
||||
4. **Verify deployment**
|
||||
- Frontend: http://your-server:3000
|
||||
- Backend API: http://your-server:8000
|
||||
- API Docs: http://your-server:8000/docs
|
||||
- Health check: `curl http://localhost:8000/health`
|
||||
|
||||
5. **Create admin user** (if not auto-created)
|
||||
```bash
|
||||
docker-compose exec backend python -c "
|
||||
from backend.db import SessionLocal, User
|
||||
db = SessionLocal()
|
||||
user = User(username='admin', hashed_password='...', is_admin=True)
|
||||
db.add(user)
|
||||
db.commit()
|
||||
"
|
||||
```
|
||||
|
||||
### Standalone Deployment Steps
|
||||
|
||||
1. **Clone repository**
|
||||
```bash
|
||||
git clone <repo_url> /opt/tfm-inventory
|
||||
cd /opt/tfm-inventory
|
||||
```
|
||||
|
||||
2. **Configure environment**
|
||||
```bash
|
||||
cp inventory.env.template inventory.env
|
||||
# Edit with same settings as Docker mode
|
||||
```
|
||||
|
||||
3. **Install dependencies**
|
||||
```bash
|
||||
# Backend
|
||||
cd backend
|
||||
python3 -m venv venv
|
||||
source venv/bin/activate
|
||||
pip install -r requirements.txt
|
||||
cd ..
|
||||
|
||||
# Frontend
|
||||
cd frontend
|
||||
npm install
|
||||
cd ..
|
||||
```
|
||||
|
||||
4. **Deploy**
|
||||
```bash
|
||||
chmod +x start_server.sh scripts/backup.sh scripts/restore.sh
|
||||
./start_server.sh
|
||||
```
|
||||
|
||||
5. **Verify deployment**
|
||||
- Frontend: http://localhost:3000
|
||||
- Backend API: http://localhost:8000
|
||||
- Health check: `curl http://localhost:8000/health`
|
||||
|
||||
---
|
||||
|
||||
## 2. Daily Operations
|
||||
|
||||
### Health Checks (Daily, ~5 minutes)
|
||||
|
||||
**Docker mode:**
|
||||
```bash
|
||||
# Check all services
|
||||
docker-compose ps
|
||||
# Expected: All services "Up"
|
||||
|
||||
# Check API health
|
||||
curl http://localhost:8000/health
|
||||
# Expected: 200 OK, response time <100ms
|
||||
|
||||
# Check database
|
||||
du -h data/inventory.db
|
||||
|
||||
# Check for errors
|
||||
docker-compose logs | grep ERROR | tail -5
|
||||
```
|
||||
|
||||
**Standalone mode:**
|
||||
```bash
|
||||
# Check processes
|
||||
ps aux | grep -E "(uvicorn|next)" | grep -v grep
|
||||
|
||||
# Check API health
|
||||
curl http://localhost:8000/health
|
||||
|
||||
# Check database
|
||||
du -h data/inventory.db
|
||||
|
||||
# Check logs
|
||||
tail -50 logs/backend.log logs/frontend.log 2>/dev/null
|
||||
```
|
||||
|
||||
### Backup (Automated)
|
||||
|
||||
```bash
|
||||
# Verify automatic backup ran (cron jobs)
|
||||
ls -lh backups/ | head -1
|
||||
# Expected: File timestamp within last 24 hours
|
||||
|
||||
# Manual backup (if needed)
|
||||
./scripts/backup.sh manual
|
||||
|
||||
# View backup schedule
|
||||
sudo crontab -l | grep backup
|
||||
```
|
||||
|
||||
### Monitoring
|
||||
|
||||
**Docker mode:**
|
||||
```bash
|
||||
# Real-time logs
|
||||
docker-compose logs -f
|
||||
|
||||
# Backend performance
|
||||
docker stats --no-stream | grep backend
|
||||
|
||||
# Database status
|
||||
docker-compose exec backend sqlite3 /app/data/inventory.db \
|
||||
"SELECT COUNT(*) as item_count, SUM(quantity) as total_qty FROM items;"
|
||||
```
|
||||
|
||||
**Standalone mode:**
|
||||
```bash
|
||||
# Real-time logs
|
||||
tail -f logs/backend.log logs/frontend.log
|
||||
|
||||
# System resources
|
||||
top -p $(pgrep -f uvicorn | head -1)
|
||||
|
||||
# Database status
|
||||
sqlite3 data/inventory.db \
|
||||
"SELECT COUNT(*) as item_count, SUM(quantity) as total_qty FROM items;"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Troubleshooting
|
||||
|
||||
### Service Won't Start
|
||||
|
||||
**Docker mode:**
|
||||
```bash
|
||||
# Check Docker daemon
|
||||
docker ps
|
||||
|
||||
# Check port conflicts
|
||||
netstat -tuln | grep -E "8000|3000|8906|8907"
|
||||
|
||||
# View service logs
|
||||
docker-compose logs backend
|
||||
docker-compose logs frontend
|
||||
docker-compose logs proxy
|
||||
```
|
||||
|
||||
**Standalone mode:**
|
||||
```bash
|
||||
# Check if processes are running
|
||||
ps aux | grep -E "(uvicorn|next)"
|
||||
|
||||
# Check port conflicts
|
||||
netstat -tuln | grep -E "8000|3000"
|
||||
|
||||
# Check logs
|
||||
cat logs/backend.log | tail -50
|
||||
```
|
||||
|
||||
### High CPU/Memory
|
||||
|
||||
**Docker mode:**
|
||||
```bash
|
||||
# Identify container
|
||||
docker stats --no-stream
|
||||
|
||||
# Restart container
|
||||
docker-compose restart backend
|
||||
|
||||
# Check for slow queries
|
||||
docker-compose logs backend | grep "slow"
|
||||
```
|
||||
|
||||
**Standalone mode:**
|
||||
```bash
|
||||
# Kill and restart
|
||||
pkill -f uvicorn
|
||||
pkill -f "next start"
|
||||
sleep 2
|
||||
./start_server.sh
|
||||
```
|
||||
|
||||
### Database Locked
|
||||
|
||||
**Docker mode:**
|
||||
```bash
|
||||
docker-compose restart backend
|
||||
# Wait 30 seconds
|
||||
docker-compose exec backend sqlite3 /app/data/inventory.db "PRAGMA journal_mode;"
|
||||
```
|
||||
|
||||
**Standalone mode:**
|
||||
```bash
|
||||
pkill -f uvicorn
|
||||
sleep 2
|
||||
# Restart backend only (no need for frontend restart)
|
||||
cd backend && source venv/bin/activate && uvicorn main:app --host 0.0.0.0 --port 8000 &
|
||||
```
|
||||
|
||||
### HTTPS Certificate Issues
|
||||
|
||||
**Docker mode:**
|
||||
```bash
|
||||
# Certificates regenerated automatically
|
||||
# If issues persist:
|
||||
rm -rf data/caddy_*
|
||||
docker-compose restart proxy
|
||||
# Wait 30 seconds for new certs to generate
|
||||
```
|
||||
|
||||
**Standalone mode:**
|
||||
```bash
|
||||
# For local development/testing, HTTP is sufficient
|
||||
# For production HTTPS, configure reverse proxy (nginx/Caddy) separately
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4. Backup & Restore
|
||||
|
||||
### Automated Backups
|
||||
|
||||
```bash
|
||||
# Verify cron jobs are installed
|
||||
sudo bash config/backup-cron.sh
|
||||
|
||||
# View backup history
|
||||
ls -lh backups/
|
||||
|
||||
# Check backup log
|
||||
tail -20 logs/backup-daily.log
|
||||
```
|
||||
|
||||
**Backup Schedule:**
|
||||
- Daily: 2 AM, retention 30 days
|
||||
- Weekly: 3 AM Sundays, retention 90 days
|
||||
|
||||
### Manual Backup
|
||||
|
||||
```bash
|
||||
# Create backup
|
||||
./scripts/backup.sh manual
|
||||
|
||||
# Verify backup created and is valid
|
||||
tar -tzf backups/inventory-*.tar.gz | head
|
||||
```
|
||||
|
||||
### Manual Restore
|
||||
|
||||
```bash
|
||||
# List available backups
|
||||
ls backups/
|
||||
|
||||
# Restore specific backup (Docker mode)
|
||||
./scripts/restore.sh backups/inventory-2026-04-22_14-30-15.tar.gz --validate
|
||||
|
||||
# Restore specific backup (Standalone mode)
|
||||
./scripts/restore.sh backups/inventory-2026-04-22_14-30-15.tar.gz
|
||||
# Then restart: ./start_server.sh
|
||||
|
||||
# Validate data after restore
|
||||
curl http://localhost:8000/health
|
||||
```
|
||||
|
||||
**Recovery Objectives:**
|
||||
- RTO (Recovery Time): <10 minutes
|
||||
- RPO (Recovery Point): 1 day (daily backup)
|
||||
|
||||
---
|
||||
|
||||
## 5. Disaster Recovery
|
||||
|
||||
### Complete System Failure (Hardware or Data Corruption)
|
||||
|
||||
**For Docker:**
|
||||
1. Provision new Ubuntu 22.04 LTS server (same specs)
|
||||
2. Clone repository: `git clone <repo> /opt/tfm-inventory && cd /opt/tfm-inventory`
|
||||
3. Copy latest backup from offsite or previous backup directory
|
||||
4. Restore: `./scripts/restore.sh /path/to/backup.tar.gz --validate`
|
||||
5. Update DNS/load balancer to new server IP
|
||||
6. Verify all services healthy and data present
|
||||
|
||||
**For Standalone:**
|
||||
1. Provision new Ubuntu 22.04 LTS server
|
||||
2. Clone repository
|
||||
3. Install dependencies (Python venv, Node.js)
|
||||
4. Copy latest backup
|
||||
5. Restore: `./scripts/restore.sh /path/to/backup.tar.gz`
|
||||
6. Start services: `./start_server.sh`
|
||||
7. Verify connectivity and data
|
||||
|
||||
**RTO**: <30 minutes (provisioning + restore)
|
||||
**RPO**: 1 day (latest backup)
|
||||
|
||||
### Data Center Failure
|
||||
|
||||
1. Activate secondary site or failover to cloud
|
||||
2. Clone repository on new infrastructure
|
||||
3. Restore latest backup: `./scripts/restore.sh backup.tar.gz --validate`
|
||||
4. Update DNS to new location
|
||||
5. Notify users of recovery (1-day data loss acceptable)
|
||||
|
||||
---
|
||||
|
||||
## 6. Scaling Operations
|
||||
|
||||
### Adding Users (5+ concurrent)
|
||||
|
||||
Current configuration supports 5 concurrent users safely.
|
||||
|
||||
**Docker mode:**
|
||||
```bash
|
||||
# Increase backend memory
|
||||
# Edit docker-compose.yml:
|
||||
# backend:
|
||||
# mem_limit: 4g
|
||||
|
||||
# Increase database connections
|
||||
docker-compose exec backend \
|
||||
python -c "import backend.config; print(backend.config.DB_POOL_SIZE)"
|
||||
```
|
||||
|
||||
**Standalone mode:**
|
||||
```bash
|
||||
# Increase Python process resources
|
||||
# Edit start_server.sh to add workers/processes if using Gunicorn
|
||||
|
||||
# Monitor memory usage
|
||||
ps aux | grep uvicorn
|
||||
```
|
||||
|
||||
### Database Growth (10K+ items)
|
||||
|
||||
As inventory grows beyond 10K items:
|
||||
1. Monitor query performance: `PRAGMA optimize;`
|
||||
2. Create indexes on frequently searched columns
|
||||
3. Vacuum database: `VACUUM;`
|
||||
4. Consider archiving old audit logs (v3 feature)
|
||||
|
||||
---
|
||||
|
||||
## 7. Updates & Upgrades
|
||||
|
||||
### Patch Update (v1.14.x → v1.14.y)
|
||||
|
||||
```bash
|
||||
# Backup first
|
||||
./scripts/backup.sh manual
|
||||
|
||||
# Pull latest code
|
||||
git pull origin main
|
||||
|
||||
# Docker mode:
|
||||
./deploy.sh production --rebuild
|
||||
|
||||
# Standalone mode:
|
||||
pkill -f uvicorn
|
||||
pkill -f "next start"
|
||||
cd backend && source venv/bin/activate && pip install -r requirements.txt
|
||||
cd ../frontend && npm install && npm run build
|
||||
./start_server.sh
|
||||
|
||||
# Verify
|
||||
curl http://localhost:8000/health
|
||||
```
|
||||
|
||||
### Major Update (v1.x → v2.x)
|
||||
|
||||
```bash
|
||||
# Create backup before proceeding
|
||||
./scripts/backup.sh manual
|
||||
|
||||
# Review CHANGELOG for breaking changes
|
||||
cat CHANGELOG.md | grep "v2.0"
|
||||
|
||||
# Test in staging first (restore backup there)
|
||||
./scripts/restore.sh backups/production.tar.gz
|
||||
|
||||
# If staging successful, proceed to production
|
||||
git checkout v2.0
|
||||
./deploy.sh production --rebuild # Docker mode
|
||||
# OR
|
||||
./start_server.sh # Standalone mode
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8. Performance Baseline
|
||||
|
||||
- Backend: <100ms API response time at 5 concurrent users
|
||||
- Frontend: <1s page load
|
||||
- Database: <500 queries/min with 10K items
|
||||
- Memory: Backend <500MB, Frontend <200MB
|
||||
- CPU: Both services <70% usage under normal load
|
||||
|
||||
---
|
||||
|
||||
## 9. Emergency Contacts & Escalation
|
||||
|
||||
- **Developer Support**: dev@example.com
|
||||
- **Infrastructure**: ops@example.com
|
||||
- **24/7 On-call**: [contact info]
|
||||
|
||||
---
|
||||
|
||||
## Appendix: Quick Reference
|
||||
|
||||
| Task | Docker Command | Standalone Command |
|
||||
|------|---|---|
|
||||
| Health Check | `docker-compose ps` | `ps aux \| grep -E "(uvicorn\|next)"` |
|
||||
| View Logs | `docker-compose logs -f` | `tail -f logs/*.log` |
|
||||
| Restart Backend | `docker-compose restart backend` | `pkill -f uvicorn; ./start_server.sh` |
|
||||
| Backup | `./scripts/backup.sh manual` | `./scripts/backup.sh manual` |
|
||||
| Restore | `./scripts/restore.sh file.tar.gz` | `./scripts/restore.sh file.tar.gz` |
|
||||
| Stop Services | `docker-compose down` | `pkill -f uvicorn; pkill -f "next"` |
|
||||
|
||||
---
|
||||
|
||||
**Version**: 1.0
|
||||
**Last Updated**: 2026-04-22
|
||||
**Maintained By**: Operations Team
|
||||
**Next Review**: 2026-05-22
|
||||
@@ -1,5 +1,10 @@
|
||||
FROM node:20-alpine AS base
|
||||
|
||||
# Metadata labels
|
||||
LABEL maintainer="TFM aInventory Team"
|
||||
LABEL version="2.0.0"
|
||||
LABEL description="TFM aInventory Frontend Web Service"
|
||||
|
||||
# Step 1: Install dependencies
|
||||
FROM base AS deps
|
||||
RUN apk add --no-cache libc6-compat
|
||||
@@ -19,7 +24,7 @@ RUN npm run build
|
||||
|
||||
# Step 3: Production image
|
||||
FROM base AS runner
|
||||
RUN apk add --no-cache su-exec
|
||||
RUN apk add --no-cache su-exec curl
|
||||
WORKDIR /app
|
||||
|
||||
ENV NODE_ENV production
|
||||
@@ -47,5 +52,8 @@ EXPOSE 3000
|
||||
ENV PORT 3000
|
||||
ENV HOSTNAME "0.0.0.0"
|
||||
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
# Health check — verify frontend is responsive
|
||||
HEALTHCHECK --interval=30s --timeout=10s --start-period=40s --retries=3 \
|
||||
CMD curl -f http://localhost:3000/ || exit 1
|
||||
|
||||
ENTRYPOINT ["/app/entrypoint.sh"]
|
||||
|
||||
128
scripts/backup.sh
Normal file
128
scripts/backup.sh
Normal file
@@ -0,0 +1,128 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Phase 6, Plan 02, Task 1: Automated Backup Script
|
||||
# Supports both Docker and Standalone deployment modes
|
||||
# Usage: ./backup.sh [daily|weekly|manual] [retention_days]
|
||||
|
||||
BACKUP_TYPE="${1:-manual}"
|
||||
RETENTION_DAYS="${2:-30}"
|
||||
BACKUP_DIR="./backups"
|
||||
DATA_DIR="./data"
|
||||
TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S)
|
||||
BACKUP_FILE="$BACKUP_DIR/inventory-$TIMESTAMP.tar.gz"
|
||||
|
||||
# Colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
|
||||
|
||||
# Create backup directory
|
||||
mkdir -p "$BACKUP_DIR"
|
||||
|
||||
# Determine deployment mode
|
||||
if command -v docker-compose &> /dev/null; then
|
||||
log_info "Docker deployment mode detected"
|
||||
IS_DOCKER=true
|
||||
|
||||
# Verify Docker daemon is running
|
||||
docker ps > /dev/null 2>&1 || log_error "Docker daemon not running"
|
||||
|
||||
# 1. Verify services are running
|
||||
log_info "Checking services..."
|
||||
if ! docker-compose ps | grep -q "Up"; then
|
||||
log_warn "Not all services running; attempting to start..."
|
||||
docker-compose up -d
|
||||
fi
|
||||
|
||||
# 2. Stop backend to ensure DB consistency
|
||||
log_info "Stopping backend service (DB consistency)..."
|
||||
docker-compose stop backend
|
||||
else
|
||||
log_info "Standalone deployment mode detected"
|
||||
IS_DOCKER=false
|
||||
|
||||
# For standalone mode, we'll just backup while running
|
||||
# (Process is single-threaded, so minimal locking risk)
|
||||
log_warn "Standalone mode: backing up with minimal service pause"
|
||||
fi
|
||||
|
||||
# Wait for graceful shutdown (Docker only)
|
||||
[[ "$IS_DOCKER" == "true" ]] && sleep 2
|
||||
|
||||
# 3. Create backup tarball
|
||||
log_info "Creating backup: $BACKUP_FILE"
|
||||
|
||||
tar --exclude="$DATA_DIR/caddy_*" \
|
||||
--exclude="$BACKUP_DIR" \
|
||||
--exclude=".git" \
|
||||
--exclude="node_modules" \
|
||||
--exclude=".next" \
|
||||
--exclude=".venv" \
|
||||
--exclude="__pycache__" \
|
||||
-czf "$BACKUP_FILE" \
|
||||
"$DATA_DIR/inventory.db" \
|
||||
"$DATA_DIR/inventory.db-wal" \
|
||||
"$DATA_DIR/inventory.db-shm" \
|
||||
"config/" \
|
||||
"inventory.env" \
|
||||
2>/dev/null || log_error "Backup creation failed"
|
||||
|
||||
# Restart backend (Docker only)
|
||||
if [[ "$IS_DOCKER" == "true" ]]; then
|
||||
log_info "Restarting backend service..."
|
||||
docker-compose start backend
|
||||
|
||||
# Wait for backend to be ready
|
||||
sleep 5
|
||||
|
||||
# Verify backend is healthy
|
||||
if curl -sf "http://localhost:8000/health" > /dev/null; then
|
||||
log_info "Backend restored and healthy"
|
||||
else
|
||||
log_warn "Backend not responding yet; check logs with 'docker-compose logs -f backend'"
|
||||
fi
|
||||
fi
|
||||
|
||||
# 4. Verify backup integrity
|
||||
log_info "Verifying backup integrity..."
|
||||
if tar -tzf "$BACKUP_FILE" > /dev/null 2>&1; then
|
||||
log_info "✓ Backup verified"
|
||||
else
|
||||
log_error "Backup corrupted"
|
||||
fi
|
||||
|
||||
# 5. Log backup metadata
|
||||
BACKUP_SIZE=$(du -h "$BACKUP_FILE" | cut -f1)
|
||||
BACKUP_VERSION=$(cat VERSION.json 2>/dev/null | grep version | cut -d'"' -f4 || echo "unknown")
|
||||
log_info "Backup size: $BACKUP_SIZE, Version: $BACKUP_VERSION"
|
||||
|
||||
# 6. Cleanup old backups
|
||||
log_info "Cleaning up backups older than $RETENTION_DAYS days..."
|
||||
find "$BACKUP_DIR" -name "inventory-*.tar.gz" -type f -mtime +$RETENTION_DAYS -delete
|
||||
|
||||
# 7. Create backup manifest
|
||||
cat > "$BACKUP_DIR/MANIFEST.txt" << EOF
|
||||
Backup Metadata
|
||||
===============
|
||||
Timestamp: $TIMESTAMP
|
||||
Type: $BACKUP_TYPE
|
||||
Retention: $RETENTION_DAYS days
|
||||
Size: $BACKUP_SIZE
|
||||
Version: $BACKUP_VERSION
|
||||
Deployment Mode: $([ "$IS_DOCKER" == "true" ] && echo "Docker" || echo "Standalone")
|
||||
Contents: DB, config, certificates (excluding certs)
|
||||
Restored: Not yet
|
||||
|
||||
Command to restore:
|
||||
./restore.sh $BACKUP_FILE
|
||||
EOF
|
||||
|
||||
log_info "Backup complete: $BACKUP_FILE"
|
||||
log_info "Retention policy: Delete after $RETENTION_DAYS days"
|
||||
log_info "Next backup: $(date -d '+1 day' '+%Y-%m-%d')"
|
||||
140
scripts/restore.sh
Normal file
140
scripts/restore.sh
Normal file
@@ -0,0 +1,140 @@
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# Phase 6, Plan 02, Task 2: Restore from Backup
|
||||
# Supports both Docker and Standalone deployment modes
|
||||
# Usage: ./restore.sh <backup_file> [--validate]
|
||||
|
||||
BACKUP_FILE="${1:-}"
|
||||
VALIDATE="${2:---validate}"
|
||||
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
NC='\033[0m'
|
||||
|
||||
log_info() { echo -e "${GREEN}[INFO]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
|
||||
|
||||
# Validate input
|
||||
[[ -z "$BACKUP_FILE" ]] && log_error "Usage: ./restore.sh <backup_file>"
|
||||
[[ ! -f "$BACKUP_FILE" ]] && log_error "Backup file not found: $BACKUP_FILE"
|
||||
|
||||
log_info "Restoring from: $BACKUP_FILE"
|
||||
|
||||
# Triple confirmation (security requirement)
|
||||
echo "⚠️ WARNING: This will overwrite current data!"
|
||||
read -p "Type 'RESTORE' to confirm (1/3): " confirm1
|
||||
[[ "$confirm1" != "RESTORE" ]] && log_error "Restore cancelled"
|
||||
|
||||
read -p "Type 'RESTORE' again to confirm (2/3): " confirm2
|
||||
[[ "$confirm2" != "RESTORE" ]] && log_error "Restore cancelled"
|
||||
|
||||
read -p "Type 'RESTORE' one more time to confirm (3/3): " confirm3
|
||||
[[ "$confirm3" != "RESTORE" ]] && log_error "Restore cancelled"
|
||||
|
||||
log_warn "Proceeding with restore..."
|
||||
|
||||
# Determine deployment mode
|
||||
if command -v docker-compose &> /dev/null; then
|
||||
log_info "Docker deployment mode detected"
|
||||
IS_DOCKER=true
|
||||
|
||||
# 1. Stop services
|
||||
log_info "Stopping services..."
|
||||
docker-compose down
|
||||
|
||||
# 2. Backup current data (safety copy)
|
||||
log_info "Creating safety backup of current data..."
|
||||
mkdir -p data/backups_before_restore
|
||||
tar -czf "data/backups_before_restore/backup-before-restore-$(date +%s).tar.gz" \
|
||||
data/inventory.db data/inventory.db-* 2>/dev/null || true
|
||||
|
||||
# 3. Extract backup
|
||||
log_info "Extracting backup..."
|
||||
tar -xzf "$BACKUP_FILE" -C . || log_error "Backup extraction failed"
|
||||
|
||||
# 4. Verify essential files
|
||||
log_info "Verifying restored files..."
|
||||
[[ -f "data/inventory.db" ]] || log_error "Database not found in backup"
|
||||
[[ -f "inventory.env" ]] || log_error "inventory.env not found in backup"
|
||||
|
||||
# 5. Restart services
|
||||
log_info "Restarting services..."
|
||||
docker-compose up -d
|
||||
|
||||
# 6. Wait for health
|
||||
log_info "Waiting for services to be healthy..."
|
||||
max_attempts=30
|
||||
attempt=0
|
||||
while [[ $attempt -lt $max_attempts ]]; do
|
||||
if curl -sf "http://localhost:8000/health" > /dev/null 2>&1; then
|
||||
log_info "Services healthy!"
|
||||
break
|
||||
fi
|
||||
attempt=$((attempt + 1))
|
||||
sleep 2
|
||||
done
|
||||
|
||||
if [[ $attempt -eq $max_attempts ]]; then
|
||||
log_warn "Services did not become healthy within 60 seconds"
|
||||
docker-compose logs backend
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# 7. Validation tests (if requested)
|
||||
if [[ "$VALIDATE" == "--validate" ]]; then
|
||||
log_info "Running validation tests..."
|
||||
|
||||
# Test 1: Database accessible
|
||||
DB_ITEMS=$(docker-compose exec backend sqlite3 /app/data/inventory.db \
|
||||
"SELECT COUNT(*) FROM items" 2>/dev/null || echo "0")
|
||||
log_info "✓ Database has $DB_ITEMS items"
|
||||
|
||||
# Test 2: API responsive
|
||||
if curl -sf "http://localhost:8000/health" > /dev/null; then
|
||||
log_info "✓ API health check passed"
|
||||
else
|
||||
log_error "API health check failed"
|
||||
fi
|
||||
|
||||
# Test 3: Frontend loads
|
||||
if curl -sf "http://localhost:3000" > /dev/null 2>&1; then
|
||||
log_info "✓ Frontend loads"
|
||||
else
|
||||
log_warn "⚠ Frontend check failed (normal on first startup)"
|
||||
fi
|
||||
fi
|
||||
|
||||
else
|
||||
log_info "Standalone deployment mode detected"
|
||||
IS_DOCKER=false
|
||||
|
||||
# 1. Kill any running servers
|
||||
log_info "Stopping running services..."
|
||||
pkill -f "next start" || true
|
||||
pkill -f "uvicorn" || true
|
||||
sleep 2
|
||||
|
||||
# 2. Backup current data
|
||||
log_info "Creating safety backup of current data..."
|
||||
mkdir -p data/backups_before_restore
|
||||
tar -czf "data/backups_before_restore/backup-before-restore-$(date +%s).tar.gz" \
|
||||
data/inventory.db data/inventory.db-* 2>/dev/null || true
|
||||
|
||||
# 3. Extract backup
|
||||
log_info "Extracting backup..."
|
||||
tar -xzf "$BACKUP_FILE" -C . || log_error "Backup extraction failed"
|
||||
|
||||
# 4. Verify essential files
|
||||
log_info "Verifying restored files..."
|
||||
[[ -f "data/inventory.db" ]] || log_error "Database not found in backup"
|
||||
[[ -f "inventory.env" ]] || log_error "inventory.env not found in backup"
|
||||
|
||||
log_info "✓ Restore complete"
|
||||
log_warn "Remember to start services: ./start_server.sh"
|
||||
fi
|
||||
|
||||
log_info "Restore complete!"
|
||||
log_warn "Remember to verify data in the application before returning to production"
|
||||
392
start_server.sh
392
start_server.sh
@@ -1,190 +1,220 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
# --- CONFIGURATION (Default values, will be overridden by network_configinventory.env) ---
|
||||
BACKEND_PORT=8000
|
||||
FRONTEND_PORT=3001
|
||||
BACKEND_SSL_PORT=3002
|
||||
FRONTEND_SSL_PORT=3003
|
||||
SERVER_IP="localhost"
|
||||
# Phase 6, Plan 1: Standalone Server Start Script
|
||||
# Usage: ./start_server.sh
|
||||
# Purpose: Start backend and frontend servers without Docker
|
||||
# Requirements: Python 3.12+, Node.js 20+, and dependencies installed
|
||||
|
||||
# Load Configuration from file if it exists
|
||||
CONFIG_PATH="$(cd "$(dirname "$0")" && pwd)/inventory.env"
|
||||
if [ -f "$CONFIG_PATH" ]; then
|
||||
echo "⚙️ Loading network configuration from $CONFIG_PATH..."
|
||||
# Export variables from inventory.env file (ignoring comments and empty lines)
|
||||
export $(grep -v '^#' "$CONFIG_PATH" | xargs)
|
||||
fi
|
||||
|
||||
echo "🚀 Starting TFM aInventory Stack with Dual Proxy..."
|
||||
|
||||
# 1. COMPREHENSIVE process cleanup - ensure absolute clean state
|
||||
echo "Sweep: Comprehensive process cleanup..."
|
||||
|
||||
# Kill all known hanging processes
|
||||
pkill -9 -f "uvicorn" 2>/dev/null || true
|
||||
pkill -9 -f "next-server" 2>/dev/null || true
|
||||
pkill -9 -f "local-ssl-proxy" 2>/dev/null || true
|
||||
pkill -9 -f "python.*backend" 2>/dev/null || true
|
||||
pkill -9 -f "python.*main:app" 2>/dev/null || true
|
||||
pkill -9 -f "npm run dev" 2>/dev/null || true
|
||||
pkill -9 -f "node.*next" 2>/dev/null || true
|
||||
|
||||
# Kill any remaining Python processes on port 8000 (backend)
|
||||
fuser -k 8000/tcp 2>/dev/null || true
|
||||
fuser -k 3001/tcp 2>/dev/null || true
|
||||
fuser -k 3002/tcp 2>/dev/null || true
|
||||
fuser -k 3003/tcp 2>/dev/null || true
|
||||
|
||||
# Extra wait to ensure processes are fully dead
|
||||
sleep 1
|
||||
|
||||
echo "✅ Cleanup complete - all processes terminated"
|
||||
|
||||
# 2. Setup/Activate Virtual Environment
|
||||
if [ ! -f ".venv/bin/activate" ]; then
|
||||
echo "🌑 Creating virtual environment (.venv)..."
|
||||
rm -rf .venv
|
||||
apt install python3-venv
|
||||
python3 -m venv .venv || { echo "❌ Failed to create venv"; exit 1; }
|
||||
fi
|
||||
source .venv/bin/activate || { echo "❌ Failed to activate venv"; exit 1; }
|
||||
|
||||
# 3. Check and Install Backend Dependencies
|
||||
echo "📦 Updating Python dependencies..."
|
||||
.venv/bin/pip install -r backend/requirements.txt
|
||||
|
||||
# 4. Get Local IP and set environment variables
|
||||
LOCAL_IP=$(hostname -I | awk '{print $1}' || echo "localhost")
|
||||
export ALLOWED_ORIGINS="http://localhost:$FRONTEND_PORT,http://localhost:$BACKEND_PORT,https://localhost:$FRONTEND_SSL_PORT,https://localhost:$BACKEND_SSL_PORT,https://$LOCAL_IP:$FRONTEND_SSL_PORT,https://$LOCAL_IP:$BACKEND_SSL_PORT"
|
||||
|
||||
# 4.0 Include EXTRA_ALLOWED_ORIGINS from inventory.env (VPN, Tailscale, etc.)
|
||||
if [ ! -z "$EXTRA_ALLOWED_ORIGINS" ]; then
|
||||
echo "🔌 Adding extra CORS origins from inventory.env..."
|
||||
IFS=',' read -ra EXTRA_ADDRS <<< "$EXTRA_ALLOWED_ORIGINS"
|
||||
for addr in "${EXTRA_ADDRS[@]}"; do
|
||||
TRIMMED=$(echo "$addr" | xargs)
|
||||
# Add both HTTP (for localhost dev) and HTTPS (for production)
|
||||
export ALLOWED_ORIGINS="$ALLOWED_ORIGINS,http://$TRIMMED:$FRONTEND_PORT,http://$TRIMMED:$BACKEND_PORT,https://$TRIMMED:$FRONTEND_SSL_PORT,https://$TRIMMED:$BACKEND_SSL_PORT"
|
||||
done
|
||||
fi
|
||||
|
||||
export JWT_SECRET_KEY="${JWT_SECRET_KEY:-ephemeral-dev-key-$(date +%s)}"
|
||||
export DATA_DIR="$(cd "$(dirname "$0")" && pwd)/data"
|
||||
export LOGS_DIR="$(cd "$(dirname "$0")" && pwd)/logs"
|
||||
|
||||
# First-run: initialize data directories and config templates
|
||||
echo "🔧 Initializing runtime data directories..."
|
||||
bash "$(cd "$(dirname "$0")" && pwd)/scripts/init_data.sh"
|
||||
|
||||
# 4.1 Sync Network Config to Frontend for runtime discovery
|
||||
echo "🔌 Syncing network configuration to frontend..."
|
||||
cat <<EOF > frontend/public/network.json
|
||||
{
|
||||
"SERVER_IP": "$SERVER_IP",
|
||||
"BACKEND_PORT": $BACKEND_PORT,
|
||||
"BACKEND_SSL_PORT": $BACKEND_SSL_PORT,
|
||||
"FRONTEND_PORT": $FRONTEND_PORT,
|
||||
"FRONTEND_SSL_PORT": $FRONTEND_SSL_PORT
|
||||
}
|
||||
EOF
|
||||
|
||||
echo "🔥 Starting Backend on port $BACKEND_PORT..."
|
||||
echo " CORS origins: $ALLOWED_ORIGINS"
|
||||
.venv/bin/python -m uvicorn backend.main:app --host 0.0.0.0 --port $BACKEND_PORT --reload &
|
||||
|
||||
# 5. Prepare Frontend Dev Origins from EXTRA_ALLOWED_ORIGINS
|
||||
# Convert subnet notation (10.0.0.0/24) to wildcard patterns (10.0.0.*)
|
||||
ALLOWED_DEV_ORIGINS="localhost,127.0.0.1,*.local"
|
||||
if [ ! -z "$EXTRA_ALLOWED_ORIGINS" ]; then
|
||||
IFS=',' read -ra EXTRA_ADDRS <<< "$EXTRA_ALLOWED_ORIGINS"
|
||||
for addr in "${EXTRA_ADDRS[@]}"; do
|
||||
TRIMMED=$(echo "$addr" | xargs)
|
||||
if [[ "$TRIMMED" == *"/"* ]]; then
|
||||
# Subnet notation: convert 100.78.182.0/24 -> 100.78.182.*
|
||||
SUBNET_PREFIX=$(echo "$TRIMMED" | cut -d'/' -f1)
|
||||
SUBNET_PATTERN="${SUBNET_PREFIX%.*}.*"
|
||||
ALLOWED_DEV_ORIGINS="$ALLOWED_DEV_ORIGINS,$SUBNET_PATTERN"
|
||||
else
|
||||
# Individual IP: add wildcard for nearby IPs (e.g., 192.168.1.100 -> 192.168.1.*)
|
||||
IP_PATTERN="${TRIMMED%.*}.*"
|
||||
ALLOWED_DEV_ORIGINS="$ALLOWED_DEV_ORIGINS,$IP_PATTERN"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
export ALLOWED_DEV_ORIGINS
|
||||
|
||||
# 5. Start Frontend (Next.js)
|
||||
echo "💻 Starting Frontend on port $FRONTEND_PORT..."
|
||||
echo " Dev origins: $ALLOWED_DEV_ORIGINS"
|
||||
|
||||
# Check Node.js version
|
||||
NODE_VERSION=$(node -v | cut -d'v' -f2 | cut -d'.' -f1)
|
||||
if [ "$NODE_VERSION" -lt 20 ]; then
|
||||
echo "⚠️ WARNING: Node.js v20+ required, but found $(node -v)"
|
||||
echo " Please upgrade Node.js: https://nodejs.org/"
|
||||
fi
|
||||
|
||||
cd frontend
|
||||
echo "📦 Installing frontend dependencies..."
|
||||
npm install
|
||||
ALLOWED_DEV_ORIGINS="$ALLOWED_DEV_ORIGINS" npm run dev -- -p $FRONTEND_PORT &
|
||||
cd ..
|
||||
|
||||
# 6. Start Proxies (Crucial for Mobile/Tablet Camera & Sync)
|
||||
# Bind to SERVER_IP (not 0.0.0.0) so VPN/remote clients can reach the server
|
||||
PROXY_HOSTNAME=${SERVER_IP:-0.0.0.0}
|
||||
echo "🔐 Starting SSL proxies on $PROXY_HOSTNAME..."
|
||||
npx local-ssl-proxy --source $BACKEND_SSL_PORT --target $BACKEND_PORT --hostname $PROXY_HOSTNAME > /dev/null 2>&1 &
|
||||
npx local-ssl-proxy --source $FRONTEND_SSL_PORT --target $FRONTEND_PORT --hostname $PROXY_HOSTNAME > /dev/null 2>&1 &
|
||||
|
||||
# 7. Signal handlers for clean shutdown
|
||||
cleanup() {
|
||||
echo ""
|
||||
echo "🛑 Shutting down services..."
|
||||
pkill -P $$ > /dev/null 2>&1
|
||||
kill $(jobs -p) 2>/dev/null || true
|
||||
pkill -f "uvicorn" 2>/dev/null || true
|
||||
pkill -f "next-server" 2>/dev/null || true
|
||||
pkill -f "local-ssl-proxy" 2>/dev/null || true
|
||||
echo "✅ Cleanup complete"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Trap Ctrl-C (SIGINT) and other termination signals
|
||||
trap cleanup SIGINT SIGTERM EXIT
|
||||
|
||||
# 8. Print Unified Access Banner
|
||||
|
||||
# Colors
|
||||
# Color output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
BOLD='\033[1m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
echo ""
|
||||
echo -e "${GREEN}=======================================================${NC}"
|
||||
echo -e "${GREEN}${BOLD} 🚀 TFM aInventory UNIFIED ACCESS${NC}"
|
||||
echo -e "${GREEN}=======================================================${NC}"
|
||||
echo ""
|
||||
echo -e " USE THIS URL ON BOTH DESKTOP & MOBILE:"
|
||||
echo -e " 👉 ${GREEN}${BOLD}https://$LOCAL_IP:$FRONTEND_SSL_PORT${NC}"
|
||||
echo -e " (Or ${GREEN}https://localhost:$FRONTEND_SSL_PORT${NC} on this Mac)"
|
||||
log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
|
||||
log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
|
||||
log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; }
|
||||
log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
|
||||
|
||||
if [ ! -z "$EXTRA_ALLOWED_ORIGINS" ]; then
|
||||
echo -e " EXTERNAL/VPN ACCESS points:"
|
||||
IFS=',' read -ra ADDR <<< "$EXTRA_ALLOWED_ORIGINS"
|
||||
for exp in "${ADDR[@]}"; do
|
||||
# Trim spaces and print
|
||||
TRIMMED=$(echo $exp | xargs)
|
||||
echo -e " 👉 ${GREEN}${BOLD}https://$TRIMMED:$FRONTEND_SSL_PORT${NC}"
|
||||
done
|
||||
fi
|
||||
echo ""
|
||||
echo -e " ${YELLOW}${BOLD}NOTE:${NC} If you see a 'Not Private' warning,"
|
||||
echo -e " Click 'Advanced' -> 'Proceed' to continue."
|
||||
echo -e "${GREEN}=======================================================${NC}"
|
||||
echo "Keep this window open while working. Press Ctrl-C to stop."
|
||||
echo -e "${GREEN}=======================================================${NC}"
|
||||
log_info "=== TFM aInventory Standalone Server Start ==="
|
||||
|
||||
# Wait for background processes
|
||||
# ============================================================================
|
||||
# STEP 1: Pre-flight checks
|
||||
# ============================================================================
|
||||
log_info "Step 1/7: Running pre-flight checks..."
|
||||
|
||||
# Check Python version
|
||||
command -v python3 &> /dev/null || log_error "Python 3 not installed"
|
||||
PYTHON_VERSION=$(python3 --version 2>&1 | awk '{print $2}')
|
||||
log_success " ✓ Python $PYTHON_VERSION found"
|
||||
|
||||
# Check Node.js version
|
||||
command -v node &> /dev/null || log_error "Node.js not installed"
|
||||
NODE_VERSION=$(node --version)
|
||||
log_success " ✓ Node.js $NODE_VERSION found"
|
||||
|
||||
# Check required files
|
||||
[[ -f "inventory.env" ]] || log_error "inventory.env not found. Copy from inventory.env.template"
|
||||
log_success " ✓ inventory.env found"
|
||||
|
||||
[[ -d "backend" ]] || log_error "backend/ directory not found"
|
||||
log_success " ✓ backend/ directory found"
|
||||
|
||||
[[ -d "frontend" ]] || log_error "frontend/ directory not found"
|
||||
log_success " ✓ frontend/ directory found"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 2: Load environment
|
||||
# ============================================================================
|
||||
log_info "Step 2/7: Loading environment configuration..."
|
||||
|
||||
source inventory.env
|
||||
|
||||
BACKEND_PORT=${BACKEND_PORT:-8000}
|
||||
FRONTEND_PORT=${FRONTEND_PORT:-3000}
|
||||
DATA_DIR=${DATA_DIR:-./data}
|
||||
LOGS_DIR=${LOGS_DIR:-./logs}
|
||||
LOG_LEVEL=${LOG_LEVEL:-INFO}
|
||||
|
||||
log_success " ✓ Environment loaded (Backend: $BACKEND_PORT, Frontend: $FRONTEND_PORT)"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 3: Prepare directories
|
||||
# ============================================================================
|
||||
log_info "Step 3/7: Preparing data and log directories..."
|
||||
|
||||
mkdir -p "$DATA_DIR" "$LOGS_DIR"
|
||||
mkdir -p "$DATA_DIR/temp"
|
||||
chmod -R 755 "$DATA_DIR" "$LOGS_DIR"
|
||||
|
||||
log_success " ✓ Directories ready"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 4: Check port availability
|
||||
# ============================================================================
|
||||
log_info "Step 4/7: Checking port availability..."
|
||||
|
||||
for port in $BACKEND_PORT $FRONTEND_PORT; do
|
||||
if command -v netstat &> /dev/null; then
|
||||
if netstat -tuln 2>/dev/null | grep -q ":$port "; then
|
||||
log_error "Port $port is already in use. Stop the application using it or change the port."
|
||||
fi
|
||||
else
|
||||
log_warn " netstat not available; skipping port check"
|
||||
fi
|
||||
done
|
||||
|
||||
log_success " ✓ Required ports are available"
|
||||
|
||||
# ============================================================================
|
||||
# STEP 5: Start backend server
|
||||
# ============================================================================
|
||||
log_info "Step 5/7: Starting backend API server..."
|
||||
|
||||
# Create backend log file
|
||||
BACKEND_LOG="$LOGS_DIR/backend.log"
|
||||
touch "$BACKEND_LOG"
|
||||
|
||||
# Start backend in background
|
||||
cd backend || log_error "Failed to enter backend directory"
|
||||
|
||||
if [[ ! -f "requirements.txt" ]]; then
|
||||
log_error "backend/requirements.txt not found"
|
||||
fi
|
||||
|
||||
# Install dependencies if needed
|
||||
if ! python3 -c "import fastapi" &> /dev/null; then
|
||||
log_info " Installing Python dependencies..."
|
||||
pip install -q -r requirements.txt || log_error "Failed to install backend dependencies"
|
||||
fi
|
||||
|
||||
# Initialize database if needed
|
||||
if [[ ! -f "../$DATA_DIR/inventory.db" ]]; then
|
||||
log_info " Initializing database..."
|
||||
python3 -c "from db_manager import init_db; init_db()" || true
|
||||
fi
|
||||
|
||||
# Start uvicorn in background
|
||||
log_info " Starting uvicorn server..."
|
||||
python3 -m uvicorn main:app --host 0.0.0.0 --port "$BACKEND_PORT" --log-level "${LOG_LEVEL,,}" >> "$BACKEND_LOG" 2>&1 &
|
||||
BACKEND_PID=$!
|
||||
|
||||
log_success " ✓ Backend started (PID: $BACKEND_PID, Port: $BACKEND_PORT)"
|
||||
|
||||
cd - > /dev/null || true
|
||||
|
||||
# ============================================================================
|
||||
# STEP 6: Start frontend server
|
||||
# ============================================================================
|
||||
log_info "Step 6/7: Starting frontend web server..."
|
||||
|
||||
FRONTEND_LOG="$LOGS_DIR/frontend.log"
|
||||
touch "$FRONTEND_LOG"
|
||||
|
||||
cd frontend || log_error "Failed to enter frontend directory"
|
||||
|
||||
if [[ ! -f "package.json" ]]; then
|
||||
log_error "frontend/package.json not found"
|
||||
fi
|
||||
|
||||
# Install dependencies if needed
|
||||
if [[ ! -d "node_modules" ]]; then
|
||||
log_info " Installing npm dependencies..."
|
||||
npm ci --silent || log_error "Failed to install frontend dependencies"
|
||||
fi
|
||||
|
||||
# Build Next.js if not already built
|
||||
if [[ ! -d ".next" ]]; then
|
||||
log_info " Building Next.js application..."
|
||||
npm run build || log_error "Failed to build frontend"
|
||||
fi
|
||||
|
||||
# Start Next.js in production mode
|
||||
log_info " Starting Next.js server..."
|
||||
PORT="$FRONTEND_PORT" node .next/standalone/server.js >> "$FRONTEND_LOG" 2>&1 &
|
||||
FRONTEND_PID=$!
|
||||
|
||||
log_success " ✓ Frontend started (PID: $FRONTEND_PID, Port: $FRONTEND_PORT)"
|
||||
|
||||
cd - > /dev/null || true
|
||||
|
||||
# ============================================================================
|
||||
# STEP 7: Verify services
|
||||
# ============================================================================
|
||||
log_info "Step 7/7: Verifying services..."
|
||||
|
||||
# Wait a few seconds for services to start
|
||||
sleep 3
|
||||
|
||||
# Check backend
|
||||
if curl -sf "http://localhost:${BACKEND_PORT}/health" &> /dev/null; then
|
||||
log_success " ✓ Backend API responding at http://localhost:${BACKEND_PORT}/health"
|
||||
else
|
||||
log_warn " Backend not responding yet; it may still be initializing"
|
||||
fi
|
||||
|
||||
# Check frontend
|
||||
if curl -sf "http://localhost:${FRONTEND_PORT}/" &> /dev/null; then
|
||||
log_success " ✓ Frontend responding at http://localhost:${FRONTEND_PORT}"
|
||||
else
|
||||
log_warn " Frontend not responding yet; it may still be initializing"
|
||||
fi
|
||||
|
||||
# ============================================================================
|
||||
# Summary and instructions
|
||||
# ============================================================================
|
||||
echo ""
|
||||
echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}"
|
||||
echo -e "${GREEN}║${NC} Standalone servers started successfully! ${GREEN}║${NC}"
|
||||
echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}"
|
||||
echo ""
|
||||
echo "Access points:"
|
||||
echo " Frontend: http://localhost:${FRONTEND_PORT}"
|
||||
echo " Backend: http://localhost:${BACKEND_PORT}"
|
||||
echo " API Docs: http://localhost:${BACKEND_PORT}/docs"
|
||||
echo ""
|
||||
echo "Process IDs:"
|
||||
echo " Backend: $BACKEND_PID"
|
||||
echo " Frontend: $FRONTEND_PID"
|
||||
echo ""
|
||||
echo "Log files:"
|
||||
echo " Backend: $BACKEND_LOG"
|
||||
echo " Frontend: $FRONTEND_LOG"
|
||||
echo ""
|
||||
echo "Monitoring:"
|
||||
echo " Backend: tail -f $BACKEND_LOG"
|
||||
echo " Frontend: tail -f $FRONTEND_LOG"
|
||||
echo " All logs: tail -f $LOGS_DIR/*.log"
|
||||
echo ""
|
||||
echo "To stop servers:"
|
||||
echo " kill $BACKEND_PID $FRONTEND_PID"
|
||||
echo ""
|
||||
log_success "Servers are running!"
|
||||
|
||||
# Keep script running and handle signals
|
||||
trap "log_info 'Received interrupt signal'; kill $BACKEND_PID $FRONTEND_PID 2>/dev/null || true; exit 0" SIGINT SIGTERM
|
||||
|
||||
# Wait for both processes
|
||||
wait
|
||||
|
||||
Reference in New Issue
Block a user