diff --git a/.gitignore b/.gitignore index 50641428..f831273e 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,7 @@ backend/config/ldap_config.json !backend/config/ldap_config.json.example # ── Environment files (secrets) ────────────────────────────── +# [D-04] inventory.env deprecated — see config/ folder instead .env .env.* inventory.env @@ -52,6 +53,15 @@ backend/.env.* docker-compose.override.yml .env.docker +# ── Configuration consolidation (Phase 7) ──────────────────── +# Ignore actual YAML config files which contain real values/secrets +config/*.yaml +# But ensure example/schema files are ALWAYS tracked +!config/*.yaml.example +# Specifically ignore secrets.yaml +config/secrets.yaml +!config/secrets.yaml.example + # ── Application logs ───────────────────────────────────────── # (also covered by /logs/* above, these catch any other locations) frontend/logs/ diff --git a/.planning/phases/07-config-consolidation/07-01-PLAN.md b/.planning/phases/07-config-consolidation/07-01-PLAN.md new file mode 100644 index 00000000..de415a27 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-01-PLAN.md @@ -0,0 +1,395 @@ +--- +phase: 07-config-consolidation +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - config/backend.yaml + - config/backend.yaml.example + - config/frontend.yaml + - config/frontend.yaml.example + - config/network.yaml + - config/network.yaml.example + - config/docker.yaml + - config/docker.yaml.example + - config/secrets.yaml.example + - config/README.md + - .gitignore +autonomous: true +requirements: + - PHASE-7-CONFIG-STRUCT + - PHASE-7-YAML-FORMAT + - PHASE-7-SECRETS-MGMT + - PHASE-7-DOCUMENTATION +user_setup: [] + +must_haves: + truths: + - "config/ folder exists with all YAML files and examples" + - "YAML structure matches backend, frontend, network, docker, and secrets domains" + - "All *.yaml.example files committed to git showing complete schema" + - "config/secrets.yaml is git-ignored, with example provided" + - ".gitignore correctly tracks examples, ignores actual secrets" + - "config/README.md documents every YAML file with required variables and setup instructions" + artifacts: + - path: "config/backend.yaml.example" + provides: "Backend configuration schema (database, AI keys, auth, logging)" + min_lines: 40 + - path: "config/frontend.yaml.example" + provides: "Frontend configuration schema (API endpoints, feature flags, PWA)" + min_lines: 25 + - path: "config/network.yaml.example" + provides: "Network configuration schema (ports, SSL, CORS, server IPs)" + min_lines: 20 + - path: "config/docker.yaml.example" + provides: "Docker-specific overrides (container resources, mount paths)" + min_lines: 20 + - path: "config/secrets.yaml.example" + provides: "Secrets template (JWT, API keys, passwords)" + min_lines: 15 + - path: "config/README.md" + provides: "Comprehensive documentation of all config files and setup" + min_lines: 100 + key_links: + - from: "config/backend.yaml" + to: "backend/config_loader.py" + via: "YAML parsing in config loader" + pattern: "load.*backend\\.yaml" + - from: "config/secrets.yaml.example" + to: ".gitignore" + via: "git ignore rule" + pattern: "config/\\*\\.yaml.*!.*\\.example" + - from: "config/" + to: "docker-compose.yml" + via: "volume mount in service definition" + pattern: "\\./config:/app/config" +--- + + +Create the config/ folder structure with YAML files for backend, frontend, network, docker, and secrets configurations. Establish the foundation for centralized configuration management with clear schema documentation. + +Purpose: Consolidate scattered configuration (currently in inventory.env) into a structured, domain-specific YAML format (per D-01, D-02, D-03). + +Output: config/ folder with 4 YAML config files + examples, secrets template, comprehensive README, and .gitignore updates. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@DEPLOYMENT.md +@inventory.env +@inventory.env.example +@docker-compose.yml + + + + + + Task 1: Create config/ folder structure and YAML example files + + config/backend.yaml.example + config/frontend.yaml.example + config/network.yaml.example + config/docker.yaml.example + config/secrets.yaml.example + + + - inventory.env (current config source) + - inventory.env.example (existing schema) + - PROJECT_ARCHITECTURE.md (tech stack, components) + - DEPLOYMENT.md (current config categories) + - docker-compose.yml (container environment vars) + + + Create config/ folder in project root with 5 YAML example files (per D-01, D-03): + + **config/backend.yaml.example** — Backend configuration template covering: + - Database: sqlite_path, log_retention_days, wal_mode (from current code patterns) + - AI: primary_ai_provider (gemini|claude), gemini_api_key, claude_api_key, fallback_provider + - Auth: jwt_secret_key, ldap_server (optional), ldap_base_dn, password_cache_path + - Logging: log_level (DEBUG|INFO|WARNING|ERROR), log_rotation_size_mb, log_rotation_count + - Application: data_dir, logs_dir, cors_origins + - Feature flags: ai_extraction_enabled, offline_sync_enabled, audit_logging_enabled + + **config/frontend.yaml.example** — Frontend configuration template covering: + - API: backend_url (e.g., http://localhost:8916), timeout_ms + - Feature flags: service_worker_enabled, offline_enabled, ai_extraction_ui_enabled + - PWA: app_name, short_name, start_url, display_mode + - Feature toggles: enable_qr_scanner, enable_barcode_scanner, enable_batch_import + + **config/network.yaml.example** — Network/deployment configuration template covering: + - Ports: backend_port (8916), frontend_port (8917), backend_ssl_port (8918), frontend_ssl_port (8919) + - SSL: ssl_enabled (true|false), certificate_path, key_path (or auto-generated by Caddy) + - Proxy: caddy_log_level, proxy_read_timeout_s, max_request_size_mb + - CORS: allowed_origins (comma-separated), allowed_methods, allowed_headers + + **config/docker.yaml.example** — Docker-specific overrides covering: + - Images: backend_image, frontend_image, proxy_image (with tags) + - Resources: backend_cpu_limit, backend_memory_limit, frontend_cpu_limit, frontend_memory_limit + - Volumes: data_volume_driver, logs_volume_driver, use_named_volumes (true|false) + - Networks: network_name, network_driver (bridge|overlay) + + **config/secrets.yaml.example** — Secrets template (git-ignored) covering: + - JWT_SECRET_KEY: "CHANGE_ME_IN_PRODUCTION" (minimum 32 chars) + - GEMINI_API_KEY: "your-api-key" + - CLAUDE_API_KEY: "your-api-key" + - DATABASE_PASSWORD: "db-password" (if using external DB) + - LDAP_PASSWORD: "ldap-password" (if using LDAP) + - CORS_ORIGIN_PASSWORD: (if CORS origins require auth) + + All examples should have comments explaining: + - What each variable controls + - Default values + - Valid value ranges + - Where to obtain secrets (e.g., "Generate with: openssl rand -hex 32") + - Whether the variable is required or optional + + + - `test -d config` (folder exists) + - `test -f config/backend.yaml.example && grep -q "primary_ai_provider" config/backend.yaml.example` (backend schema present) + - `test -f config/frontend.yaml.example && grep -q "backend_url" config/frontend.yaml.example` (frontend schema present) + - `test -f config/network.yaml.example && grep -q "backend_port" config/network.yaml.example` (network schema present) + - `test -f config/docker.yaml.example && grep -q "backend_cpu_limit" config/docker.yaml.example` (docker schema present) + - `test -f config/secrets.yaml.example && grep -q "JWT_SECRET_KEY" config/secrets.yaml.example` (secrets template present) + - All 5 files are readable and valid YAML syntax: `python3 -c "import yaml; [yaml.safe_load(open(f)) for f in ['config/backend.yaml.example', 'config/frontend.yaml.example', 'config/network.yaml.example', 'config/docker.yaml.example', 'config/secrets.yaml.example']]"` + + + config/ folder created with 5 YAML example files, all valid YAML syntax, comprehensive comments documenting schema, required/optional status, and secret generation instructions. + + + + + Task 2: Create actual config files from examples and establish .gitignore rules + + config/backend.yaml + config/frontend.yaml + config/network.yaml + config/docker.yaml + .gitignore + + + - config/backend.yaml.example (just created) + - config/frontend.yaml.example (just created) + - config/network.yaml.example (just created) + - config/docker.yaml.example (just created) + - inventory.env (current actual values to migrate) + - .gitignore (current ignore rules) + + + Create actual (non-example) YAML config files by copying examples and filling in values from inventory.env (per D-03 pattern): + + **config/backend.yaml** — Copy from backend.yaml.example and fill in values from inventory.env: + - primary_ai_provider: (from PRIMARY_AI_PROVIDER in inventory.env, default: gemini) + - gemini_api_key: (from GEMINI_API_KEY if present) + - claude_api_key: (from CLAUDE_API_KEY if present) + - ldap_server: (from LDAP_SERVER if present, or empty) + - log_level: (from LOG_LEVEL if present, default: INFO) + - data_dir: ./data + - logs_dir: ./logs + - cors_origins: (from CORS_ORIGINS if present, or default: http://localhost:8917) + + **config/frontend.yaml** — Create from frontend.yaml.example: + - backend_url: (from BACKEND_URL or derived from BACKEND_PORT, default: http://localhost:8916) + - timeout_ms: 30000 + - service_worker_enabled: true + - offline_enabled: true + - ai_extraction_ui_enabled: true + + **config/network.yaml** — Create from network.yaml.example: + - backend_port: (from BACKEND_PORT in inventory.env, default: 8916) + - frontend_port: (from FRONTEND_PORT in inventory.env, default: 8917) + - backend_ssl_port: (from BACKEND_SSL_PORT if present, default: 8918) + - frontend_ssl_port: (from FRONTEND_SSL_PORT if present, default: 8919) + - ssl_enabled: true + - allowed_origins: (from CORS_ORIGINS if present) + + **config/docker.yaml** — Create from docker.yaml.example: + - backend_cpu_limit: "1.0" + - backend_memory_limit: "1G" + - frontend_cpu_limit: "0.5" + - frontend_memory_limit: "512M" + - use_named_volumes: true + + **Update .gitignore** (per D-08): + Add rules to IGNORE actual config files (secrets protection) but TRACK examples: + ``` + # Config files — ignore all .yaml except examples + config/*.yaml + !config/*.yaml.example + config/secrets.yaml + !config/secrets.yaml.example + ``` + + Do NOT delete old inventory.env yet (backward compatibility until Phase 7 complete, per D-04 deprecation timeline). + + + - `test -f config/backend.yaml && grep -q "primary_ai_provider" config/backend.yaml` (backend.yaml exists and has content) + - `test -f config/frontend.yaml && grep -q "backend_url" config/frontend.yaml` (frontend.yaml exists) + - `test -f config/network.yaml && grep -q "backend_port" config/network.yaml` (network.yaml exists) + - `test -f config/docker.yaml && grep -q "backend_cpu_limit" config/docker.yaml` (docker.yaml exists) + - All 4 files are valid YAML: `python3 -c "import yaml; [yaml.safe_load(open(f)) for f in ['config/backend.yaml', 'config/frontend.yaml', 'config/network.yaml', 'config/docker.yaml']]"` + - .gitignore contains rules: `grep -q "config/\*\.yaml" .gitignore && grep -q "!config/\*\.yaml\.example" .gitignore` + - config/secrets.yaml does NOT exist (will be created manually by developers from example) + + + Actual config files created with production values migrated from inventory.env, .gitignore updated to track examples and ignore actual config files (including secrets). + + + + + Task 3: Create comprehensive config/README.md documentation + + config/README.md + + + - config/backend.yaml.example + - config/frontend.yaml.example + - config/network.yaml.example + - config/docker.yaml.example + - config/secrets.yaml.example + - DEPLOYMENT.md (current deployment docs) + + + Create config/README.md with comprehensive documentation (per D-08) covering: + + **Section 1: Overview** + - Explain that config/ is the single source of truth for application configuration + - Mention load order: system environment variables > config/*.yaml > defaults in code + - Note that secrets are separate (git-ignored) + + **Section 2: Quick Start** + - Copy all *.example files to remove .example suffix + - Fill in required values (especially JWT_SECRET_KEY, API keys) + - Create secrets.yaml from secrets.yaml.example with actual values + + **Section 3: backend.yaml** + - Explain each variable: purpose, valid values, defaults, required/optional + - List how to obtain each value (e.g., "Generate JWT key with: openssl rand -hex 32") + - Show example values + - List environment variable override names (e.g., BACKEND_PRIMARY_AI_PROVIDER) + + **Section 4: frontend.yaml** + - Explain each variable: purpose, valid values, defaults + - List required vs optional flags + - Show how feature flags affect UI behavior + - List environment variable override names + + **Section 5: network.yaml** + - Explain port assignments and SSL settings + - Show how CORS origins work + - List default values and adjustment guidance + - List environment variable override names + + **Section 6: docker.yaml** + - Explain container resource limits (CPU, memory) + - Show how to adjust for different hardware + - Explain volume management + - List environment variable override names + + **Section 7: secrets.yaml** + - Explain git-ignore protection + - List all required secrets with generation/obtainment instructions + - Warn about security implications + - Show correct file permissions (600) + + **Section 8: Environment Variable Override** + - Explain how system environment variables take precedence over YAML + - Show naming convention (e.g., BACKEND_PRIMARY_AI_PROVIDER -> backend.yaml:primary_ai_provider) + - Useful for Docker deployments where secrets come from docker run -e + + **Section 9: Troubleshooting** + - Common issues: missing secrets, invalid YAML syntax, missing required values + - How to validate YAML syntax + - How to debug which config source is being used + + Include helpful tables showing all variables, their purposes, defaults, and how to override them. + + + - `test -f config/README.md && wc -l config/README.md | awk '{print $1}' | awk '$1 >= 100 {print "pass"}'` (README has at least 100 lines) + - `grep -q "Quick Start" config/README.md && grep -q "backend.yaml" config/README.md && grep -q "secrets.yaml" config/README.md` (README covers all files) + - `grep -q "environment variable" config/README.md` (override mechanism documented) + - `grep -q "JWT_SECRET_KEY" config/README.md && grep -q "openssl rand" config/README.md` (secret generation instructions present) + + + config/README.md created with comprehensive documentation of all YAML files, required variables, generation instructions, environment variable overrides, and troubleshooting guide. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| Filesystem → Application | Config files loaded from disk must not be tampered with | +| Environment → Application | System environment variables override YAML (untrusted if exposed) | +| Git repository → Deployment | .gitignore must prevent accidental commit of secrets | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-01 | Tampering | config/*.yaml | mitigate | File permissions enforced via git (0644 for examples, 0600 for secrets). Read-only volume mounts in Docker per docker-compose.yml line 19 `:ro` flag. | +| T-07-02 | Information Disclosure | config/secrets.yaml | mitigate | .gitignore rule prevents accidental commit. File permissions enforced (chmod 600). config/README.md warns developers about security. Example file included to guide setup. | +| T-07-03 | Denial of Service | config/*.yaml parsing | mitigate | PyYAML used with safe_load() only (no arbitrary code execution). Backend config_loader.py validates syntax before loading. Invalid YAML causes logged error + default fallback per D-06 load order. | +| T-07-04 | Elevation of Privilege | JWT_SECRET_KEY exposure | accept | JWT secret hardcoded in docker-compose.yml example (line 25) warns with comment. Developers must provide production value. Risk low for development environments. | + + + + +**Phase 7, Plan 1 Verification Checklist:** + +1. **Config Folder Structure** + - [ ] `config/` folder exists in project root + - [ ] 5 example files present: backend.yaml.example, frontend.yaml.example, network.yaml.example, docker.yaml.example, secrets.yaml.example + - [ ] All examples contain valid YAML syntax (parseable by python3 -m yaml) + - [ ] All examples have comprehensive comments explaining each variable + +2. **Actual Config Files** + - [ ] 4 actual config files exist: backend.yaml, frontend.yaml, network.yaml, docker.yaml + - [ ] All actual files contain valid YAML syntax + - [ ] Values migrated from inventory.env are present and reasonable + - [ ] secrets.yaml does NOT exist (will be created manually) + +3. **Git Integration** + - [ ] .gitignore updated with rules: `config/*.yaml`, `!config/*.yaml.example`, `config/secrets.yaml`, `!config/secrets.yaml.example` + - [ ] Examples are tracked: `git status config/*.example` shows "new file" or no changes + - [ ] Actual configs are ignored: `git check-ignore config/backend.yaml` returns success + - [ ] Secrets example is tracked but secrets themselves are ignored + +4. **Documentation** + - [ ] config/README.md exists with 100+ lines + - [ ] README covers all 5 YAML files (backend, frontend, network, docker, secrets) + - [ ] README documents environment variable override mechanism + - [ ] README includes secret generation/obtainment instructions + - [ ] README has troubleshooting section + +5. **Backward Compatibility (D-04)** + - [ ] inventory.env still exists (will be fully deprecated after backend refactor in Plan 2) + - [ ] No code changes yet (Config loading still uses inventory.env, Phase 7 Plan 2 updates backend) + + + +- config/ folder created with 5 YAML example files defining complete schema +- 4 actual YAML config files created with values migrated from inventory.env +- config/secrets.yaml.example provides template (actual secrets.yaml created manually by developers) +- .gitignore updated to track examples, ignore actual config files and secrets +- config/README.md provides comprehensive documentation and setup instructions +- All YAML files are syntactically valid and parseable +- No backend code changes yet (backward compatibility maintained per D-04) +- Foundation ready for backend refactoring in Plan 2 + + + +After completion, create `.planning/phases/07-config-consolidation/07-01-SUMMARY.md` + diff --git a/.planning/phases/07-config-consolidation/07-01-SUMMARY.md b/.planning/phases/07-config-consolidation/07-01-SUMMARY.md new file mode 100644 index 00000000..24aa748e --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-01-SUMMARY.md @@ -0,0 +1,67 @@ +--- +phase: 07-config-consolidation +plan: 01 +subsystem: configuration +tags: [yaml, config, secrets, documentation] +dependency_graph: + requires: [PHASE-6-STABILITY] + provides: [PHASE-7-CONFIG-STRUCT, PHASE-7-YAML-FORMAT] + affects: [backend, frontend, deployment] +tech_stack: + added: [PyYAML] + patterns: [domain-driven-yaml, environment-overrides] +key_files: + created: + - config/backend.yaml.example + - config/frontend.yaml.example + - config/network.yaml.example + - config/docker.yaml.example + - config/secrets.yaml.example + - config/backend.yaml + - config/frontend.yaml + - config/network.yaml + - config/docker.yaml + - config/README.md + modified: + - .gitignore +decisions: + - D-01: Centralize configuration into config/ folder + - D-02: Use domain-specific YAML files (backend, frontend, network, docker) + - D-03: Separate secrets into git-ignored secrets.yaml and tracked examples + - D-08: Use .gitignore to protect actual values while tracking schemas +metrics: + duration: 15m + completed_date: "2024-04-23" +--- + +# Phase 07 Plan 01: Config Folder Structure and YAML Schemas Summary + +## Substantive One-liner +Established a structured configuration framework with 10 YAML files (5 schemas, 4 actual configs, 1 secrets template) and comprehensive documentation in `config/`. + +## Progress Summary +All tasks in the plan were completed successfully. The project now has a dedicated `config/` directory with domain-specific YAML files for backend, frontend, network, and docker orchestration. Each configuration file has a corresponding `.example` file that defines its schema and is tracked by Git, while actual values are protected via `.gitignore`. A 155-line `README.md` provides complete documentation for the new system. + +### Key Achievements +- **Config Folder Structure:** Created `config/` in project root, housing all configuration assets. +- **YAML Schemas:** Created 5 `.yaml.example` files (backend, frontend, network, docker, secrets) with comprehensive comments documenting every variable, its default, and its environment override. +- **Data Migration:** Migrated existing values from `inventory.env` into 4 actual YAML files (`backend.yaml`, `frontend.yaml`, `network.yaml`, `docker.yaml`) for immediate developer use. +- **Git Protection:** Updated `.gitignore` with strict rules to ignore actual YAML files while ensuring schemas remain tracked. +- **Documentation:** Created a massive 150+ line `README.md` in the `config/` folder, covering setup, load order, security practices, and troubleshooting. + +## Deviations from Plan +None - plan executed exactly as written. + +## Known Stubs +None. All files are complete and use valid YAML syntax. + +## Threat Flags +None. All security-relevant practices (ignoring secrets, protecting actual values) were implemented. + +## Self-Check: PASSED +- [x] `config/` folder exists +- [x] 10 files in `config/` (5 examples, 4 actuals, 1 README) +- [x] All YAML files parseable +- [x] .gitignore rules verified +- [x] `inventory.env` preserved for backward compatibility +- [x] Commits made for each task (Note: Task 2 commit only includes `.gitignore` as the actual YAML files are correctly ignored) diff --git a/.planning/phases/07-config-consolidation/07-02-PLAN.md b/.planning/phases/07-config-consolidation/07-02-PLAN.md new file mode 100644 index 00000000..bc658250 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-02-PLAN.md @@ -0,0 +1,421 @@ +--- +phase: 07-config-consolidation +plan: 02 +type: execute +wave: 2 +depends_on: + - 07-01 +files_modified: + - backend/config_loader.py + - backend/config_manager.py + - backend/main.py + - backend/entrypoint.sh +autonomous: true +requirements: + - PHASE-7-BACKEND-YAML + - PHASE-7-ENV-OVERRIDE + - PHASE-7-NO-FALLBACK +user_setup: [] + +must_haves: + truths: + - "Backend loads from config/backend.yaml + config/secrets.yaml (YAML parsing with PyYAML)" + - "System environment variables override YAML values (per D-06 load order)" + - "NO fallback to inventory.env—deprecation complete after Phase 7 (per D-04)" + - "Config loading logs which source is being used for debugging" + - "Backend starts successfully with new config structure and passes health checks" + artifacts: + - path: "backend/config_loader.py" + provides: "YAML config parsing with env var override and load order enforcement" + exports: ["load_config()", "get_config()", "validate_config()"] + min_lines: 80 + - path: "backend/config_manager.py" + provides: "Config management and updates with YAML support" + min_lines: 50 + - path: "backend/main.py" + provides: "Updated main() to use new config_loader (no inventory.env references)" + pattern: "from backend.config_loader import load_config" + - path: "backend/entrypoint.sh" + provides: "Updated Docker entrypoint sourcing YAML config paths" + pattern: "config/backend.yaml" + key_links: + - from: "backend/config_loader.py" + to: "config/backend.yaml" + via: "PyYAML parsing" + pattern: "yaml\\.safe_load.*backend\\.yaml" + - from: "backend/config_loader.py" + to: "config/secrets.yaml" + via: "PyYAML parsing with file existence check" + pattern: "yaml\\.safe_load.*secrets\\.yaml" + - from: "backend/main.py" + to: "backend/config_loader.py" + via: "import and call load_config()" + pattern: "from backend.config_loader import load_config" +--- + + +Refactor backend configuration loading from .env to YAML (backend.yaml + secrets.yaml) with system environment variable override support. Remove all inventory.env fallback paths and ensure deprecation is complete. + +Purpose: Implement D-06 load order (env vars > YAML > defaults) with proper logging and validation. + +Output: Updated config_loader.py with YAML parsing, config_manager.py with YAML support, main.py using new loader, and updated Docker entrypoint. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@backend/config_loader.py +@backend/config_manager.py +@backend/main.py +@backend/entrypoint.sh +@config/backend.yaml.example +@config/secrets.yaml.example + + + + + + Task 1: Refactor backend/config_loader.py for YAML parsing with env var override + + backend/config_loader.py + + + - backend/config_loader.py (current implementation using dotenv) + - config/backend.yaml.example (schema to parse) + - config/secrets.yaml.example (secrets schema) + - backend/main.py (to understand how config is used) + + + Refactor backend/config_loader.py to implement D-06 load order: system env vars > config/backend.yaml > config/secrets.yaml > defaults. + + **Required changes:** + + 1. **Replace dotenv with PyYAML:** Add `import yaml` and remove `from dotenv import load_dotenv` + + 2. **Implement load_config() function** that: + - Locates config/ folder (one level up from backend/) + - Attempts to load config/backend.yaml (required if exists) + - Attempts to load config/secrets.yaml (optional, file may not exist) + - Defines hard defaults for all variables (fallback if YAML missing) + - Merges in order: defaults <- YAML values <- environment variable overrides + - Returns a dict/object with all configuration + + 3. **Environment variable override pattern:** + - System env var takes precedence over YAML + - Naming convention: BACKEND_ or just + - Examples: + - BACKEND_PRIMARY_AI_PROVIDER -> backend.yaml:primary_ai_provider + - JWT_SECRET_KEY (from secrets.yaml or env) + - LOG_LEVEL -> backend.yaml:log_level + - All env vars checked with os.getenv() + + 4. **Load order example (pseudocode):** + ``` + defaults = {primary_ai_provider: "gemini", log_level: "INFO", ...} + backend_yaml = yaml.safe_load(open("config/backend.yaml")) if exists else {} + secrets_yaml = yaml.safe_load(open("config/secrets.yaml")) if exists else {} + config = merge(defaults, backend_yaml, secrets_yaml) + config = merge(config, env_var_overrides()) + return config + ``` + + 5. **Implement validate_config() function** that: + - Checks required variables are present (JWT_SECRET_KEY, primary_ai_provider, etc.) + - Validates enum values (primary_ai_provider must be gemini|claude|fallback) + - Validates log levels (DEBUG|INFO|WARNING|ERROR) + - Raises ConfigError if validation fails with descriptive message + + 6. **Implement get_config() function** that: + - Returns the loaded configuration dict + - Allows other modules to import: `from backend.config_loader import get_config` + + 7. **Logging:** + - Log which files were loaded: "Loaded backend.yaml from config/" + - Log env var overrides: "Override primary_ai_provider from environment: gemini" + - Log final validated config (without secrets): "Config validated: primary_ai_provider=gemini, log_level=INFO" + - Use log.info() and log.warning() (not print) + + 8. **Remove all inventory.env references:** + - Delete any checks for inventory_env_path + - Delete fallback to backend/.env + - NO fallback to old locations (per D-04 deprecation) + + 9. **Error handling:** + - If config/backend.yaml missing: raise ConfigError with instructions to copy from .example + - If secrets.yaml missing: log warning but continue (secrets can come from env vars) + - If required variables missing after all sources: raise ConfigError listing missing vars + + 10. **Function signature** (updated): + ```python + def load_config() -> dict: + """Load config from YAML files with env var overrides (D-06 load order).""" + + def get_config() -> dict: + """Get loaded config.""" + + def validate_config(config: dict) -> bool: + """Validate config has all required values.""" + ``` + + Keep the auto-run at module load: `load_config()` and `validate_config()` called on import. + + + - `grep -q "import yaml" backend/config_loader.py` (PyYAML imported) + - `grep -q "def load_config" backend/config_loader.py && grep -q "def get_config" backend/config_loader.py` (required functions exist) + - `grep -q "config/backend.yaml" backend/config_loader.py` (reads backend YAML) + - `grep -q "config/secrets.yaml" backend/config_loader.py` (reads secrets YAML) + - `grep -q "os.getenv" backend/config_loader.py` (env var overrides present) + - `grep -v "inventory.env" backend/config_loader.py | grep -q "inventory"` should return empty (no inventory.env references) + - `grep -q "validate_config" backend/config_loader.py` (validation function present) + - `grep -q "log.info\|log.warning" backend/config_loader.py` (logging present) + - File should be valid Python: `python3 -m py_compile backend/config_loader.py` + + + config_loader.py refactored to parse YAML files with env var override, remove inventory.env completely, implement D-06 load order with validation and logging. + + + + + Task 2: Update backend/config_manager.py for YAML config updates + + backend/config_manager.py + + + - backend/config_manager.py (current implementation) + - config/backend.yaml.example (schema) + - backend/config_loader.py (just updated) + + + Update backend/config_manager.py to support YAML config file updates (if runtime updates are needed): + + **If config_manager.py currently reads/writes .env files:** + + 1. **Replace dotenv with PyYAML:** + - Add `import yaml` + - Remove any dotenv usage + + 2. **Implement update_config() function** that: + - Takes key-value pairs to update + - Loads current config/backend.yaml + - Updates values in-memory + - Writes back to config/backend.yaml with safe_dump() + - NEVER writes to config/secrets.yaml (secrets are git-ignored for a reason) + - Logs what was updated + + 3. **Implement read_config() function** that: + - Reads config/backend.yaml and returns dict + - Uses yaml.safe_load() + + 4. **Handle errors gracefully:** + - If config/backend.yaml not found, raise error (it should exist from Plan 1) + - If YAML syntax error, log and return current in-memory config + - Preserve file comments if possible (or warn user they will be lost) + + 5. **Function signature** (updated): + ```python + def read_config() -> dict: + """Read backend.yaml and return current config.""" + + def update_config(updates: dict) -> dict: + """Update backend.yaml with new values and return updated config.""" + + def validate_config_file() -> bool: + """Validate backend.yaml syntax and required fields.""" + ``` + + **If config_manager.py is minimal/unused:** + - Add basic functions as above for future extensibility + - Add docstrings explaining YAML handling + - Import and use config_loader.load_config() as primary source + + + - `grep -q "import yaml" backend/config_manager.py` (PyYAML imported) + - `grep -q "def read_config\|def update_config\|def validate_config_file" backend/config_manager.py` (functions present) + - `grep -q "config/backend.yaml" backend/config_manager.py` (references YAML file) + - `grep -q "yaml.safe_load\|yaml.safe_dump" backend/config_manager.py` (YAML parsing present) + - File should be valid Python: `python3 -m py_compile backend/config_manager.py` + + + config_manager.py updated to support YAML config file updates with safe_load/safe_dump, no dotenv dependencies. + + + + + Task 3: Update backend/main.py to use new YAML config loader + + backend/main.py + + + - backend/main.py (current implementation) + - backend/config_loader.py (just updated) + + + Update backend/main.py to use the refactored config_loader: + + 1. **Update imports:** + - Replace any `from dotenv import load_dotenv` with `from backend.config_loader import load_config, get_config` + - Remove `load_dotenv()` calls + + 2. **Update main startup:** + - Call `load_config()` at app startup (or rely on module-level auto-run) + - Retrieve config with `get_config()` instead of `os.getenv()` + - Example: `config = get_config()` then `db_path = config['database']['sqlite_path']` + + 3. **Update environment variable access:** + - Replace `os.getenv("BACKEND_PORT")` with `config.get("backend_port")` + - Replace `os.getenv("JWT_SECRET_KEY")` with `config.get("jwt_secret_key")` + - All refs should come from config dict, not os.getenv() + + 4. **Remove inventory.env references:** + - Delete any checks for inventory.env existence + - Delete fallback logic to root-level config + - Ensure NO hardcoded "inventory.env" strings remain + + 5. **Logging:** + - Log at startup which config was loaded (already done by config_loader, but confirm) + - Example: "Backend initialized with config from config/backend.yaml" + + Note: This should be minimal changes if main.py already calls config_loader.load_config() at startup. + + + - `grep -q "from backend.config_loader import" backend/main.py` (imports from new loader) + - `grep -q "load_dotenv" backend/main.py` should return empty (no dotenv) + - `grep -q "inventory.env" backend/main.py` should return empty (no old config refs) + - File should be valid Python: `python3 -m py_compile backend/main.py` + - Check for os.getenv() calls and ensure they reference config dict instead: `grep "os.getenv" backend/main.py | head -5` (should be minimal or zero) + + + backend/main.py updated to import and use new YAML-based config_loader, remove all inventory.env and dotenv references. + + + + + Task 4: Update backend/entrypoint.sh for new config paths + + backend/entrypoint.sh + + + - backend/entrypoint.sh (current Docker entrypoint) + - docker-compose.yml (volumes mapping config/) + + + Update backend/entrypoint.sh to reference new config/ folder paths (per D-07 Docker support): + + 1. **If entrypoint sources config:** + - Remove any sourcing of inventory.env + - Add comment: "Config is loaded from /app/config/ (YAML format) per Phase 7" + - Ensure /app/config/ path is correct (mapped from host config/ via docker-compose.yml line 19) + + 2. **Update environment variable documentation:** + - Add comment: "Environment variables override YAML config (D-06 load order)" + - List key variables that can be overridden: JWT_SECRET_KEY, PRIMARY_AI_PROVIDER, etc. + - Example: `export JWT_SECRET_KEY="$(openssl rand -hex 32)"` (if not set) + + 3. **Ensure startup doesn't fail if config missing:** + - Add check: if config/backend.yaml not found, log error and instructions + - Python code will raise ConfigError, so entrypoint can remain simple + - Just ensure permissions are correct: `chmod 644 config/*.yaml` + + 4. **Update Dockerfile comment (if present):** + - Reference config/ volume mount + - Explain YAML loading approach + + + - `grep -q "config/backend.yaml\|config/secrets.yaml" backend/entrypoint.sh || echo "pass"` (references new config paths or doesn't source at all) + - `grep -q "inventory.env" backend/entrypoint.sh` should return empty (no old config) + - File should be valid bash: `bash -n backend/entrypoint.sh` + + + backend/entrypoint.sh updated to reference config/ paths and document environment variable override behavior. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| Filesystem → Backend | Backend reads from config/backend.yaml and config/secrets.yaml | +| Environment → Backend | System environment variables override config files (untrusted if exposed in logs) | +| Network → Backend | API receives JWT which is read from config (must be kept secret) | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-05 | Information Disclosure | Config logging | mitigate | Log config values WITHOUT secrets. config_loader.py logs final config but masks JWT_SECRET_KEY, API keys. Pattern: log keys but not values for sensitive fields. | +| T-07-06 | Denial of Service | Invalid YAML parsing | mitigate | yaml.safe_load() prevents code injection. ConfigError raised with clear message if required vars missing. validate_config() checks all required fields. | +| T-07-07 | Tampering | Environment variables | mitigate | Log env var overrides so operators know what was applied. Env vars documented in config/README.md. | +| T-07-08 | Elevation of Privilege | Backend startup | mitigate | ConfigError on missing JWT_SECRET_KEY prevents insecure defaults. Backend refuses to start without proper config. | + + + + +**Phase 7, Plan 2 Verification Checklist:** + +1. **config_loader.py Refactoring** + - [ ] PyYAML is imported, dotenv is removed + - [ ] load_config() loads config/backend.yaml + - [ ] load_config() loads config/secrets.yaml (optional) + - [ ] Environment variable overrides are applied correctly (D-06) + - [ ] validate_config() function checks required variables + - [ ] No inventory.env references remain + - [ ] Logging shows which config source was used + - [ ] File is valid Python syntax + +2. **config_manager.py Updates** + - [ ] PyYAML is imported + - [ ] read_config() and update_config() functions exist + - [ ] YAML file operations use safe_load/safe_dump + - [ ] No dotenv references + - [ ] File is valid Python syntax + +3. **main.py Updates** + - [ ] Imports from backend.config_loader + - [ ] Calls load_config() or relies on module-level auto-run + - [ ] Uses get_config() to retrieve configuration + - [ ] No dotenv or inventory.env references + - [ ] No direct os.getenv() calls for app config + - [ ] File is valid Python syntax + +4. **entrypoint.sh Updates** + - [ ] References config/ paths (not inventory.env) + - [ ] Documents environment variable override behavior + - [ ] Bash syntax is valid + +5. **Load Order Verification (D-06)** + - [ ] System env vars > config/backend.yaml > config/secrets.yaml > defaults + - [ ] Integration test: set BACKEND_LOG_LEVEL=DEBUG, verify backend uses DEBUG level + - [ ] Integration test: remove config/backend.yaml, verify defaults are used + - [ ] Integration test: set JWT_SECRET_KEY in env, verify it overrides YAML value + +6. **Deprecation Verification (D-04)** + - [ ] inventory.env no longer used by backend + - [ ] No fallback code remains + - [ ] Backend fails clearly (ConfigError) if required config missing (instead of silent defaults) + + + +- backend/config_loader.py refactored to use PyYAML with env var override support (D-06) +- D-06 load order implemented: env vars > YAML > defaults +- All inventory.env references removed from backend code (D-04) +- config_manager.py updated for YAML file operations +- backend/main.py uses new config loader +- backend/entrypoint.sh references config/ paths +- Configuration validation ensures required variables are present +- Logging shows which sources were used for debugging +- Backend starts successfully with new config structure + + + +After completion, create `.planning/phases/07-config-consolidation/07-02-SUMMARY.md` + diff --git a/.planning/phases/07-config-consolidation/07-02-SUMMARY.md b/.planning/phases/07-config-consolidation/07-02-SUMMARY.md new file mode 100644 index 00000000..25a9b024 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-02-SUMMARY.md @@ -0,0 +1,48 @@ +# Phase 7 Wave 2 Summary: Backend Config Refactoring (07-02) + +**Completed:** 2026-04-23 +**Status:** [COMPLETED] +**Commits:** ae61fa63, 28cdc900, 938fd2da, 225972b8 + +--- + +## Accomplishments + +1. **backend/config_loader.py Refactored** + - Implemented YAML parsing using PyYAML + - Enforced D-06 load order: System Env Vars > `config/backend.yaml` > `config/secrets.yaml` > Defaults + - Added robust validation for required fields (JWT_SECRET_KEY, primary_ai_provider) + - Masked sensitive values in logs + +2. **backend/config_manager.py Updated** + - Refactored to handle YAML file operations (`read_config`, `update_config`) + - Removed legacy `.env` file manipulation logic + +3. **backend/main.py Updated** + - Switched from direct `os.environ` access to the centralized `get_config()` dict + - Removed `load_dotenv()` and `inventory.env` references + +4. **backend/entrypoint.sh Updated** + - Updated to document the new YAML configuration structure + - Added safety checks for the existence of `backend.yaml` + +5. **Additional Backend Files Cleaned Up** + - Refactored `backend/ai_vision.py`, `backend/check_models.py`, `backend/ai/gemini.py`, and `backend/ai/claude.py` to use `config_loader` + - Completely removed `python-dotenv` dependency from backend + +--- + +## Technical Details + +- **Load Order (D-06):** Env Vars override YAML, YAML overrides Defaults. +- **Deprecation (D-04):** `inventory.env` is no longer used by the backend application logic. +- **Security:** Sensitive configuration values are masked in application logs to prevent data leaks. + +--- + +## Verification + +- [x] All updated Python files passed `py_compile` checks +- [x] No `load_dotenv()` or `inventory.env` references remain in backend code +- [x] Configuration loading logs which source was used for each value +- [x] Environment variables correctly override YAML values in `config_loader.py` diff --git a/.planning/phases/07-config-consolidation/07-03-PLAN.md b/.planning/phases/07-config-consolidation/07-03-PLAN.md new file mode 100644 index 00000000..bf70d11c --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-03-PLAN.md @@ -0,0 +1,583 @@ +--- +phase: 07-config-consolidation +plan: 03 +type: execute +wave: 2 +depends_on: + - 07-01 +files_modified: + - scripts/deploy.py + - scripts/run_standalone.py + - scripts/install_service.py + - scripts/export_prod.py +autonomous: true +requirements: + - PHASE-7-PYTHON-SCRIPTS + - PHASE-7-YAML-PARSING + - PHASE-7-DEPLOYMENT +user_setup: [] + +must_haves: + truths: + - "Python deployment scripts (deploy.py, run_standalone.py, install_service.py, export_prod.py) exist and parse YAML config" + - "All scripts parse config/*.yaml files using PyYAML (per D-05)" + - "deploy.py handles Docker deployment with health checks" + - "run_standalone.py launches backend and frontend without Docker" + - "install_service.py installs systemd service with new config paths" + - "export_prod.py exports production data/config for backups" + - "All scripts are executable and tested" + artifacts: + - path: "scripts/deploy.py" + provides: "Docker deployment with YAML config parsing, pre-flight checks, health validation" + exports: ["main()"] + min_lines: 150 + - path: "scripts/run_standalone.py" + provides: "Standalone launcher for backend (FastAPI) and frontend (Next.js) with YAML config" + exports: ["main()"] + min_lines: 120 + - path: "scripts/install_service.py" + provides: "Systemd service installation with config paths" + exports: ["main()"] + min_lines: 100 + - path: "scripts/export_prod.py" + provides: "Production export/backup script with YAML config support" + exports: ["main()"] + min_lines: 100 + key_links: + - from: "scripts/deploy.py" + to: "config/docker.yaml" + via: "PyYAML parsing for container config" + pattern: "yaml\\.safe_load.*docker\\.yaml" + - from: "scripts/run_standalone.py" + to: "config/backend.yaml" + via: "Read port and path config" + pattern: "yaml\\.safe_load.*backend\\.yaml" + - from: "scripts/install_service.py" + to: "inventory.service.template" + via: "Service file generation" + pattern: "template|service" +--- + + +Convert bash deployment scripts (deploy.sh, run_standalone.sh, install_service.sh, export_prod.sh) to Python with YAML config parsing. Provide consistent, maintainable deployment tooling that understands the new config structure. + +Purpose: Implement D-05 (Python scripts with YAML parsing) for modern deployment infrastructure. + +Output: 4 Python scripts in scripts/ folder with full deployment functionality and YAML config support. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@DEPLOYMENT.md +@deploy.sh +@run_standalone.sh +@install_service.sh +@export_prod.sh +@config/backend.yaml.example +@config/docker.yaml.example +@docker-compose.yml + + + + + + Task 1: Create scripts/deploy.py (Docker deployment with YAML config) + + scripts/deploy.py + + + - deploy.sh (current bash implementation to port) + - docker-compose.yml (structure and environment variables) + - config/docker.yaml.example (schema) + - config/backend.yaml.example (config structure) + - DEPLOYMENT.md (deployment procedure documentation) + + + Create scripts/deploy.py to replace deploy.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature and usage:** + ```bash + python3 scripts/deploy.py [production|staging|development] [--rebuild] + ``` + + 2. **Core functionality:** + - Pre-flight checks: Docker, Docker Compose, docker-compose.yml, config files + - Load config from config/docker.yaml and config/network.yaml + - Port availability checks (from network.yaml: backend_port, frontend_port, etc.) + - Environment file validation (config/backend.yaml exists and has required values) + - Docker Compose up with appropriate flags (rebuild if --rebuild) + - Health check polling (curl to /health endpoints) + - Deployment report (services running, ports, access URLs) + + 3. **Config file parsing:** + - Use PyYAML to load config/docker.yaml (for container resource limits, image names) + - Use PyYAML to load config/network.yaml (for port numbers and SSL settings) + - Use PyYAML to load config/backend.yaml (to validate required values) + - Fallback to sensible defaults if config files missing (but log warnings) + + 4. **Pre-flight checks (Step 1-5):** + - [ ] docker command available + - [ ] docker-compose command available + - [ ] docker-compose.yml exists + - [ ] config/backend.yaml exists (with helpful error if missing) + - [ ] config/network.yaml exists (with helpful error if missing) + + 5. **Port availability check (Step 6):** + - Read backend_port, frontend_port, backend_ssl_port, frontend_ssl_port from network.yaml + - Use netstat or ss to check if ports are in use + - Error if ports occupied, suggest alternatives + + 6. **Environment validation (Step 7):** + - Check config/backend.yaml for required values: JWT_SECRET_KEY, primary_ai_provider + - Warn if API keys are placeholders + - Proceed with warning (not error) for optional values + + 7. **Docker Compose deployment (Step 8-9):** + - Run `docker-compose up -d` (or with --build if --rebuild flag) + - Capture and display output with color codes + - Catch errors and provide helpful debugging steps + + 8. **Health checks (Step 10-11):** + - Poll backend health: `curl http://localhost:{backend_port}/health` (retry logic) + - Poll frontend health: `curl http://localhost:{frontend_port}/` (retry logic) + - Wait up to 2 minutes for services to become healthy + - Display health status to user + + 9. **Deployment report (Step 12):** + - Display service status: `docker-compose ps` + - Display access URLs: + - Frontend: http://localhost:{frontend_port} + - Backend API: http://localhost:{backend_port}/docs + - HTTPS: https://localhost:{frontend_ssl_port} (if SSL enabled in network.yaml) + - Display next steps (logs, troubleshooting, etc.) + + 10. **Error handling:** + - Descriptive error messages with suggested fixes + - Log all actions and results to stdout/stderr + - Use color output (GREEN for success, RED for errors, YELLOW for warnings, BLUE for info) + - Exit codes: 0 for success, 1 for fatal error + + 11. **Logging:** + - Use Python logging module (not print) + - Log level: INFO by default, DEBUG if --verbose flag + - Each step logged: "Step N/M: Description..." + - Results logged at end: "Deployment complete, services healthy" + + 12. **Required libraries:** + - sys, os, subprocess, time, socket (built-in) + - yaml (PyYAML) + - argparse (for CLI args) + - logging (for logging) + - No external deployment libraries (keep it simple) + + 13. **Make executable:** `chmod +x scripts/deploy.py` and include shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/deploy.py && head -1 scripts/deploy.py | grep -q "python3"` (shebang present) + - `test -x scripts/deploy.py` (executable) + - `python3 -m py_compile scripts/deploy.py` (valid Python syntax) + - `python3 scripts/deploy.py --help | grep -q "deployment"` (help works) + - `grep -q "import yaml" scripts/deploy.py` (PyYAML imported) + - `grep -q "config/docker.yaml\|config/network.yaml" scripts/deploy.py` (loads config files) + - `grep -q "docker-compose" scripts/deploy.py` (calls docker-compose) + - `grep -q "curl.*health" scripts/deploy.py` (health checks present) + + + scripts/deploy.py created with Docker deployment, YAML config parsing, health checks, and error handling. + + + + + Task 2: Create scripts/run_standalone.py (Standalone launcher with YAML config) + + scripts/run_standalone.py + + + - run_standalone.sh (current bash implementation to port) + - config/backend.yaml.example (schema) + - config/frontend.yaml.example (schema) + - backend/main.py (backend entry point) + - frontend package.json or next.config.js (frontend startup) + + + Create scripts/run_standalone.py to replace run_standalone.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature:** + ```bash + python3 scripts/run_standalone.py [--backend-only|--frontend-only] + ``` + + 2. **Core functionality:** + - Load config from config/backend.yaml and config/frontend.yaml + - Start FastAPI backend (uvicorn) + - Start Next.js frontend (npm run dev or node server.js) + - Display console output from both processes + - Handle shutdown gracefully (SIGTERM/SIGINT kills both services) + - Display health status and access URLs + + 3. **Config file parsing:** + - Load config/backend.yaml to get: backend_port, data_dir, logs_dir, log_level + - Load config/frontend.yaml to get: frontend_port, backend_url + - Use defaults if config files missing (with warnings) + + 4. **Backend startup (--backend-only or default):** + - Command: `uvicorn backend.main:app --host 0.0.0.0 --port {backend_port} --reload` + - Set environment: DATA_DIR, LOGS_DIR, LOG_LEVEL (from config) + - Capture output and display with [BACKEND] prefix + - Wait for backend to log "Uvicorn running on..." or similar + - Verify backend is listening on backend_port + + 5. **Frontend startup (--frontend-only or default):** + - Command: `npm run dev` (if in development) or `node server.js` (if built) + - Set environment: NEXT_PUBLIC_API_URL (from config:frontend:backend_url) + - Capture output and display with [FRONTEND] prefix + - Wait for frontend to log "ready - started server on..." or similar + - Verify frontend is listening on frontend_port + + 6. **Process management:** + - Use subprocess.Popen with shell=False (for security) + - Manage both processes in list/tuple + - Handle SIGTERM/SIGINT (Ctrl+C) to kill both processes + - Display "Shutting down..." and wait for clean shutdown + - Exit with code 0 if both shut down cleanly + + 7. **Health monitoring:** + - Periodically check if processes are alive (poll returncode) + - If one process dies, log error and optionally shutdown other (per config flag) + - Display uptime and status every 30 seconds + + 8. **Logging and output:** + - Use Python logging module + - Log each process with [BACKEND] / [FRONTEND] prefix + - Merge stdout/stderr from both processes to terminal + - Show final status: "Backend running on http://localhost:{backend_port}, Frontend on http://localhost:{frontend_port}" + + 9. **Error handling:** + - If uvicorn not installed, error and suggest: `pip install uvicorn` + - If npm not found, error and suggest: install Node.js + - If ports already in use, error with port number + - If config files missing, log warnings but use defaults + + 10. **Required libraries:** + - sys, os, subprocess, signal, time (built-in) + - yaml (PyYAML) + - argparse (for CLI args --backend-only, --frontend-only) + - logging (for logging) + + 11. **Make executable:** `chmod +x scripts/run_standalone.py` with shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/run_standalone.py && head -1 scripts/run_standalone.py | grep -q "python3"` (shebang present) + - `test -x scripts/run_standalone.py` (executable) + - `python3 -m py_compile scripts/run_standalone.py` (valid Python syntax) + - `grep -q "import yaml" scripts/run_standalone.py` (PyYAML imported) + - `grep -q "config/backend.yaml\|config/frontend.yaml" scripts/run_standalone.py` (loads config) + - `grep -q "uvicorn\|subprocess.Popen" scripts/run_standalone.py` (backend startup present) + - `grep -q "npm\|node server" scripts/run_standalone.py` (frontend startup present) + - `grep -q "signal.signal\|SIGTERM" scripts/run_standalone.py` (signal handling present) + + + scripts/run_standalone.py created with backend/frontend startup, YAML config parsing, process management, and graceful shutdown. + + + + + Task 3: Create scripts/install_service.py (Systemd service installation) + + scripts/install_service.py + + + - install_service.sh (current bash implementation to port) + - inventory.service.template (systemd service template) + - config/backend.yaml.example (to understand config structure) + - config/network.yaml.example (for port information) + + + Create scripts/install_service.py to replace install_service.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature:** + ```bash + sudo python3 scripts/install_service.py [--user=service_user] [--port=port] + ``` + + 2. **Core functionality:** + - Read inventory.service.template (or create template inline) + - Load config from config/backend.yaml (for paths, ports) + - Generate systemd service file with correct paths and user/group + - Install service file to /etc/systemd/system/ainventory.service + - Enable service (systemctl enable) + - Display installation summary and next steps + + 3. **Config file parsing:** + - Load config/backend.yaml to get: data_dir, logs_dir + - Load config/network.yaml to get: backend_port (for documentation) + - Use defaults if missing + + 4. **Service file generation:** + - Read inventory.service.template + - Replace placeholders: + - {PROJECT_DIR}: current working directory (project root) + - {SERVICE_USER}: service user (default: www-data, configurable via --user) + - {BACKEND_PORT}: from config/network.yaml + - {DATA_DIR}: from config/backend.yaml + - {LOGS_DIR}: from config/backend.yaml + - Template should: + - Type=simple + - ExecStart=/usr/bin/python3 {PROJECT_DIR}/scripts/run_standalone.py --backend-only + - WorkingDirectory={PROJECT_DIR} + - User={SERVICE_USER} + - Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin + - Restart=on-failure + - RestartSec=10 + + 5. **Permission checks:** + - Require sudo/root: `if os.getuid() != 0: error("Must run with sudo")` + - Check project directory is readable + - Check data_dir and logs_dir exist or can be created + + 6. **Service file installation:** + - Write service file to /etc/systemd/system/ainventory.service + - Set permissions: 644 (readable, not writable by non-root) + - Run `systemctl daemon-reload` + - Run `systemctl enable ainventory.service` + - Optionally start service: `systemctl start ainventory.service` + + 7. **Status display:** + - Show service file location + - Show service user and group + - Show project directory + - Show next steps: `systemctl start ainventory`, `systemctl status ainventory` + - Show logs: `journalctl -u ainventory -f` + + 8. **Error handling:** + - Check if service already installed (offer --force to overwrite) + - Check if user exists (suggest: `useradd -r -s /bin/false {user}`) + - Check if directories are writable + - Descriptive errors with suggested fixes + + 9. **Required libraries:** + - sys, os, subprocess, pwd, grp (built-in) + - yaml (PyYAML) + - argparse (for CLI args) + - logging (for logging) + + 10. **Make executable:** `chmod +x scripts/install_service.py` with shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/install_service.py && head -1 scripts/install_service.py | grep -q "python3"` (shebang present) + - `test -x scripts/install_service.py` (executable) + - `python3 -m py_compile scripts/install_service.py` (valid Python syntax) + - `grep -q "import yaml" scripts/install_service.py` (PyYAML imported) + - `grep -q "config/backend.yaml\|config/network.yaml" scripts/install_service.py` (loads config) + - `grep -q "/etc/systemd/system\|systemctl" scripts/install_service.py` (systemd integration present) + - `grep -q "os.getuid\|sudo" scripts/install_service.py` (permission check present) + + + scripts/install_service.py created with systemd service generation, config parsing, and installation logic. + + + + + Task 4: Create scripts/export_prod.py (Production export/backup) + + scripts/export_prod.py + + + - export_prod.sh (current bash implementation to port) + - config/backend.yaml.example (for data_dir) + - DEPLOYMENT.md (backup procedures) + + + Create scripts/export_prod.py to replace export_prod.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature:** + ```bash + python3 scripts/export_prod.py [--output=/path/to/backup.tar.gz] [--include-logs] + ``` + + 2. **Core functionality:** + - Load config from config/backend.yaml (to find data_dir, logs_dir) + - Create tar.gz archive of production data + - Include database file(s), config files (no secrets), and optionally logs + - Output to specified location or default: backups/{timestamp}.tar.gz + - Display archive size and location + + 3. **Config file parsing:** + - Load config/backend.yaml to get: data_dir, logs_dir + - Use defaults if missing: data_dir=./data, logs_dir=./logs + + 4. **Archive creation:** + - Include: {data_dir}/* (all application data, database, etc.) + - Include: config/*.yaml.example (config templates) + - Include: config/backend.yaml, config/frontend.yaml, config/network.yaml (actual configs, no secrets) + - Include: config/secrets.yaml.example (secrets template only, NOT actual secrets.yaml) + - Include: logs/* (optional, if --include-logs flag) + - Exclude: config/secrets.yaml (never backup actual secrets) + - Exclude: node_modules/, __pycache__/, .git/, .venv/ + - Exclude: temporary files, cache + + 5. **Archive naming:** + - Default: backups/ainventory_{timestamp}.tar.gz + - Timestamp format: YYYY-MM-DD_HH-MM-SS + - Custom path via --output flag + + 6. **Backup directory:** + - Create backups/ directory if not exists + - Set directory permissions: 750 (rwxr-x---) + + 7. **Verification:** + - Verify tar.gz was created successfully + - Display archive size: X.XX MB + - Display archive contents summary: "Includes database, data, and config (secrets excluded)" + + 8. **Error handling:** + - If data_dir doesn't exist, error and suggest creating it + - If no write permission to backups/, error and suggest location + - If tar command fails, show error and suggest troubleshooting + + 9. **Output example:** + ``` + [INFO] Loading config from config/backend.yaml + [INFO] Data directory: ./data + [INFO] Creating backup... + [INFO] Archive created: backups/ainventory_2026-04-23_14-30-45.tar.gz + [INFO] Archive size: 125.43 MB + [INFO] Contents: database, data, config (secrets excluded) + [INFO] Backup complete! + ``` + + 10. **Required libraries:** + - sys, os, subprocess, datetime, tarfile (built-in) + - yaml (PyYAML) + - argparse (for CLI args) + - logging (for logging) + + 11. **Make executable:** `chmod +x scripts/export_prod.py` with shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/export_prod.py && head -1 scripts/export_prod.py | grep -q "python3"` (shebang present) + - `test -x scripts/export_prod.py` (executable) + - `python3 -m py_compile scripts/export_prod.py` (valid Python syntax) + - `grep -q "import yaml" scripts/export_prod.py` (PyYAML imported) + - `grep -q "config/backend.yaml" scripts/export_prod.py` (loads config) + - `grep -q "tarfile\|tar.gz" scripts/export_prod.py` (tar archive creation present) + - `grep -q "secrets.yaml" scripts/export_prod.py | grep -q "exclude"` (secrets excluded from backup) + + + scripts/export_prod.py created with production data export, YAML config parsing, archive creation, and secrets exclusion. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| User input → Script | Script arguments and config files must be validated | +| Script → System | Scripts execute system commands (subprocess) — must escape/quote properly | +| Script → Network | Health checks make HTTP requests (must handle timeouts) | +| Script → Filesystem | Scripts read/write files (must respect permissions) | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-09 | Injection | deploy.py subprocess | mitigate | Use subprocess with shell=False and list args (not f-strings). Example: `subprocess.run(["docker-compose", "up", "-d"], ...)` not `subprocess.run(f"docker-compose up -d", shell=True)`. | +| T-07-10 | Elevation of Privilege | install_service.py sudo | mitigate | Check `os.getuid() != 0` at start. Require sudo for systemd operations only. Log all systemctl calls. | +| T-07-11 | Information Disclosure | export_prod.py backup | mitigate | Exclude config/secrets.yaml explicitly in tarfile. Log what is excluded. Verify file permissions (backups/ dir 750). | +| T-07-12 | Denial of Service | Health checks timeout | mitigate | Set socket timeout to 10 seconds. Limit retry attempts to 12 (2 minutes total). Log timeout errors. | + + + + +**Phase 7, Plan 3 Verification Checklist:** + +1. **scripts/deploy.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Loads config/docker.yaml and config/network.yaml + - [ ] Pre-flight checks for Docker, Docker Compose, config files + - [ ] Port availability checks implemented + - [ ] Health checks poll backend and frontend endpoints + - [ ] Color output for info/warning/success/error + - [ ] Displays deployment summary and access URLs + - [ ] Valid Python syntax + +2. **scripts/run_standalone.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Loads config/backend.yaml and config/frontend.yaml + - [ ] Launches uvicorn for backend with correct port and settings + - [ ] Launches frontend (npm dev or node server.js) with correct port + - [ ] Signal handling (SIGTERM/SIGINT) for clean shutdown + - [ ] Process monitoring and output display with prefixes + - [ ] Valid Python syntax + +3. **scripts/install_service.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Checks for sudo/root permission + - [ ] Loads config/backend.yaml and config/network.yaml + - [ ] Generates systemd service file from template + - [ ] Replaces placeholders: {PROJECT_DIR}, {SERVICE_USER}, {BACKEND_PORT}, etc. + - [ ] Installs to /etc/systemd/system/ with correct permissions + - [ ] Runs systemctl daemon-reload and enable + - [ ] Valid Python syntax + +4. **scripts/export_prod.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Loads config/backend.yaml to find data_dir + - [ ] Creates tar.gz archive with data and config files + - [ ] Excludes config/secrets.yaml (actual secrets, not example) + - [ ] Includes config/*.yaml.example files + - [ ] Output to backups/{timestamp}.tar.gz or custom path + - [ ] Displays archive size and summary + - [ ] Valid Python syntax + +5. **Security (subprocess, permissions, file ops)** + - [ ] All subprocess calls use shell=False with list args + - [ ] No f-strings in shell commands + - [ ] File operations respect umask/permissions + - [ ] No hardcoded credentials in scripts + +6. **Integration** + - [ ] Each script loads YAML config files correctly + - [ ] Scripts reference new config/ structure (not inventory.env) + - [ ] Error messages are helpful and actionable + + + +- 4 Python scripts created (deploy.py, run_standalone.py, install_service.py, export_prod.py) +- All scripts use PyYAML to parse config files +- All scripts are executable with proper shebangs +- deploy.py handles Docker deployment with health checks +- run_standalone.py launches backend and frontend without Docker +- install_service.py creates systemd service with new config paths +- export_prod.py exports production data excluding secrets +- All subprocess calls use shell=False (secure) +- Error handling and logging present in all scripts +- Scripts integrate with new config/ structure (D-05, D-06, D-07) + + + +After completion, create `.planning/phases/07-config-consolidation/07-03-SUMMARY.md` + diff --git a/.planning/phases/07-config-consolidation/07-03-SUMMARY.md b/.planning/phases/07-config-consolidation/07-03-SUMMARY.md new file mode 100644 index 00000000..946650af --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-03-SUMMARY.md @@ -0,0 +1,45 @@ +# Phase 7 Wave 2 Summary: Python Deployment Scripts (07-03) + +**Completed:** 2026-04-23 +**Status:** [COMPLETED] +**Commits:** ce79c919, 1621625b, 63f72c11, 9253eb65 + +--- + +## Accomplishments + +1. **scripts/deploy.py Created** + - Replaces `deploy.sh` with a secure, robust Python implementation + - Implements pre-flight checks, port availability validation, and health check polling + - Parses YAML config from `config/docker.yaml` and `config/network.yaml` + +2. **scripts/run_standalone.py Created** + - Replaces `run_standalone.sh` for multi-process management without Docker + - Handles graceful shutdown (SIGINT/SIGTERM) of both backend and frontend + - Provides prefixed, colored console output for logs + +3. **scripts/install_service.py Created** + - Replaces `install_service.sh` for systemd service setup + - Generates service file from template with correct YAML-based paths + +4. **scripts/export_prod.py Created** + - Replaces `export_prod.sh` for production data backups + - Explicitly excludes `config/secrets.yaml` to ensure security in backups + +--- + +## Technical Details + +- **YAML Parsing (D-05):** All scripts use PyYAML to read the new centralized configuration structure. +- **Security:** Subprocess calls use `shell=False` with list arguments to prevent injection attacks. +- **Tooling:** Implemented consistent logging and color output for developer experience. + +--- + +## Verification + +- [x] All 4 Python scripts are executable (`chmod +x`) +- [x] All scripts use the proper shebang (`#!/usr/bin/env python3`) +- [x] All scripts correctly parse the YAML configuration files +- [x] Subprocess execution follows security best practices +- [x] Error handling and helpful feedback messages are present in all tools diff --git a/.planning/phases/07-config-consolidation/07-04-PLAN.md b/.planning/phases/07-config-consolidation/07-04-PLAN.md new file mode 100644 index 00000000..a0257a35 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-04-PLAN.md @@ -0,0 +1,642 @@ +--- +phase: 07-config-consolidation +plan: 04 +type: execute +wave: 3 +depends_on: + - 07-01 + - 07-02 + - 07-03 +files_modified: + - docker-compose.yml + - backend/Dockerfile + - backend/entrypoint.sh + - .gitignore + - DEPLOYMENT.md + - README.md +autonomous: true +requirements: + - PHASE-7-DOCKER-UPDATE + - PHASE-7-DOCUMENTATION + - PHASE-7-GITIGNORE +user_setup: [] + +must_haves: + truths: + - "docker-compose.yml updated to reference config/ volume and remove inventory.env env_file" + - "backend/Dockerfile and entrypoint updated for new config paths" + - ".gitignore properly configured to track examples, ignore actual configs and secrets" + - "DEPLOYMENT.md updated with YAML config structure, new Python scripts, setup instructions" + - "README.md updated with config setup and configuration management instructions" + - "Docker deployment works with new config structure (tested with docker-compose up)" + - "All documentation references config/ as single source of truth" + artifacts: + - path: "docker-compose.yml" + provides: "Docker Compose with config/ volume mount, no inventory.env env_file reference" + pattern: "\\./config:/app/config|!inventory.env" + min_lines: 120 + - path: "backend/Dockerfile" + provides: "Backend container image with new config paths" + pattern: "config/|/app/config" + - path: "backend/entrypoint.sh" + provides: "Docker entrypoint with config/ reference" + pattern: "config/|/app/config" + - path: "DEPLOYMENT.md" + provides: "Updated deployment guide with YAML config structure and Python scripts" + min_lines: 150 + - path: "README.md" + provides: "Updated README with config setup and onboarding" + min_lines: 100 + - path: ".gitignore" + provides: ".gitignore with rules for config/ folder (track examples, ignore secrets)" + pattern: "config/.*\\.yaml" + key_links: + - from: "docker-compose.yml" + to: "config/" + via: "volume mount" + pattern: "\\./config:/app/config" + - from: "DEPLOYMENT.md" + to: "config/README.md" + provides: "Cross-reference to config documentation" + pattern: "config/README.md|config/" + - from: "README.md" + to: "DEPLOYMENT.md" + provides: "Cross-reference to deployment guide" + pattern: "DEPLOYMENT.md|config/" +--- + + +Update Docker Compose, Dockerfile, documentation, and .gitignore to integrate the new config/ structure. Remove references to inventory.env from deployment infrastructure and update all deployment documentation. + +Purpose: Complete D-07 (Docker & Compose update), D-08 (documentation), and D-04 (deprecation) for cohesive deployment experience. + +Output: Updated docker-compose.yml, Dockerfile, entrypoint.sh, DEPLOYMENT.md, README.md, and .gitignore. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@docker-compose.yml +@backend/Dockerfile +@backend/entrypoint.sh +@DEPLOYMENT.md +@README.md +@.gitignore +@config/README.md + + + + + + Task 1: Update docker-compose.yml to reference config/ and remove inventory.env env_file + + docker-compose.yml + + + - docker-compose.yml (current file) + - backend/entrypoint.sh (to understand how config is used in containers) + - config/backend.yaml.example (to understand what config is needed) + - config/docker.yaml.example (Docker-specific config) + + + Update docker-compose.yml to integrate new config/ structure (per D-07): + + 1. **Remove inventory.env env_file references:** + - Delete or comment out `env_file: - inventory.env` from backend service (currently line 14) + - Delete or comment out `env_file: - inventory.env` from frontend service (currently line 51) + - Keep proxy service as is (may not need env_file) + + 2. **Add config/ volume mount to backend service:** + - Keep existing volume mounts + - Add: `- ./config:/app/config:ro` (read-only, config should not be modified in container) + - Update volumes section to reflect new mount + + 3. **Add config/ volume mount to frontend service (if frontend needs config):** + - Add: `- ./config:/app/config:ro` if frontend needs to read config files + - Or skip if frontend doesn't read YAML config directly + + 4. **Update environment variables:** + - Keep all existing environment variables (Docker overrides are still valid per D-06) + - Ensure JWT_SECRET_KEY is still set with warning: `# CHANGE THIS IN PRODUCTION!` + - Add comment: "Environment variables override config/backend.yaml per D-06 load order" + - Ensure DATA_DIR and LOGS_DIR are set to persist volume locations + + 5. **Add proxy service config/ mount (if proxy reads config):** + - Check if Caddyfile uses any dynamic config + - If not, no change needed + - If yes, add: `- ./config:/app/config:ro` + + 6. **Verify volume definitions:** + - Named volumes (backend_data, backend_logs, frontend_logs, caddy_data, caddy_config) remain unchanged + - Config mount is bind mount (./config), not named volume + + 7. **Add comments explaining the change:** + - Add section comment before volumes: "# [D-07] New config/ structure — YAML config mounted read-only" + - Add comment on env_file removal: "# [D-04] inventory.env deprecated — config now in config/ folder" + - Reference Phase 7 decisions + + 8. **Maintain backward compatibility during transition:** + - Don't delete old env_file line yet (can exist but be ignored by modern docker-compose) + - Or clearly comment it out with deprecation notice + + Example section after update: + ```yaml + backend: + ... + # [D-04] inventory.env deprecated — see config/ folder instead + # env_file: - inventory.env + volumes: + - backend_data:/app/data + - backend_logs:/app/logs + # [D-07] New config/ structure mounted read-only + - ./config:/app/config:ro + - ./scripts:/app/scripts:ro + ... + ``` + + + - `grep -n "env_file" docker-compose.yml | head` (check if inventory.env env_file is removed/commented) + - `grep -q "\\./config:/app/config:ro" docker-compose.yml` (config volume mount present) + - `docker-compose config 2>&1 | grep -q "config" || echo "valid yaml"` (valid docker-compose syntax) + - `grep -q "D-07\|D-04" docker-compose.yml || echo "pass"` (comments reference phase decisions, optional) + + + docker-compose.yml updated with config/ volume mount, inventory.env env_file removed, comments documenting changes. + + + + + Task 2: Update backend/Dockerfile for new config paths + + backend/Dockerfile + + + - backend/Dockerfile (current file) + - docker-compose.yml (just updated) + - backend/entrypoint.sh (how config is used at runtime) + + + Update backend/Dockerfile to document/support new config/ paths (per D-07): + + 1. **Add comments explaining config/ structure:** + - Add comment at top: "# [D-07] Backend container - config/ folder mounted at /app/config (read-only)" + - Add comment before WORKDIR: "# Config is expected in /app/config (mounted from host)" + + 2. **Ensure volume mount points exist:** + - Config is mounted at runtime by docker-compose, not created in Dockerfile + - No changes needed to RUN commands for config directory + - (Already handled by docker-compose volume mount) + + 3. **Update any hardcoded paths referencing inventory.env:** + - Search for "inventory.env" in Dockerfile + - Replace with reference to config/ or remove if no longer needed + - Example: If old CMD references inventory.env, update to reference config/ + + 4. **Update ENTRYPOINT or CMD if needed:** + - Ensure entrypoint.sh (or equivalent) references config/ paths + - Add environment documentation: "# Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables" + + 5. **Add healthcheck if not present:** + - Verify backend has healthcheck (curl to /health endpoint) + - Should already be in docker-compose.yml, but double-check Dockerfile + + 6. **Document environment variables:** + - Add comment: "# Environment variables override YAML config per D-06" + - List: DATA_DIR, LOGS_DIR, LOG_LEVEL (these come from env and/or config) + + Example after update: + ```dockerfile + # [D-07] Backend container - config/ folder mounted at /app/config (read-only) + # Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables + # Environment variables override YAML config per D-06 load order + + FROM python:3.12-slim + WORKDIR /app + + # Copy code, requirements, and startup scripts + COPY backend/ ./backend/ + COPY scripts/ ./scripts/ + COPY requirements.txt . + + # Install dependencies + RUN pip install --no-cache-dir -r requirements.txt + + # Config is mounted at /app/config by docker-compose + # No need to COPY config/ here (it's mounted read-only) + + ENTRYPOINT ["python", "backend/main.py"] + ``` + + + - `grep -q "D-07\|config/" backend/Dockerfile || echo "pass"` (comments reference config, optional) + - `grep -q "inventory.env" backend/Dockerfile` should return empty (no old inventory.env refs) + - `docker build -f backend/Dockerfile .` (valid Dockerfile syntax — may not succeed without full context, but no syntax errors) + + + backend/Dockerfile updated with comments documenting config/ structure, no inventory.env references. + + + + + Task 3: Update backend/entrypoint.sh for new config paths + + backend/entrypoint.sh + + + - backend/entrypoint.sh (current file) + - backend/config_loader.py (updated in Plan 2, to understand config loading) + - config/README.md (documentation on config structure) + + + Update backend/entrypoint.sh to reference and support new config/ structure (per D-07): + + 1. **Remove inventory.env sourcing:** + - Delete any lines that source or check for inventory.env + - Delete any EXPORT statements that copy inventory.env values to environment + + 2. **Add config/ path documentation:** + - Add comment at top: "# [D-07] Backend entrypoint - loads config from /app/config/ (YAML format)" + - Add comment: "# Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables" + + 3. **Add environment variable override documentation:** + - Add comment: "# [D-06] Environment variables override YAML config — set below takes precedence" + - List typical overrides: JWT_SECRET_KEY, PRIMARY_AI_PROVIDER, LOG_LEVEL, etc. + + 4. **Ensure config validation:** + - Add check: if [ ! -f "/app/config/backend.yaml" ]; then log error and instructions + - Add comment: "# Config validation handled by Python config_loader.py" + + 5. **Set working directory:** + - Ensure WORKDIR is set to /app (should be done in Dockerfile, but double-check) + + 6. **Exec main process:** + - Ensure entrypoint uses `exec` to replace shell: `exec python backend/main.py` + - This ensures signals (SIGTERM) are properly handled by Python process + + Example after update: + ```bash + #!/bin/bash + # [D-07] Backend entrypoint - loads config from /app/config/ (YAML format) + # Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables + # [D-06] Environment variables override YAML config (below takes precedence) + + set -euo pipefail + + cd /app + + # Verify config is accessible + if [ ! -f "/app/config/backend.yaml" ]; then + echo "[ERROR] /app/config/backend.yaml not found!" + echo "[ERROR] Config must be mounted from host at /app/config/" + echo "[ERROR] See config/README.md for setup instructions" + exit 1 + fi + + # Environment variables below override YAML config + # (docker run -e JWT_SECRET_KEY="..." or docker-compose environment) + + # Start backend (signals properly handled with exec) + exec python -m uvicorn backend.main:app --host 0.0.0.0 --port 8000 + ``` + + 7. **Keep it minimal:** + - Entrypoint should be simple (most logic in Python config_loader.py) + - Just verify config exists and start the app + + + - `grep -q "D-07\|D-06\|config/" backend/entrypoint.sh` (references config structure, optional) + - `grep -q "inventory.env" backend/entrypoint.sh` should return empty (no old config) + - `bash -n backend/entrypoint.sh` (valid bash syntax) + - `head -1 backend/entrypoint.sh | grep -q "bash"` (shebang present) + + + backend/entrypoint.sh updated to reference config/ paths, remove inventory.env, document env var overrides. + + + + + Task 4: Update .gitignore to track config examples and ignore actual configs/secrets + + .gitignore + + + - .gitignore (current file) + - config/backend.yaml.example (created in Plan 1) + - config/secrets.yaml.example (created in Plan 1) + + + Update .gitignore to properly handle config/ folder (per D-03, D-08): + + 1. **Add config/ rules:** + ``` + # [D-08] Config folder — track examples, ignore actual configs and secrets + config/*.yaml + !config/*.yaml.example + config/secrets.yaml + !config/secrets.yaml.example + ``` + + 2. **Rationale:** + - `config/*.yaml` — Ignore all YAML files (actual configs with real secrets) + - `!config/*.yaml.example` — Except examples (these are tracked for schema/documentation) + - `config/secrets.yaml` — Explicitly ignore secrets file (redundant but clear) + - `!config/secrets.yaml.example` — Except example (for developer setup guidance) + + 3. **Clean up old rules:** + - Remove any existing `inventory.env` entries from .gitignore (deprecated) + - Or update to comment them as deprecated: `# inventory.env # [D-04] Deprecated - use config/backend.yaml` + + 4. **Add comment at top of config section:** + - Add comment: "# [D-04] inventory.env deprecated — see config/ folder instead" + - Add comment: "# [D-08] Config structure: examples tracked, actual configs ignored" + + 5. **Example .gitignore config section:** + ``` + # [D-04] inventory.env deprecated — see config/ folder instead + # [D-08] Config structure: examples tracked (schema), actual configs ignored (secrets) + config/*.yaml + !config/*.yaml.example + config/secrets.yaml + !config/secrets.yaml.example + ``` + + 6. **Verify git status:** + - After update, git status should show: + - config/*.yaml.example as "new file" or tracked + - config/*.yaml as ignored + - config/secrets.yaml as ignored + + + - `grep -q "config/\\*\\.yaml" .gitignore` (config rule present) + - `grep -q "!config/\\*\\.yaml\\.example" .gitignore` (exception for examples present) + - `grep -q "config/secrets\\.yaml" .gitignore` (secrets ignored) + - `grep -q "!config/secrets\\.yaml\\.example" .gitignore` (example tracked) + + + .gitignore updated with config/ rules to track examples and ignore actual configs/secrets. + + + + + Task 5: Update DEPLOYMENT.md with YAML config structure and new Python scripts + + DEPLOYMENT.md + + + - DEPLOYMENT.md (current file) + - config/README.md (created in Plan 1) + - scripts/deploy.py, scripts/run_standalone.py (created in Plan 3) + + + Update DEPLOYMENT.md to document new YAML config structure and Python deployment scripts (per D-08): + + 1. **Add section: Configuration (Before Quick Start)** + - Explain config/ as single source of truth + - List config files: backend.yaml, frontend.yaml, network.yaml, docker.yaml, secrets.yaml + - Reference config/README.md for detailed setup + - Explain examples and how to create actual config from examples + + 2. **Update Quick Start section:** + - Step 1: Clone and cd + - Step 2: Copy config examples to actual files: `cp config/*.yaml.example {without .example}` + - Step 3: Edit config files (backend.yaml, network.yaml, secrets.yaml) with your values + - Step 4: Choose deployment mode (Docker or Standalone) + + 3. **Update Docker Deployment section:** + - Replace old deploy.sh instructions with new deploy.py + - Usage: `python3 scripts/deploy.py [production|staging|development] [--rebuild]` + - Script handles: pre-flight checks, port validation, config loading, health checks + - Output: service status, access URLs + + 4. **Update Standalone Deployment section:** + - Replace old run_standalone.sh with new run_standalone.py + - Usage: `python3 scripts/run_standalone.py [--backend-only|--frontend-only]` + - Script handles: config loading, backend startup, frontend startup, signal handling + + 5. **Update Configuration Reference section:** + - Refer to config/README.md for complete reference + - List key files: config/backend.yaml, config/frontend.yaml, config/network.yaml, config/docker.yaml, config/secrets.yaml + - Explain environment variable overrides (D-06) + + 6. **Add Systemd Service Installation section:** + - Usage: `sudo python3 scripts/install_service.py [--user=www-data]` + - Explain what service does: runs standalone backend + frontend + - Show how to manage: `systemctl start|stop|status ainventory` + - Show logs: `journalctl -u ainventory -f` + + 7. **Add Backup & Export section:** + - Usage: `python3 scripts/export_prod.py [--output=/path/to/backup.tar.gz] [--include-logs]` + - Explains what is included: data, config, config templates + - Explains what is excluded: actual secrets (security), logs (optional) + + 8. **Add Security section:** + - Mention secrets.yaml is git-ignored + - Explain how to set up secrets (copy from example, fill in values) + - Warn about JWT_SECRET_KEY in docker-compose.yml (must change for production) + + 9. **Add Troubleshooting section:** + - Common issues: missing config files, invalid YAML syntax, port conflicts + - Debug steps: check config syntax, verify file permissions, run health checks + - Reference config/README.md for setup help + + 10. **Add Migration section (from old inventory.env):** + - Explain Phase 7 transition from inventory.env to config/ structure + - Provide migration script or manual steps + - Clear instructions: which old values map to which config files + + Structure should be roughly: + - 1. Overview (unchanged) + - 2. Prerequisites (unchanged) + - **3. Configuration** (NEW) + - 4. Quick Start (UPDATED) + - 5. Deployment Modes (Docker, Standalone) (UPDATED to use Python scripts) + - 6. Systemd Service (UPDATED with Python script) + - 7. Backup & Export (UPDATED with Python script) + - 8. Operations & Health Monitoring (UPDATED with new paths) + - 9. Security (NEW or UPDATED) + - 10. Troubleshooting (UPDATED) + - 11. Migration from inventory.env (NEW) + + + - `grep -q "config/.*\\.yaml" DEPLOYMENT.md` (references YAML config files) + - `grep -q "scripts/deploy\\.py\|scripts/run_standalone\\.py" DEPLOYMENT.md` (references Python scripts) + - `grep -q "config/README\\.md" DEPLOYMENT.md` (cross-references config documentation) + - `grep -q "environment.*override\|D-06" DEPLOYMENT.md || echo "pass"` (explains env var overrides, optional) + - File should have 150+ lines: `wc -l DEPLOYMENT.md | awk '$1 >= 150 {print "pass"}'` + + + DEPLOYMENT.md updated with YAML config structure, Python script usage, systemd service, backup procedures, and troubleshooting. + + + + + Task 6: Update README.md with config setup and configuration management instructions + + README.md + + + - README.md (current file) + - DEPLOYMENT.md (just updated) + - config/README.md (created in Plan 1) + + + Update README.md to include configuration management and quick setup instructions (per D-08): + + 1. **Add Configuration section (after Quick Start or before Deployment):** + - Brief explanation: config/ folder is single source of truth + - Quick steps: cp config/*.example to remove .example, edit with your values + - Reference config/README.md for detailed setup + - Reference DEPLOYMENT.md for deployment options + + 2. **Update Quick Start section (if exists):** + - Add configuration step before deployment + - Example: + ``` + # 1. Clone and setup + git clone ... && cd tfm-inventory + + # 2. Configure application + cp config/*.yaml.example config/$(basename {} .example) # or similar + nano config/backend.yaml # Edit with your values + cp config/secrets.yaml.example config/secrets.yaml + nano config/secrets.yaml # Fill in API keys and secrets + + # 3. Deploy (choose one) + python3 scripts/deploy.py production # Docker + # OR + python3 scripts/run_standalone.py # Standalone + ``` + + 3. **Add note about .gitignore:** + - Config examples are tracked (for schema) + - Actual configs are git-ignored (protect secrets) + - secrets.yaml is git-ignored (never commit) + + 4. **Cross-reference documentation:** + - Add links/references to: + - config/README.md (configuration reference) + - DEPLOYMENT.md (detailed deployment guide) + - dev_docs/ (for development setup) + + 5. **Add "Getting Help" section (if not exists):** + - Point to DEPLOYMENT.md troubleshooting + - Point to config/README.md for config questions + - Reference AI_RULES.md for project conventions + + 6. **Update any hardcoded inventory.env references:** + - Replace with config/ references + - Update any env-related docs/examples + + 7. **Maintain existing structure:** + - Don't delete or significantly reorder existing sections + - Just add/update config-related content and update cross-references + + Keep README concise but informative. Detailed docs go in DEPLOYMENT.md and config/README.md. + + + - `grep -q "config/" README.md` (references config structure) + - `grep -q "DEPLOYMENT\\.md\|config/README\\.md" README.md` (cross-references detailed docs) + - `grep -q "secrets.yaml\|git.*ignore" README.md || echo "pass"` (mentions secrets and gitignore, optional) + - File should be valid markdown: `grep "^#" README.md | head -3` (has headers) + + + README.md updated with configuration management, quick setup steps, and documentation cross-references. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| Git repository → Deployment | .gitignore must prevent secrets from being committed | +| Documentation → Users | Documentation must clearly explain security requirements | +| Environment → Container | Docker environment variables can expose secrets if logged | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-13 | Tampering | docker-compose volume mount | mitigate | Config volume mounted read-only `:ro`. Backend cannot modify config at runtime. Changes require host-level edits. | +| T-07-14 | Information Disclosure | DEPLOYMENT.md instructions | mitigate | Documentation warns to generate JWT_SECRET_KEY, don't use placeholder. Warns about secrets.yaml setup. | +| T-07-15 | Information Disclosure | .gitignore config rules | mitigate | Clear rules prevent accidental secret commits. !config/*.example exception ensures schema is tracked. | +| T-07-16 | Elevation of Privilege | Docker container permissions | mitigate | No RUN as root in Dockerfile. Container runs as unprivileged user (if specified in docker-compose). | + + + + +**Phase 7, Plan 4 Verification Checklist:** + +1. **docker-compose.yml** + - [ ] inventory.env env_file removed or commented (per D-04) + - [ ] ./config:/app/config:ro volume mount added to backend service + - [ ] Syntax valid: `docker-compose config` succeeds + - [ ] Comments reference D-07, D-04 decisions + - [ ] Environment variables preserved (JWT_SECRET_KEY etc. with production warning) + +2. **backend/Dockerfile** + - [ ] No references to inventory.env + - [ ] Comments reference config/ structure and D-07 + - [ ] ENTRYPOINT or CMD properly set + - [ ] Syntax valid: `docker build --dry-run` or manual parse + +3. **backend/entrypoint.sh** + - [ ] No sourcing of inventory.env + - [ ] References /app/config/ paths + - [ ] Checks if config/backend.yaml exists + - [ ] Documents environment variable override behavior + - [ ] Bash syntax valid: `bash -n backend/entrypoint.sh` + +4. **.gitignore** + - [ ] Rules added: config/*.yaml, !config/*.yaml.example, config/secrets.yaml, !config/secrets.yaml.example + - [ ] Old inventory.env references removed or marked deprecated + - [ ] Git test: `git check-ignore config/backend.yaml` returns success (ignored) + - [ ] Git test: `git status config/*.example` shows untracked (not ignored) + +5. **DEPLOYMENT.md** + - [ ] Configuration section added before or after Quick Start + - [ ] References config/ files (backend.yaml, frontend.yaml, network.yaml, docker.yaml, secrets.yaml) + - [ ] Docker deployment updated to use scripts/deploy.py + - [ ] Standalone deployment updated to use scripts/run_standalone.py + - [ ] Systemd service section with scripts/install_service.py + - [ ] Backup section with scripts/export_prod.py + - [ ] Troubleshooting section + - [ ] Migration section (from inventory.env to config/) + - [ ] Length 150+ lines + +6. **README.md** + - [ ] Configuration section added + - [ ] Quick start updated with config setup steps + - [ ] Cross-references to config/README.md and DEPLOYMENT.md + - [ ] Mentions secrets.yaml and .gitignore + - [ ] No hardcoded inventory.env references + +7. **Cross-document consistency** + - [ ] README.md, DEPLOYMENT.md, config/README.md use consistent terminology + - [ ] All three documents reference each other appropriately + - [ ] No conflicting instructions across documents + + + +- docker-compose.yml updated with config/ volume mount, inventory.env env_file removed (D-07) +- backend/Dockerfile and entrypoint.sh updated for new config paths +- .gitignore configured to track examples, ignore actual configs and secrets (D-08) +- DEPLOYMENT.md updated with YAML structure, Python scripts, systemd setup, troubleshooting (D-08) +- README.md updated with config setup and documentation cross-references (D-08) +- All documentation references config/ as single source of truth +- Docker deployment works with new config structure +- Clear migration path from inventory.env to new config/ structure documented + + + +After completion, create `.planning/phases/07-config-consolidation/07-04-SUMMARY.md` + diff --git a/.planning/phases/07-config-consolidation/07-04-SUMMARY.md b/.planning/phases/07-config-consolidation/07-04-SUMMARY.md new file mode 100644 index 00000000..f489d846 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-04-SUMMARY.md @@ -0,0 +1,44 @@ +# Phase 7 Wave 3 Summary: Docker Integration & Documentation (07-04) + +**Completed:** 2026-04-23 +**Status:** [COMPLETED] +**Commits:** 9f267a53, 5b3a23f9, 6b7becfe, 0c6f571a, 22343941, 01e30ba7 + +--- + +## Accomplishments + +1. **docker-compose.yml Updated (D-07)** + - Removed `inventory.env` `env_file` references + - Added `./config:/app/config:ro` volume mounts for `backend`, `frontend`, and `proxy` + - Documented environment variable override behavior in comments + +2. **Dockerfile and entrypoint.sh Updated** + - Backend `Dockerfile` and `entrypoint.sh` refactored to use the new `/app/config/` paths + - Implemented config validation check during container startup + +3. **.gitignore Rules Finalized (D-08)** + - Marked `inventory.env` as deprecated + - Confirmed rules to ignore actual configurations while tracking `.example` schema files + +4. **Comprehensive Documentation (D-08)** + - **DEPLOYMENT.md:** Completely rewritten to reflect the new YAML configuration system and Python-based tooling + - **README.md:** Updated Quick Start and onboarding with the new configuration steps + - Cross-referenced all documents for consistent developer experience + +--- + +## Technical Details + +- **Single Source of Truth:** `config/` folder is now established as the central point for all configuration. +- **Security:** `secrets.yaml` is strictly ignored by git, and the Docker volume is mounted as read-only. +- **Deprecation:** All references to `inventory.env` in the deployment infrastructure have been removed. + +--- + +## Verification + +- [x] `docker compose config` passes with valid YAML syntax +- [x] `.gitignore` rules correctly protect sensitive configuration files +- [x] `DEPLOYMENT.md` and `README.md` provide clear, updated instructions +- [x] All deployment paths integrated with the new YAML structure diff --git a/.planning/phases/07-config-consolidation/07-CONTEXT.md b/.planning/phases/07-config-consolidation/07-CONTEXT.md new file mode 100644 index 00000000..f95da1b9 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-CONTEXT.md @@ -0,0 +1,174 @@ +# Phase 7: Config Consolidation - Context + +**Gathered:** 2026-04-23 +**Status:** Ready for planning +**Source:** User Requirements + +--- + +## Phase Boundary + +Consolidate all application configuration files into a centralized `config/` folder in the project root. This includes backend configurations, frontend configurations, deployment settings, and network configurations. Update all deployment scripts, application startup procedures, and backend/frontend code to load configurations from this centralized location. + +**Scope:** +- Create and establish `config/` folder as the single source of truth for all application configuration +- Migrate existing configuration files (inventory.env and variants) to config folder with meaningful names +- Update all scripts (deploy.sh, run_standalone.sh, etc.) to reference the new config location +- Refactor backend config_loader.py and config_manager.py to read from config folder +- Update frontend environment loading if applicable +- Verify and clean up root directory scripts that are no longer needed +- Ensure Docker deployment, standalone deployment, and development all work correctly with new structure + +**Deliverables:** +- `config/` folder with structured configuration files +- Updated backend configuration loading mechanism +- Updated deployment scripts +- Updated startup procedures (Docker and standalone) +- Documentation of configuration structure in README/DEPLOYMENT.md + +--- + +## Implementation Decisions + +### D-01: Configuration File Format +- **Standardize on YAML format** for all config files (backend.yaml, frontend.yaml, network.yaml, docker.yaml) +- All config files in `config/` folder will be YAML format +- Backend code updated to use PyYAML parser +- Rationale: YAML provides better structure for complex configs, easier validation, clearer schema + +### D-02: Secrets Management (Separate File) +- Create dedicated `config/secrets.yaml` file for sensitive values (API keys, JWT secrets, database passwords) +- Add `config/secrets.yaml` to `.gitignore` with strict exclusion +- Commit `config/secrets.yaml.example` with placeholder values and clear format requirements +- Include strong documentation in config/README.md explaining each secret, where to obtain it, format requirements +- Rationale: Clear separation of concerns between configuration and secrets, guides developers on required values + +### D-03: Config File Examples +- Commit `.example` files for ALL config files: `backend.yaml.example`, `frontend.yaml.example`, `network.yaml.example`, `docker.yaml.example` +- Developers copy examples to non-example versions locally and fill in values +- Example files show structure, defaults, and all available options +- Rationale: Clear onboarding path, version control of config schema, consistency guarantees + +### D-04: Backward Compatibility - Immediate Deprecation +- **NO fallback to `inventory.env`** - immediate deprecation after Phase 7 completes +- All deployments must migrate to new `config/` structure during this phase +- Remove all code paths that read from root-level `inventory.env` +- Rationale: Clean break avoids ongoing dual-path support complexity + +### D-05: Deployment Scripts - Convert Bash to Python +- Convert all necessary bash deployment scripts to Python with identical functionality +- Before conversion: **Audit all scripts to identify redundant/mergeable ones** +- Critical scripts to convert: `deploy.sh`, `run_standalone.sh`, `install_service.sh`, `export_prod.sh` +- Evaluate `__push_ALL_to_remote.sh` for necessity/consolidation +- All Python scripts will parse YAML config files +- Rationale: Consistent tooling across infrastructure, easier YAML parsing, reduced bash complexity + +### D-06: Backend Config Loading +- Update `backend/config_loader.py` to parse YAML files +- Load order: System environment variables > `config/backend.yaml` > defaults in code +- Remove any fallback to `inventory.env` (Phase 7 end → fully deprecated) +- Log which config source is being used for debugging + +### D-07: Docker & Docker Compose +- Docker Compose updated to reference `config/docker.yaml` +- Backend Dockerfile and frontend Dockerfile updated to source from new config structure +- Environment variable injection mechanism preserved (takes precedence over YAML files) + +### D-08: Documentation & Git Structure +- Update DEPLOYMENT.md with new YAML config structure, required secrets, and format specifications +- Update README.md with configuration setup and onboarding instructions +- Add comprehensive `config/README.md` explaining all YAML files, required variables, examples, secrets setup +- Update .gitignore: ignore `config/*.yaml` (except examples), track `config/*.yaml.example` + +--- + +## Specific Ideas + +1. **YAML Config Files to Create:** + - `config/backend.yaml` — Backend-specific variables (database, AI keys, auth settings, logging) + - `config/backend.yaml.example` — Template showing all available options + - `config/frontend.yaml` — Frontend-specific variables (API endpoints, feature flags, service worker settings) + - `config/frontend.yaml.example` — Frontend config template + - `config/network.yaml` — Network/deployment variables (ports, SSL, server IPs, CORS settings) + - `config/network.yaml.example` — Network config template + - `config/docker.yaml` — Docker-specific overrides (for docker-compose.yml) + - `config/docker.yaml.example` — Docker config template + - `config/secrets.yaml` — Sensitive values (git-ignored) + - `config/secrets.yaml.example` — Secrets template with placeholders + - `config/README.md` — Comprehensive documentation of all YAML files, structure, required values + +2. **Python Scripts to Create (replacing bash):** + - `scripts/deploy.py` — Docker deployment with YAML config parsing (replaces deploy.sh) + - `scripts/run_standalone.py` — Standalone mode launcher (replaces run_standalone.sh) + - `scripts/export_prod.py` — Production export/backup functionality + - `scripts/install_service.py` — Systemd service installation (replaces install_service.sh) + - Audit `__push_ALL_to_remote.sh` - determine if needed or consolidate into another script + - All scripts will use PyYAML for config file parsing + +3. **Backend Changes:** + - `backend/config_loader.py` — Update to parse YAML files (backend.yaml + secrets.yaml) + - Load order: System env vars > config/backend.yaml > config/secrets.yaml > defaults in code + - Remove all code paths for reading inventory.env + - Implement environment variable override mechanism (system env vars take precedence) + - `backend/config_manager.py` — Update to read/write YAML (if config updates are needed at runtime) + - `backend/entrypoint.sh` — Reference new config paths in container + +4. **Testing Requirements:** + - Docker deployment with YAML config structure + - Standalone Python launcher with YAML config parsing + - Environment variable override behavior with YAML configs + - Secrets file permissions and git-ignore verification + - All deployment paths tested end-to-end with new Python scripts + - Confirm old inventory.env paths are NOT accessible (no fallback) + +--- + +## Claude's Discretion + +- Structure and organization of Python scripts in `scripts/` folder +- YAML validation schema and enforcement approach (basic vs. strict validation) +- Logging verbosity and format in Python deployment scripts + +--- + +## Canonical References + +**Downstream agents MUST read these before planning or implementing.** + +### Configuration & Deployment +- `DEPLOYMENT.md` — Current deployment procedures (to be updated with YAML structure) +- `README.md` — Project setup instructions (to be updated with config onboarding) +- `PROJECT_ARCHITECTURE.md` — Technical stack and component overview + +### Backend Configuration Loading +- `backend/config_loader.py` — Current config loading implementation (will be refactored for YAML) +- `backend/config_manager.py` — Current config management (will be updated) + +### Deployment Scripts (to be rewritten in Python) +- `deploy.sh` — Docker deployment script +- `run_standalone.sh` — Standalone mode launcher +- `export_prod.sh` — Production export +- `install_service.sh` — Systemd service setup +- `__push_ALL_to_remote.sh` — Remote push utility (to be audited for necessity) + +### Infrastructure Files +- `docker-compose.yml` — Docker composition (to be updated to reference config/docker.yaml) +- `backend/Dockerfile` — Backend container image (to be updated for YAML config paths) +- `frontend/Dockerfile` — Frontend container image (if applicable) +- `inventory.env` — Current config location (will be deprecated after Phase 7) + +--- + +## Deferred Ideas + +- Dynamic config hot-reload without restart (future optimization — Phase N) +- Config validation framework with JSON Schema (future enhancement — Phase N) +- Encrypted sensitive values in config files via KMS or Vault (future security enhancement — Phase N+1) +- Web UI for configuration management (future feature — Phase N+2) +- Config versioning and rollback mechanism (future ops enhancement) + +--- + +*Phase: 7-config-consolidation* +*Context gathered: 2026-04-23 via structured discussion* +*Status: Ready for detailed planning with locked decisions* diff --git a/.planning/phases/07-config-consolidation/07-DISCUSSION-LOG.md b/.planning/phases/07-config-consolidation/07-DISCUSSION-LOG.md new file mode 100644 index 00000000..9ca16f01 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-DISCUSSION-LOG.md @@ -0,0 +1,158 @@ +# Phase 7: Config Consolidation - Discussion Log + +> **Audit trail only.** Do not use as input to planning, research, or execution agents. +> Decisions are captured in CONTEXT.md — this log preserves the alternatives considered. + +**Date:** 2026-04-23 +**Phase:** 7-config-consolidation +**Areas discussed:** Secrets Management, Config File Examples, Backward Compatibility Timeline, Config File Format & Scripts + +--- + +## 1. Secrets & Sensitive Values + +**Question:** How should sensitive values (API keys, JWT secrets, database passwords) be managed in the config structure? + +| Option | Description | Selected | +|--------|-------------|----------| +| Store in config/backend.env (git-ignored) | Developers create their own local config/backend.env with secrets. File is in .gitignore so secrets never reach git. Example files committed separately for reference. | | +| Environment variables only | No sensitive values in config files. Deploy processes inject secrets via OS environment variables (Docker secrets, systemd, k8s secrets). Config files contain only non-sensitive settings. | | +| Separate secrets file | Create config/secrets.env (separate from backend.env) with stricter .gitignore rules. Non-sensitive backend config still in backend.env. Clearer separation of concerns. | ✓ | +| External secrets manager reference | Config files contain references/URLs to external secret storage (HashiCorp Vault, AWS Secrets Manager, etc.). Application loads secrets at runtime from external source. | | + +**User's choice:** Separate secrets file +**Notes:** Clear separation of concerns. Dedicated file for sensitive values with strict handling. + +--- + +## 1b. Secrets Template Approach + +**Question:** Should config/secrets.env.example be committed to git to guide developers on required secrets? + +| Option | Description | Selected | +|--------|-------------|----------| +| Yes, commit example with placeholder values | config/secrets.env.example in git (not git-ignored) shows all required secrets and example formats. Developers copy it to config/secrets.env and fill in real values locally. | | +| No, document in README only | No example file in git. Required secrets documented in config/README.md and DEPLOYMENT.md. Reduces git noise, relies on good documentation. | | +| Both — example + strong documentation | Commit config/secrets.env.example AND provide detailed docs explaining each secret, where to get it, format requirements. Best for onboarding but more maintenance. | ✓ | + +**User's choice:** Both — example + strong documentation +**Notes:** Provides clear onboarding path for developers while maintaining comprehensive documentation. + +--- + +## 2. Config File Format & Examples + +**Question:** How should non-sensitive config file examples be managed? + +| Option | Description | Selected | +|--------|-------------|----------| +| Commit all .example files | All config files have .example variants in git. Developers copy them locally. Easy onboarding, version control of all config schemas. | ✓ | +| Only document in config/README.md | No .example files. config/README.md lists all variables, defaults, and descriptions. Cleaner git repo, single source of truth is docs. | | +| Template files (versioned, not copyrighted) | Commit minimal template files with just variable names (no values). Like .example but more concise. Developers fill in values. Balance between clarity and repo cleanliness. | | +| Docker compose example in separate file | Only docker.env.example committed (since it's tied to docker-compose.yml). Backend/frontend examples in docs only. Minimal git footprint for typical deployments. | | + +**User's choice:** Commit all .example files +**Notes:** Developer-friendly approach. Clear templates to copy and fill in. + +--- + +## 3. Backward Compatibility Timeline + +**Question:** How long should the system support loading from old inventory.env as fallback? + +| Option | Description | Selected | +|--------|-------------|----------| +| This phase only (immediate deprecation) | After Phase 7 completes, inventory.env support is removed entirely. Requires all deployments to migrate immediately. Cleanest going forward, but higher migration burden. | ✓ | +| 2-3 releases (gradual sunset) | Support old paths for next 2-3 releases. Deprecated in logging. Then removed. Gives production deployments time to migrate gracefully. | | +| Keep indefinitely for backward compatibility | Never remove old paths. Scripts support both config/ and inventory.env forever. Highest compatibility, but code complexity persists. | | + +**User's choice:** This phase only (immediate deprecation) +**Notes:** Clean break. All deployments must migrate during Phase 7. No ongoing dual-path complexity. + +--- + +## 4. Config File Format & Deployment Scripts + +**Question:** What file format should the consolidated config files use? + +| Option | Description | Selected | +|--------|-------------|----------| +| Keep .env format (minimal change) | Stick with KEY=VALUE .env files already used by python-dotenv. Least disruptive. Backend and deployment scripts already know how to read this. | | +| Standardize on .yaml (structured but verbose setup) | Migrate to YAML format. More structured, easier to validate, better for complex configs. Requires adding pyyaml dependency, updating all loaders. | ✓ | +| Use .json (most rigid, easiest parsing) | JSON format for all configs. Very structured, standard parsing libraries, but less friendly for manual editing and comments. | | +| Multi-format support (backend handles both) | Support both .env AND .yaml/.json. Loader auto-detects format. Flexibility, but adds parsing complexity and documentation burden. | | + +**User's choice:** Standardize on .yaml +**Notes:** More structured format. Better for complex configurations and validation. + +--- + +## 4b. YAML Configuration in Deployment Scripts + +**Question:** For YAML-based config, how should deployment scripts (bash) access configuration? + +| Option | Description | Selected | +|--------|-------------|----------| +| Scripts read .env files, backend reads YAML — dual formats | Keep scripts using .env (familiar bash tooling). Backend uses YAML. Translation/sync between the two. More maintenance, but each tool uses its native format. | | +| Export YAML to env vars, scripts use vars | Config loading converts YAML to environment variables before scripts run. Scripts source env vars. Cleaner separation, requires loader to run first. | | +| Convert to .yaml everywhere, write yaml parser for scripts | Everything uses YAML. Write or use lightweight YAML parser in bash (yq). Consistent format end-to-end, adds bash dependency. | | +| Keep .env for network/docker, YAML for backend only | config/network.env and config/docker.env stay as .env (for scripts). Only config/backend.yaml uses YAML. Hybrid approach based on where each is used. | | + +**User's choice (custom):** Convert all config to YAML files, and convert all bash deployment scripts to Python scripts +**Notes:** Comprehensive modernization. Replace bash scripts with Python equivalents that natively parse YAML. More significant scope, but cleaner end-to-end architecture. + +--- + +## 4c. Script Conversion Scope + +**Question:** Is script conversion (bash → Python) part of Phase 7 scope, or a follow-up phase? + +| Option | Description | Selected | +|--------|-------------|----------| +| Include in Phase 7 | Rewrite all deployment/utility scripts in Python as part of this phase. Phase 7 scope expands significantly but delivers complete Python-based stack. | ✓ | +| Phase 7: YAML config only, follow-up phase for scripts | Phase 7 focuses on YAML config consolidation. Bash scripts adapted to read YAML (via yq or similar). Script conversion deferred to a future phase. | | +| Hybrid approach this phase | Phase 7: Convert the critical deployment scripts (deploy.sh, run_standalone.sh) to Python. Defer non-critical scripts (export_prod.sh, __push_ALL_to_remote.sh) to future phase. | | + +**User's choice:** Include in Phase 7 +**Notes:** Comprehensive Phase 7 scope. All necessary Python scripts completed in this phase. + +--- + +## 4d. Script Priority & Consolidation + +**Question:** Which scripts are critical for Phase 7 conversion? Which can be deferred? + +**User's choice (custom):** Audit all bash scripts to identify redundant/mergeable ones. Convert all needed scripts to Python. +**Notes:** Consolidation-focused approach. Before converting, identify opportunities to merge redundant functionality and eliminate unnecessary scripts. Then convert only what's essential. + +--- + +## Summary of Locked Decisions + +- **D-01:** YAML format for all config files (backend.yaml, frontend.yaml, network.yaml, docker.yaml) +- **D-02:** Separate secrets.yaml file (git-ignored) + secrets.yaml.example (committed) +- **D-03:** Commit all .example files for config schema reference +- **D-04:** Immediate deprecation of inventory.env (no fallback after Phase 7) +- **D-05:** Convert all necessary bash deployment scripts to Python +- **D-06:** Audit scripts first to consolidate redundancy before conversion + +--- + +## Claude's Discretion + +Areas where the user deferred to Claude's judgment: +- Structure and organization of Python scripts in `scripts/` folder +- YAML validation schema and enforcement approach +- Logging verbosity and format in Python deployment scripts + +--- + +## Deferred Ideas + +(None mentioned during discussion) + +--- + +*Discussion conducted: 2026-04-23* +*Format: Structured Q&A with alternatives considered* +*Outcome: All gray areas resolved; ready for detailed planning* diff --git a/.planning/phases/07-config-consolidation/07-PLAN.md b/.planning/phases/07-config-consolidation/07-PLAN.md new file mode 100644 index 00000000..56d3f9cd --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-PLAN.md @@ -0,0 +1,502 @@ +--- +wave: 1 +depends_on: [] +files_modified: [ + "config/backend.env", + "config/frontend.env", + "config/network.env", + "config/docker.env", + "config/README.md", + "backend/config_loader.py", + "backend/config_manager.py", + "backend/entrypoint.sh", + "deploy.sh", + "run_standalone.sh", + "export_prod.sh", + "install_service.sh", + "docker-compose.yml", + "DEPLOYMENT.md", + "README.md" +] +autonomous: true +--- + +# Phase 7: Config Consolidation - Implementation Plan + +**Objective:** Establish a centralized config/ folder structure, migrate all configurations from root level to config/, and update all scripts and code to use the new structure while maintaining backward compatibility. + +**Success Criteria:** +- Config folder exists with all required configuration files +- All scripts reference config folder instead of root-level env files +- Docker deployment works with new config structure +- Standalone deployment works with new config structure +- Backward compatibility: old inventory.env still loads if needed +- All documentation updated +- Root directory cleaned of unnecessary files + +--- + +## Task 1: Create Config Folder Structure + +**Read First:** +- Current project root layout (understand what we're migrating from) +- Current inventory.env and variants +- PROJECT_ARCHITECTURE.md (reference tech stack and requirements) + +**Action:** +1. Create `config/` folder in project root: `mkdir -p config` +2. Create `config/README.md` with documentation of all config files and their purposes +3. Ensure config/ is tracked in git (add to .gitignore if needed, or ensure it's not in .gitignore) + +**Acceptance Criteria:** +- `config/` directory exists in project root +- `config/README.md` exists and documents the purpose of each config file +- `config/` appears in git status (is tracked) + +--- + +## Task 2: Create backend.env Configuration File + +**Read First:** +- Current `inventory.env` content and structure +- `backend/config_loader.py` to understand what variables are expected +- `backend/config_manager.py` to understand all used environment variables +- `backend/main.py` to see what environment variables are loaded + +**Action:** +1. Read existing `inventory.env` file +2. Extract backend-specific environment variables (database, AI keys, auth settings, JWT secrets) +3. Create `config/backend.env` with all backend-specific variables from inventory.env +4. Include meaningful comments explaining each variable +5. Use same values as inventory.env to maintain current functionality +6. Ensure format matches python-dotenv expectations + +**Acceptance Criteria:** +- `config/backend.env` exists and contains all backend-specific variables +- File format is valid for python-dotenv (KEY=VALUE format with comments) +- Contains at least: JWT_SECRET_KEY, GEMINI_API_KEY, LDAP settings, database config +- All values match original inventory.env + +--- + +## Task 3: Create network.env Configuration File + +**Read First:** +- Current `inventory.env` file +- `run_standalone.sh` to see what network variables it uses +- `deploy.sh` to see what network variables it references +- `docker-compose.yml` to understand port and network configuration + +**Action:** +1. Extract network/deployment-specific variables from inventory.env (ports, server IPs, SSL config, CORS settings) +2. Create `config/network.env` with these variables +3. Include meaningful comments for each variable +4. Ensure variables match what deploy.sh and run_standalone.sh expect +5. Include default values for development + +**Acceptance Criteria:** +- `config/network.env` exists with network-specific variables +- Contains at least: BACKEND_PORT, BACKEND_SSL_PORT, FRONTEND_PORT, FRONTEND_SSL_PORT, SERVER_IP, SSL_ENABLED +- All values match original inventory.env +- Format is bash-sourceable (KEY=VALUE) + +--- + +## Task 4: Create docker.env Configuration File + +**Read First:** +- Current `docker-compose.yml` file +- `inventory.env` file for current values +- Dockerfile files (backend/Dockerfile, frontend/Dockerfile) + +**Action:** +1. Create `config/docker.env` with variables specifically for docker-compose +2. Include variables that docker-compose.yml references in its environment sections +3. These may overlap with network.env but are docker-compose specific +4. Add comments explaining docker-specific context +5. Include any build arguments and docker-specific settings + +**Acceptance Criteria:** +- `config/docker.env` exists +- Contains Docker-specific environment variables +- Format is valid for docker-compose (KEY=VALUE) +- All required docker-compose.yml variables are present + +--- + +## Task 5: Create frontend.env Configuration File + +**Read First:** +- `frontend/package.json` to see if environment variables are used +- `frontend/next.config.mjs` to understand what env vars are needed +- `frontend/entrypoint.sh` to see how frontend loads configuration +- Any frontend environment setup in current codebase + +**Action:** +1. Create `config/frontend.env` with frontend-specific variables +2. Include API endpoint configuration, feature flags, service worker settings, etc. +3. Add comments explaining each variable's purpose +4. If frontend doesn't currently use env files, create minimal defaults for future use +5. Ensure Next.js compatible format + +**Acceptance Criteria:** +- `config/frontend.env` exists +- Contains frontend-specific variables (API_BASE_URL, feature flags, etc.) +- Format is valid for frontend configuration +- Documented with clear comments + +--- + +## Task 6: Update backend/config_loader.py + +**Read First:** +- Current `backend/config_loader.py` implementation +- Current `backend/config_manager.py` implementation +- `backend/main.py` to see how config_loader is used +- Current load order and fallback logic + +**Action:** +1. Update `config_loader.py` to change config loading order: + - Priority 1: System environment variables (already set by Docker/deployment) + - Priority 2: `config/backend.env` (new centralized location) + - Priority 3: `inventory.env` (backward compatibility) + - Priority 4: `backend/.env` (legacy location) + - Priority 5: Hardcoded defaults +2. Update file paths to look in config/ folder first +3. Update log messages to indicate which config file is being loaded +4. Ensure backward compatibility: if config/backend.env doesn't exist, fall back to inventory.env +5. Test that load_dotenv() calls work correctly with new paths + +**Acceptance Criteria:** +- `config_loader.py` contains logic to load from `config/backend.env` first +- Falls back to `inventory.env` if `config/backend.env` not found +- Load order matches: env vars > config/backend.env > inventory.env > backend/.env +- Log messages indicate which config file was loaded +- All environment variables are still accessible to rest of backend +- File contains comment explaining the new config structure + +--- + +## Task 7: Update backend/config_manager.py + +**Read First:** +- Current `backend/config_manager.py` implementation +- Look for any file paths that hardcode inventory.env +- Understand how config updates are written back to disk + +**Action:** +1. Update file paths to use `config/backend.env` instead of root `inventory.env` +2. Ensure write operations go to `config/backend.env` +3. Update comments to reflect new path +4. Verify get_config_path() returns path to config/backend.env +5. Ensure file operations handle non-existent config/ folder gracefully + +**Acceptance Criteria:** +- `config_manager.py` references `config/backend.env` instead of `inventory.env` +- get_config_path() returns correct path to config/backend.env +- Config updates are written to config/backend.env +- Error handling works if config/ folder doesn't exist + +--- + +## Task 8: Update backend/entrypoint.sh + +**Read First:** +- Current `backend/entrypoint.sh` content +- How environment variables are sourced +- Docker ENTRYPOINT and CMD configuration + +**Action:** +1. Update entrypoint.sh to source from `config/backend.env` instead of root location +2. Update path references to point to /app/config/backend.env (inside Docker container) +3. Maintain backward compatibility: try config/backend.env first, fall back to inventory.env +4. Add logging to show which config was loaded +5. Ensure entrypoint handles missing config gracefully + +**Acceptance Criteria:** +- `entrypoint.sh` sources from `config/backend.env` +- Falls back to `inventory.env` if config/backend.env not found +- Inside Docker, path is /app/config/backend.env +- Script logs which config file was loaded +- Script doesn't fail if config files don't exist + +--- + +## Task 9: Update deploy.sh Script + +**Read First:** +- Current `deploy.sh` implementation +- How environment variables are currently sourced +- Lines that reference inventory.env + +**Action:** +1. Update script to load from `config/network.env` and `config/docker.env` instead of `inventory.env` +2. Change: `export $(grep -v '^#' "inventory.env" | xargs)` to `export $(grep -v '^#' "config/network.env" | xargs)` +3. Add fallback: if config/network.env doesn't exist, use inventory.env +4. Update docker-compose calls to use config/docker.env via environment variable sourcing +5. Add validation: check that config/ folder exists before sourcing +6. Add helpful error message if config files are missing + +**Acceptance Criteria:** +- `deploy.sh` sources from `config/network.env` instead of `inventory.env` +- Includes fallback to `inventory.env` if config/network.env not found +- Validates config folder exists with helpful error message +- Docker-compose gets correct environment variables from config files +- Script still functions with new structure + +--- + +## Task 10: Update run_standalone.sh Script + +**Read First:** +- Current `run_standalone.sh` implementation +- Lines that reference CONFIG_PATH or inventory.env +- How network configuration is loaded +- Backend and frontend startup logic + +**Action:** +1. Update CONFIG_PATH to point to `config/network.env` +2. Change line: `CONFIG_PATH="$(cd "$(dirname "$0")" && pwd)/inventory.env"` to reference config/network.env +3. Add fallback: if config/network.env not found, try inventory.env +4. Update comment to reflect new config location +5. Ensure backend environment loading also uses config/backend.env (via PYTHONPATH or direct sourcing) +6. Add logging showing which config files are being used + +**Acceptance Criteria:** +- `run_standalone.sh` loads from `config/network.env` +- Falls back to `inventory.env` if config files not found +- Backend loads from `config/backend.env` via config_loader.py +- Script logs which config files are loaded +- Standalone mode works with new config structure + +--- + +## Task 11: Update export_prod.sh Script + +**Read First:** +- Current `export_prod.sh` implementation +- How it uses environment variables +- What configuration it needs + +**Action:** +1. Identify all environment variables used by export_prod.sh +2. Update script to source from `config/backend.env` and `config/network.env` +3. Add fallback to inventory.env for backward compatibility +4. Update comments to reflect new config loading +5. Ensure export functionality works with new config structure + +**Acceptance Criteria:** +- `export_prod.sh` sources correct config files from config/ folder +- Falls back to `inventory.env` if needed +- All required environment variables are available to the script +- Script functions correctly with new configuration structure + +--- + +## Task 12: Update install_service.sh Script + +**Read First:** +- Current `install_service.sh` implementation +- How it references configuration +- What paths it sets in systemd service files + +**Action:** +1. Update script to reference config/ folder in environment file paths +2. Update systemd service file generation to point to config/backend.env +3. If service uses EnvironmentFile, ensure it points to config/backend.env or both config/backend.env and config/network.env +4. Add validation that config/ folder exists +5. Update comments to explain new config structure + +**Acceptance Criteria:** +- `install_service.sh` references config/backend.env in service configuration +- Systemd service file has correct EnvironmentFile paths +- Service can load all required environment variables +- Script validates config folder exists + +--- + +## Task 13: Update docker-compose.yml + +**Read First:** +- Current `docker-compose.yml` file +- All env_file and environment references +- How inventory.env is currently used + +**Action:** +1. Update env_file directives to reference config/docker.env instead of inventory.env +2. Update any hardcoded environment variable references to use config/ equivalents +3. For services using env_file: `env_file: config/docker.env` +4. Ensure Docker build arguments reference correct config location +5. Add validation or comment explaining config folder requirement +6. Test that docker-compose can still read all needed variables + +**Acceptance Criteria:** +- `docker-compose.yml` references `config/docker.env` in env_file +- All services get correct environment variables +- Docker-compose validates successfully +- Docker services can access all required configuration + +--- + +## Task 14: Update DEPLOYMENT.md + +**Read First:** +- Current `DEPLOYMENT.md` content +- Current documentation structure +- Instructions for setting up configuration + +**Action:** +1. Add new section explaining config folder structure and purpose +2. Document each config file (backend.env, frontend.env, network.env, docker.env) +3. Explain which variables go in each config file +4. Update deployment instructions to reference config/ instead of inventory.env +5. Document backward compatibility behavior (still reads inventory.env if config/ not found) +6. Add troubleshooting section for common config issues +7. Update any examples to use new config paths + +**Acceptance Criteria:** +- DEPLOYMENT.md has section explaining config folder structure +- All four config files are documented with their purpose +- Deployment instructions reference config/ folder +- Backward compatibility is explained +- Examples use new config paths + +--- + +## Task 15: Update README.md + +**Read First:** +- Current `README.md` content +- Setup/quickstart section + +**Action:** +1. Add or update configuration setup section +2. Explain that config/ folder is where all configuration lives +3. For quick start, show example of creating config/backend.env +4. Link to DEPLOYMENT.md for detailed configuration reference +5. Keep it concise - detailed docs go in DEPLOYMENT.md + +**Acceptance Criteria:** +- README.md mentions config/ folder +- Setup section references config/ not inventory.env +- Configuration setup is clear for new users + +--- + +## Task 16: Verify Backward Compatibility and Test All Deployment Methods + +**Read First:** +- Current DEPLOYMENT.md +- All scripts that were updated +- Docker Compose setup +- Standalone setup requirements + +**Action:** +1. Ensure all scripts still function if inventory.env exists but config/ doesn't (backward compatibility) +2. Test Docker deployment: `./deploy.sh production` + - Verify: services start, environment variables are loaded correctly + - Check logs: which config file was loaded +3. Test Standalone deployment: `./run_standalone.sh` + - Verify: backend and frontend start correctly + - Verify: all environment variables available + - Verify: correct ports from config/network.env +4. Test environment variable override: set env var on command line, verify it takes precedence +5. Verify config/backend.env is loaded by backend: grep logs for config path message + +**Acceptance Criteria:** +- Docker deployment works with config/ structure +- Standalone deployment works with config/ structure +- Backward compatibility works: scripts still read inventory.env if config/ doesn't exist +- Environment variable precedence works: system env > config files +- All tests pass +- Logs show correct config file was loaded + +--- + +## Task 17: Audit and Clean Up Root Directory + +**Read First:** +- All files in project root directory +- Understand what each script does +- Current .gitignore + +**Action:** +1. Review all *.sh scripts in root: deploy.sh, run_standalone.sh, export_prod.sh, install_service.sh, __push_ALL_to_remote.sh +2. For each script, determine: + - Is it still used? (check git history, comments, references) + - Can it be archived/removed? + - Does it need updating for config folder? +3. For scripts that are truly obsolete: + - Move to dev_docs/ARCHIVE_LOGS.md or note in comment + - Do NOT delete without understanding purpose +4. Review inventory.env* files: + - Are inventory.env.example and inventory.env.template still needed? + - Update .gitignore to exclude inventory.env (old) but track config/ structure +5. Create summary of what's obsolete and what's actively used + +**Acceptance Criteria:** +- Review completed for all root-level scripts +- Documented which scripts are obsolete (if any) +- Confirmed which scripts are still actively used +- .gitignore updated if needed +- Summary created of root directory cleanup + +--- + +## Task 18: Final Validation and Documentation + +**Read First:** +- Updated DEPLOYMENT.md +- Updated README.md +- Updated scripts +- config/README.md + +**Action:** +1. Create comprehensive test checklist: + - Docker deployment test + - Standalone deployment test + - Environment variable override test + - Backward compatibility test + - Config file format validation +2. Run all tests and document results +3. Verify all config files exist and are properly formatted +4. Verify all scripts source from correct locations +5. Verify backend loads from config/backend.env +6. Create deployment.md section summarizing changes +7. Add entry to dev_docs/PLAN.md documenting completion + +**Acceptance Criteria:** +- All deployment methods tested and working +- Config files exist and are properly formatted +- Documentation updated and clear +- No errors in script execution +- Environment variables load from config/ as expected +- Phase marked complete in documentation + +--- + +## Notes + +**Backward Compatibility:** +- All scripts include fallback logic: try config/ first, then fall back to inventory.env +- This allows gradual migration without breaking existing deployments +- Eventually, old inventory.env files can be deprecated after migration period + +**Configuration Priority (from highest to lowest):** +1. System environment variables (set by Docker, deployment platform) +2. config/backend.env (new centralized backend config) +3. inventory.env (legacy, for backward compatibility) +4. backend/.env (legacy backend-specific) +5. Hardcoded defaults in code + +**Docker Deployment Flow:** +1. docker-compose.yml loads config/docker.env +2. Services get environment variables from docker-compose.yml +3. Backend entrypoint sources config/backend.env for additional variables +4. System env vars override everything + +**Standalone Deployment Flow:** +1. run_standalone.sh sources config/network.env +2. Backend activation sources config/backend.env via config_loader.py +3. All environment variables available to both backend and frontend + diff --git a/DEPLOYMENT.md b/DEPLOYMENT.md index 050e5e10..067d4753 100644 --- a/DEPLOYMENT.md +++ b/DEPLOYMENT.md @@ -1,15 +1,15 @@ # TFM aInventory — Unified Deployment & Operations Guide **Audience**: System administrators, DevOps teams, Site managers -**Version**: 1.14.6 (Phase 6) -**Last Updated**: 2026-04-23 +**Version**: 1.15.0 (Phase 7 - Config Consolidation) +**Last Updated**: 2026-05-15 --- ## 1. Overview -TFM aInventory is a unified inventory management system supporting web administration, field scanning (QR/barcode), AI-powered label extraction, and offline sync. This guide provides instructions for both **Docker** and **Standalone** deployment modes. +TFM aInventory is a unified inventory management system supporting web administration, field scanning (QR/barcode), AI-powered label extraction, and offline sync. -Both modes share the same configuration file (`inventory.env`) and operational scripts. +[D-07] Since Phase 7, the application uses a consolidated configuration structure in the `config/` directory. The legacy `inventory.env` file is deprecated in favor of YAML-based configuration for better structure and validation. --- @@ -19,77 +19,120 @@ Both modes share the same configuration file (`inventory.env`) and operational s - **OS**: Ubuntu 22.04 LTS or similar Linux distribution - **RAM**: 2GB minimum (4GB recommended for production) - **Disk**: 10GB free space (50GB recommended for logs/backups) -- **Network**: Internet access (first-time setup), Ports 8916 (Backend) & 8917 (Frontend) available +- **Network**: Internet access (first-time setup), Ports 8000 (Backend) & 3000 (Frontend) available ### 2.2 Software Requirements - **Docker Mode**: Docker 24.0+ and Docker Compose 2.0+ - **Standalone Mode**: Python 3.12+, Node.js 20+, npm 10+ +- **All Modes**: Python 3.12+ (for deployment scripts) --- -## 3. Quick Start +## 3. Configuration -### 3.1 Step 1: Prepare Environment -```bash -git clone tfm-inventory -cd tfm-inventory -cp inventory.env.example inventory.env +[D-08] The `config/` directory is the single source of truth for all application settings. -# Generate a secure JWT secret -openssl rand -hex 32 # Copy this to JWT_SECRET_KEY in inventory.env +### 3.1 Configuration Files +| File | Description | +|------|-------------| +| `config/backend.yaml` | Backend API, database, and AI settings | +| `config/frontend.yaml` | Frontend UI and connection settings | +| `config/network.yaml` | Port assignments and SSL configuration | +| `config/docker.yaml` | Docker resource limits and volume drivers | +| `config/secrets.yaml` | Sensitive keys (API keys, JWT secrets) | -# Customize other settings (ports, AI keys, LDAP) -nano inventory.env -``` +### 3.2 Setup Configuration +1. **Clone and enter repository:** + ```bash + git clone tfm-inventory + cd tfm-inventory + ``` -### 3.2 Step 2: Deployment Mode +2. **Initialize config from examples:** + ```bash + # Copy all examples to actual config files + for f in config/*.yaml.example; do cp "$f" "${f%.example}"; done + ``` -#### Option A: Docker Deployment (Recommended for Production) -```bash -chmod +x deploy.sh -./deploy.sh production -``` -- **Access**: http://localhost:8917 (Frontend), http://localhost:8916/docs (API) -- **HTTPS**: https://localhost:8909 (via Caddy proxy) +3. **Customize your settings:** + - Edit `config/backend.yaml` for application behavior. + - Edit `config/network.yaml` for port assignments. + - Edit `config/secrets.yaml` with your API keys. -#### Option B: Standalone Deployment (Recommended for Development/Low-Resource) -```bash -chmod +x start_server.sh -./start_server.sh -``` -- **Access**: http://localhost:8917 (Frontend), http://localhost:8916 (API) +4. **Generate JWT Secret:** + ```bash + # Generate a 64-character hex secret + openssl rand -hex 32 + # Copy this value to jwt_secret_key in config/secrets.yaml + ``` + +[D-06] **Environment Variable Overrides**: System environment variables take precedence over YAML config values. This is useful for Docker overrides or CI/CD pipelines. --- -## 4. Configuration Reference (`inventory.env`) +## 4. Quick Start -| Category | Variable | Default | Description | -|----------|----------|---------|-------------| -| **Network** | `BACKEND_PORT` | 8916 | Port for FastAPI backend | -| | `FRONTEND_PORT` | 8917 | Port for Next.js frontend | -| **Security** | `JWT_SECRET_KEY` | - | **REQUIRED**: Generate with `openssl rand -hex 32` | -| | `LDAP_SERVER` | - | LDAP server for enterprise auth (Optional) | -| **AI** | `PRIMARY_AI_PROVIDER` | `gemini` | `gemini` or `claude` | -| | `GEMINI_API_KEY` | - | Required if using Gemini | -| | `CLAUDE_API_KEY` | - | Required if using Claude | -| **Data** | `DATA_DIR` | `./data` | Persistent data location | -| | `LOG_LEVEL` | `INFO` | `DEBUG`, `INFO`, `WARNING`, `ERROR` | -| **Backups** | `BACKUP_RETENTION_DAILY` | 30 | Daily backup retention (days) | +### 4.1 Option A: Docker Deployment (Recommended) +```bash +python3 scripts/deploy.py production +``` +- **Access**: http://localhost:3000 (Frontend), http://localhost:8000/docs (API) +- **HTTPS**: https://localhost:8919 (via Caddy proxy) + +### 4.2 Option B: Standalone Deployment +```bash +python3 scripts/run_standalone.py +``` +- **Access**: http://localhost:3000 (Frontend), http://localhost:8000 (API) --- -## 5. Operations & Health Monitoring +## 5. Deployment Modes -### 5.1 Health Checks -- **Docker**: `docker-compose ps` (All services should be `healthy`) -- **Standalone**: `ps aux | grep -E "(uvicorn|next)"` -- **API Health**: `curl http://localhost:8916/health` +### 5.1 Docker Deployment (scripts/deploy.py) +The `deploy.py` script manages the Docker lifecycle, including configuration validation and health checks. -### 5.2 Logging -- **Docker**: `docker-compose logs -f [service_name]` -- **Standalone**: `tail -f logs/backend.log` and `tail -f logs/frontend.log` +**Usage:** +```bash +python3 scripts/deploy.py [production|staging|development] [--rebuild] +``` -### 5.3 Automated Backups +- **Production**: Optimized images, resource limits enforced. +- **Staging**: Mirror of production for testing. +- **Development**: Hot-reloading enabled, debug logging. + +### 5.2 Standalone Deployment (scripts/run_standalone.py) +For environments without Docker, use the standalone runner. It manages both backend and frontend processes. + +**Usage:** +```bash +python3 scripts/run_standalone.py [--backend-only|--frontend-only] +``` + +### 5.3 Systemd Service Installation +To run aInventory as a background service on Linux: + +```bash +sudo python3 scripts/install_service.py [--user=www-data] +``` + +- **Start**: `sudo systemctl start ainventory` +- **Status**: `sudo systemctl status ainventory` +- **Logs**: `journalctl -u ainventory -f` + +--- + +## 6. Backup & Export + +### 6.1 Production Export (scripts/export_prod.py) +Create a production-ready bundle including data and sanitized configuration. + +```bash +python3 scripts/export_prod.py [--output=/path/to/backup.tar.gz] [--include-logs] +``` +*Note: actual secrets in `secrets.yaml` are excluded for security; config examples are included.* + +### 6.2 Automated Backups Automated backups are configured via cron: ```bash sudo bash config/backup-cron.sh @@ -100,30 +143,50 @@ sudo bash config/backup-cron.sh --- -## 6. Disaster Recovery & Troubleshooting +## 7. Operations & Health Monitoring -### 6.1 Restore Procedure -```bash -# Docker mode -./scripts/restore.sh backups/inventory-2026-04-23.tar.gz --validate +### 7.1 Health Checks +- **Docker**: `docker compose ps` (All services should be `running` and `healthy`) +- **API Health**: `curl http://localhost:8000/health` +- **Frontend Health**: `curl -f http://localhost:3000/` -# Standalone mode -./scripts/restore.sh backups/inventory-2026-04-23.tar.gz -``` - -### 6.2 Common Issues -- **Port Already in Use**: Check `lsof -i :8916` and kill or change port in `inventory.env`. -- **Database Locked**: Restart backend service. -- **HTTPS Warning**: Caddy uses self-signed certs for local HTTPS; click "Proceed anyway". -- **Out of Space**: Clean old backups in `./backups/`. +### 7.2 Logging +- **Docker**: `docker compose logs -f [backend|frontend|proxy]` +- **Standalone**: Check files in `./logs/` directory. --- -## 7. Performance & Scaling -- **Concurrent Users**: Optimized for ~5 concurrent users. -- **Item Capacity**: Handles 10K+ items on standard SSD hardware. -- **Optimization**: Use `LOG_LEVEL=WARNING` in production to reduce I/O. +## 8. Security + +- **secrets.yaml**: This file is excluded from Git via `.gitignore`. Never commit it. +- **JWT Secrets**: Always rotate `jwt_secret_key` before production deployment. +- **File Permissions**: The `deploy.py` and `install_service.py` scripts attempt to set restrictive permissions on config files. +- **Read-Only Mounts**: In Docker mode, the `config/` directory is mounted as read-only (`:ro`) to prevent the container from modifying its own configuration. --- -**Next Steps**: See `USER_GUIDE.md` for application usage or `PROJECT_ARCHITECTURE.md` for technical deep-dives. +## 9. Troubleshooting + +- **Missing Config**: Ensure you copied `.yaml.example` files to `.yaml`. +- **Invalid YAML**: Check your config files with a YAML validator. +- **Port Conflict**: Update `config/network.yaml` if ports 8000 or 3000 are in use. +- **Permission Denied**: Run scripts with `sudo` if they need to write to system paths (like systemd). +- **AI Failures**: Verify your API keys in `config/secrets.yaml` and check `backend.log`. + +--- + +## 10. Migration from inventory.env + +[D-04] To migrate from a legacy `inventory.env` file: + +1. Locate your old `inventory.env`. +2. Map the variables to the new YAML files: + - `BACKEND_PORT` -> `config/network.yaml` (`backend_port`) + - `JWT_SECRET_KEY` -> `config/secrets.yaml` (`jwt_secret_key`) + - `GEMINI_API_KEY` -> `config/secrets.yaml` (`gemini_api_key`) + - `DATA_DIR` -> `config/backend.yaml` (`application.data_dir`) +3. Delete the old `inventory.env` once migration is verified. + +--- + +**Next Steps**: See `config/README.md` for detailed configuration reference or `README.md` for general project overview. diff --git a/README.md b/README.md index 30c68af0..6fe37199 100644 --- a/README.md +++ b/README.md @@ -11,14 +11,21 @@ For production environments, Docker is the recommended deployment method: ```bash git clone tfm-inventory cd tfm-inventory -cp inventory.env.example inventory.env -# Edit inventory.env with your JWT_SECRET_KEY and AI keys -./deploy.sh production + +# [D-08] Configuration - Copy examples to actual config files +for f in config/*.yaml.example; do cp "$f" "${f%.example}"; done + +# Edit config/secrets.yaml with your JWT_SECRET_KEY and AI keys +nano config/secrets.yaml + +# Deploy using the new Python deployment script +python3 scripts/deploy.py production ``` -- **Frontend**: http://localhost:8917 (or https://localhost:8909 via proxy) -- **Backend API**: http://localhost:8916/docs +- **Frontend**: http://localhost:3000 (or https://localhost:8919 via proxy) +- **Backend API**: http://localhost:8000/docs +For detailed configuration reference, see **[config/README.md](config/README.md)**. For detailed deployment instructions (Docker vs Standalone), see **[DEPLOYMENT.md](DEPLOYMENT.md)**. --- @@ -28,6 +35,7 @@ For detailed deployment instructions (Docker vs Standalone), see **[DEPLOYMENT.m * **Frontend:** Next.js 15+ (React PWA) with responsive Tailwind CSS * **Database:** SQLite (SQLAlchemy) with Dexie.js (IndexedDB) for client-side sync. * **AI Engine:** Google Gemini (Primary) & Anthropic Claude (Fallback). +* **Configuration:** [D-07] Consolidated YAML-based config in `config/` directory. For more details on system logic, see **[PROJECT_ARCHITECTURE.md](PROJECT_ARCHITECTURE.md)**. @@ -48,5 +56,5 @@ To generate a clean production package: --- -**Last Updated**: 2026-04-23 -**Version**: 1.14.6 +**Last Updated**: 2026-05-15 +**Version**: 1.15.0 (Phase 7) diff --git a/backend/Dockerfile b/backend/Dockerfile index 4d9e9907..a2ba69c8 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,3 +1,7 @@ +# [D-07] Backend container - config/ folder mounted at /app/config (read-only) +# Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables +# Environment variables override YAML config per D-06 load order + FROM python:3.12-slim # Metadata labels diff --git a/backend/ai/claude.py b/backend/ai/claude.py index e82bd79c..5cd6db63 100644 --- a/backend/ai/claude.py +++ b/backend/ai/claude.py @@ -2,9 +2,11 @@ import os import anthropic import json import base64 +from ..config_loader import get_config def extract(image_bytes: bytes, prompt: str): - api_key = os.environ.get("CLAUDE_API_KEY") + config = get_config() + api_key = config.get("ai", {}).get("claude_api_key") if not api_key: return None diff --git a/backend/ai/gemini.py b/backend/ai/gemini.py index 1c5bb496..3c643704 100644 --- a/backend/ai/gemini.py +++ b/backend/ai/gemini.py @@ -5,6 +5,7 @@ import logging from PIL import Image from google import genai from google.genai import types +from ..config_loader import get_config log = logging.getLogger("ainventory") @@ -18,9 +19,10 @@ def get_best_models(): ] def extract(image_bytes: bytes, prompt: str): - api_key = os.environ.get("GEMINI_API_KEY") + config = get_config() + api_key = config.get("ai", {}).get("gemini_api_key") if not api_key: - log.error("CRITICAL: GEMINI_API_KEY is MISSING in environment!") + log.error("CRITICAL: gemini_api_key is MISSING in configuration!") return None # Log partial key for safety debug diff --git a/backend/ai_vision.py b/backend/ai_vision.py index f4a4f1f5..c1c0c738 100644 --- a/backend/ai_vision.py +++ b/backend/ai_vision.py @@ -1,6 +1,5 @@ import os import time -from dotenv import load_dotenv from . import models from .database import SessionLocal from .ai import gemini, claude diff --git a/backend/check_models.py b/backend/check_models.py index ab82b44b..8af3528c 100644 --- a/backend/check_models.py +++ b/backend/check_models.py @@ -1,12 +1,17 @@ import os import google.generativeai as genai -from dotenv import load_dotenv +try: + from backend.config_loader import get_config +except ImportError: + import sys + sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + from backend.config_loader import get_config -load_dotenv() -API_KEY = os.environ.get("GEMINI_API_KEY") +config = get_config() +API_KEY = config.get("ai", {}).get("gemini_api_key") if not API_KEY: - print("Error: GEMINI_API_KEY not found in .env") + print("Error: gemini_api_key not found in config/backend.yaml, config/secrets.yaml or GEMINI_API_KEY environment variable") exit(1) genai.configure(api_key=API_KEY) diff --git a/backend/config_loader.py b/backend/config_loader.py index c8e461a0..e01efffa 100644 --- a/backend/config_loader.py +++ b/backend/config_loader.py @@ -1,34 +1,232 @@ import os import logging -from dotenv import load_dotenv +import yaml log = logging.getLogger("ainventory") -def load_config(): - """ - Centralized environment loader for TFM aInventory. - Prioritizes existing environment variables (Docker), - then inventory.env at project root, then backend/.env. - """ +_config = {} + +class ConfigError(Exception): + """Raised when configuration is invalid or missing.""" + pass + +def _deep_merge(base, source): + """Recursively merge dictionaries.""" + for key, value in source.items(): + if isinstance(value, dict) and key in base and isinstance(base[key], dict): + _deep_merge(base[key], value) + else: + base[key] = value + +def _map_secrets(config, secrets): + """Map secrets from secrets.yaml (flattened) to the nested config structure.""" + mapping = { + "JWT_SECRET_KEY": ("auth", "jwt_secret_key"), + "GEMINI_API_KEY": ("ai", "gemini_api_key"), + "CLAUDE_API_KEY": ("ai", "claude_api_key"), + "DATABASE_PASSWORD": ("database", "password"), + "LDAP_PASSWORD": ("auth", "ldap_password") + } + for secret_key, config_path in mapping.items(): + if secret_key in secrets: + target = config + for p in config_path[:-1]: + if p not in target: + target[p] = {} + target = target[p] + target[config_path[-1]] = secrets[secret_key] + +def _to_bool(val): + """Convert string to boolean.""" + if isinstance(val, bool): + return val + return str(val).lower() in ("true", "1", "yes", "on") + +def _apply_env_overrides(config): + """Apply environment variable overrides (D-06 load order).""" + # Mapping of environment variables to config paths + # Format: ENV_VAR: (path, type_converter) + env_mapping = { + "BACKEND_DATABASE_SQLITE_PATH": (("database", "sqlite_path"), str), + "BACKEND_DATABASE_WAL_MODE": (("database", "wal_mode"), _to_bool), + "BACKEND_DATABASE_LOG_RETENTION_DAYS": (("database", "log_retention_days"), int), + "BACKEND_AI_PRIMARY_AI_PROVIDER": (("ai", "primary_ai_provider"), str), + "PRIMARY_AI_PROVIDER": (("ai", "primary_ai_provider"), str), + "BACKEND_AI_FALLBACK_PROVIDER": (("ai", "fallback_provider"), str), + "BACKEND_AI_GEMINI_API_KEY": (("ai", "gemini_api_key"), str), + "GEMINI_API_KEY": (("ai", "gemini_api_key"), str), + "BACKEND_AI_CLAUDE_API_KEY": (("ai", "claude_api_key"), str), + "CLAUDE_API_KEY": (("ai", "claude_api_key"), str), + "BACKEND_AUTH_JWT_SECRET_KEY": (("auth", "jwt_secret_key"), str), + "JWT_SECRET_KEY": (("auth", "jwt_secret_key"), str), + "BACKEND_AUTH_LDAP_SERVER": (("auth", "ldap_server"), str), + "BACKEND_AUTH_LDAP_BASE_DN": (("auth", "ldap_base_dn"), str), + "BACKEND_AUTH_PASSWORD_CACHE_PATH": (("auth", "password_cache_path"), str), + "BACKEND_LOGGING_LOG_LEVEL": (("logging", "log_level"), str), + "LOG_LEVEL": (("logging", "log_level"), str), + "BACKEND_LOGGING_LOG_ROTATION_SIZE_MB": (("logging", "log_rotation_size_mb"), int), + "BACKEND_LOGGING_LOG_ROTATION_COUNT": (("logging", "log_rotation_count"), int), + "BACKEND_APPLICATION_DATA_DIR": (("application", "data_dir"), str), + "DATA_DIR": (("application", "data_dir"), str), + "BACKEND_APPLICATION_LOGS_DIR": (("application", "logs_dir"), str), + "LOGS_DIR": (("application", "logs_dir"), str), + "BACKEND_APPLICATION_CORS_ORIGINS": (("application", "cors_origins"), str), + "EXTRA_ALLOWED_ORIGINS": (("application", "cors_origins"), str), + "ALLOWED_ORIGINS": (("application", "cors_origins"), str), + "SERVER_IP": (("application", "server_ip"), str), + "FRONTEND_PORT": (("application", "frontend_port"), str), + "FRONTEND_SSL_PORT": (("application", "frontend_ssl_port"), str), + "BACKEND_PORT": (("application", "backend_port"), str), + "BACKEND_SSL_PORT": (("application", "backend_ssl_port"), str), + } + + sensitive_vars = [ + "JWT_SECRET_KEY", "GEMINI_API_KEY", "CLAUDE_API_KEY", + "BACKEND_AUTH_JWT_SECRET_KEY", "BACKEND_AI_GEMINI_API_KEY", "BACKEND_AI_CLAUDE_API_KEY", + "LDAP_PASSWORD", "DATABASE_PASSWORD" + ] + + for env_var, (path, converter) in env_mapping.items(): + val = os.getenv(env_var) + if val is not None: + try: + converted_val = converter(val) + target = config + for p in path[:-1]: + if p not in target: + target[p] = {} + target = target[p] + + target[path[-1]] = converted_val + + # Mask sensitive values in logs + log_val = "********" if env_var in sensitive_vars else converted_val + log.info(f"ℹ️ Override {'.'.join(path)} from environment ({env_var}): {log_val}") + except Exception as e: + log.warning(f"⚠️ Failed to convert env var {env_var}='{val}': {e}") + +def load_config() -> dict: + """Load config from YAML files with env var overrides (D-06 load order).""" + global _config + # Base directory is backend/ base_dir = os.path.dirname(os.path.abspath(__file__)) # Project root is one level up project_root = os.path.dirname(base_dir) + config_dir = os.path.join(project_root, "config") - inventory_env_path = os.path.join(project_root, "inventory.env") - backend_env_path = os.path.join(base_dir, ".env") - - # Check for inventory.env in root (Master Config) - if os.path.exists(inventory_env_path): - load_dotenv(inventory_env_path) - log.info(f"✅ Loaded master configuration from {inventory_env_path}") - - # Check for local backend/.env (Legacy/Fragmented) - elif os.path.exists(backend_env_path): - load_dotenv(backend_env_path) - log.info(f"ℹ️ Loaded local configuration from {backend_env_path}") + # 1. Define Defaults + config = { + "database": { + "sqlite_path": "data/inventory.db", + "wal_mode": True, + "log_retention_days": 30 + }, + "ai": { + "primary_ai_provider": "gemini", + "fallback_provider": "claude", + "gemini_api_key": "", + "claude_api_key": "" + }, + "auth": { + "jwt_secret_key": "change_me_in_production", + "ldap_server": "", + "ldap_base_dn": "", + "password_cache_path": "data/.passwords" + }, + "logging": { + "log_level": "INFO", + "log_rotation_size_mb": 10, + "log_rotation_count": 5 + }, + "application": { + "data_dir": "./data", + "logs_dir": "./logs", + "cors_origins": "http://localhost:8917", + "server_ip": "localhost", + "frontend_port": "8917", + "frontend_ssl_port": "8919", + "backend_port": "8918", + "backend_ssl_port": "8918" + }, + "features": { + "ai_extraction_enabled": True, + "offline_sync_enabled": True, + "audit_logging_enabled": True + } + } + + # 2. Load backend.yaml + backend_yaml_path = os.path.join(config_dir, "backend.yaml") + if os.path.exists(backend_yaml_path): + try: + with open(backend_yaml_path, 'r') as f: + yaml_data = yaml.safe_load(f) or {} + _deep_merge(config, yaml_data) + log.info(f"✅ Loaded configuration from {backend_yaml_path}") + except Exception as e: + log.warning(f"⚠️ Failed to load {backend_yaml_path}: {e}") else: - log.info("ℹ️ [CONFIG] Using system environment variables (Docker/Server environment).") + log.warning(f"ℹ️ {backend_yaml_path} not found. Using defaults.") + + # 3. Load secrets.yaml + secrets_yaml_path = os.path.join(config_dir, "secrets.yaml") + if os.path.exists(secrets_yaml_path): + try: + with open(secrets_yaml_path, 'r') as f: + secrets_data = yaml.safe_load(f) or {} + _map_secrets(config, secrets_data) + log.info(f"✅ Loaded secrets from {secrets_yaml_path}") + except Exception as e: + log.warning(f"⚠️ Failed to load {secrets_yaml_path}: {e}") + + # 4. Apply Environment Overrides (D-06) + _apply_env_overrides(config) + + _config = config + return _config + +def get_config() -> dict: + """Get loaded config.""" + if not _config: + load_config() + return _config + +def validate_config(config: dict) -> bool: + """Validate config has all required values.""" + required = [ + ("auth", "jwt_secret_key"), + ("ai", "primary_ai_provider"), + ] + + missing = [] + for path in required: + val = config + for p in path: + val = val.get(p) + if not val or val == "change_me_in_production" or val == "CHANGE_ME_IN_PRODUCTION_MIN_32_CHARS": + missing.append(".".join(path)) + + if missing: + raise ConfigError(f"Missing or default required configuration: {', '.join(missing)}") + + # Validate enums + valid_providers = ["gemini", "claude"] + if config["ai"]["primary_ai_provider"] not in valid_providers: + raise ConfigError(f"Invalid primary_ai_provider: {config['ai']['primary_ai_provider']}. Must be one of {valid_providers}") + + valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR"] + if config["logging"]["log_level"].upper() not in valid_log_levels: + raise ConfigError(f"Invalid log_level: {config['logging']['log_level']}. Must be one of {valid_log_levels}") + + log.info(f"✅ Config validated: primary_ai_provider={config['ai']['primary_ai_provider']}, log_level={config['logging']['log_level']}") + return True # Auto-run if imported -load_config() +try: + load_config() + validate_config(_config) +except ConfigError as ce: + log.error(f"❌ Configuration error: {ce}") +except Exception as e: + log.error(f"❌ Unexpected error during config load: {e}") diff --git a/backend/config_manager.py b/backend/config_manager.py index aef2b0de..3bf27119 100644 --- a/backend/config_manager.py +++ b/backend/config_manager.py @@ -1,90 +1,118 @@ import os -from dotenv import load_dotenv +import yaml +import logging +from .config_loader import load_config, get_config + +log = logging.getLogger("ainventory") class ConfigManager: - """Safely manages multi-line .env files without corrupting other content.""" - + """Manages backend.yaml configuration file updates.""" + @staticmethod - def get_root_env_path(): - # backend/config_manager.py -> backend/ -> / + def get_config_path(): + """Returns the absolute path to backend.yaml.""" base_dir = os.path.dirname(os.path.abspath(__file__)) project_root = os.path.dirname(base_dir) - return os.path.join(project_root, "inventory.env") + return os.path.join(project_root, "config", "backend.yaml") @staticmethod - def update_keys(updates: dict): - """ - Updates specific keys in inventory.env. - Preserves comments and order where possible. - Appends new keys at the end if not found. - """ - env_path = ConfigManager.get_root_env_path() + def read_config() -> dict: + """Read backend.yaml and return current config.""" + path = ConfigManager.get_config_path() + if not os.path.exists(path): + log.warning(f"ℹ️ {path} not found. Returning empty dict.") + return {} - if not os.path.exists(env_path): - # Create a basic file if it doesn't exist (unlikely in this project) - with open(env_path, 'w', encoding='utf-8') as f: - f.write("# TFM aInventory — Generated Configuration\n") + try: + with open(path, 'r', encoding='utf-8') as f: + return yaml.safe_load(f) or {} + except Exception as e: + log.error(f"❌ Failed to read {path}: {e}") + return {} - with open(env_path, 'r', encoding='utf-8') as f: - lines = f.readlines() + @staticmethod + def update_config(updates: dict) -> dict: + """ + Update backend.yaml with new values and return updated config. + Only updates sections in backend.yaml. + """ + path = ConfigManager.get_config_path() + current_yaml = ConfigManager.read_config() + + # Merge updates (deep merge for sections) + for section, values in updates.items(): + if isinstance(values, dict) and section in current_yaml and isinstance(current_yaml[section], dict): + current_yaml[section].update(values) + else: + current_yaml[section] = values - new_lines = [] - keys_to_process = set(updates.keys()) - processed_keys = set() - - for line in lines: - trimmed = line.strip() - # Skip empty lines or comments when matching - if not trimmed or trimmed.startswith('#'): - new_lines.append(line) - continue - - # Check if this line is a key assignment we want to update - found_match = False - for key in keys_to_process: - if trimmed.startswith(f"{key}="): - new_lines.append(f"{key}={updates[key]}\n") - processed_keys.add(key) - found_match = True - break + try: + with open(path, 'w', encoding='utf-8') as f: + yaml.safe_dump(current_yaml, f, default_flow_style=False, sort_keys=False) + log.info(f"✅ Updated {path} with new values.") - if not found_match: - new_lines.append(line) + # Reload the global config + load_config() + return get_config() + except Exception as e: + log.error(f"❌ Failed to write {path}: {e}") + raise - # Append keys that weren't found in the file - missing_keys = keys_to_process - processed_keys - if missing_keys: - if new_lines and not new_lines[-1].endswith('\n'): - new_lines.append('\n') - if new_lines and not new_lines[-1].strip() == '': - new_lines.append('\n') - - new_lines.append("# --- Automatically Added Keys ---\n") - for key in missing_keys: - new_lines.append(f"{key}={updates[key]}\n") - - with open(env_path, 'w', encoding='utf-8') as f: - f.writelines(new_lines) - - # Force reload environment variables for the current process - load_dotenv(env_path, override=True) - return True + @staticmethod + def validate_config_file() -> bool: + """Validate backend.yaml syntax and required fields.""" + path = ConfigManager.get_config_path() + if not os.path.exists(path): + return False + + try: + with open(path, 'r', encoding='utf-8') as f: + yaml.safe_load(f) + return True + except Exception: + return False @staticmethod def get_masked_key(key_name: str): - """Returns a masked version of the environment variable.""" - val = os.environ.get(key_name) + """Returns a masked version of a configuration value or env var.""" + config = get_config() + + # Try to find in config first + val = None + if key_name == "JWT_SECRET_KEY": + val = config.get("auth", {}).get("jwt_secret_key") + elif key_name == "GEMINI_API_KEY": + val = config.get("ai", {}).get("gemini_api_key") + elif key_name == "CLAUDE_API_KEY": + val = config.get("ai", {}).get("claude_api_key") + + # Fallback to env + if not val: + val = os.environ.get(key_name) + if not val: return None - # Determine prefix based on key type prefix = "" - if key_name == "GEMINI_API_KEY": + if "GEMINI" in key_name: prefix = "G-" - elif key_name == "CLAUDE_API_KEY": + elif "CLAUDE" in key_name: prefix = "sk-" if len(val) <= 8: return f"{prefix}****" return f"{prefix}****{val[-4:]}" + +# Module-level functions for backward compatibility and direct import +def read_config() -> dict: + """Read backend.yaml and return current config.""" + return ConfigManager.read_config() + +def update_config(updates: dict) -> dict: + """Update backend.yaml with new values and return updated config.""" + return ConfigManager.update_config(updates) + +def validate_config_file() -> bool: + """Validate backend.yaml syntax and required fields.""" + return ConfigManager.validate_config_file() diff --git a/backend/entrypoint.sh b/backend/entrypoint.sh index 307fe9fe..ff50b676 100755 --- a/backend/entrypoint.sh +++ b/backend/entrypoint.sh @@ -2,11 +2,9 @@ # ============================================================================= # backend/entrypoint.sh # ============================================================================= -# Docker container entrypoint for TFM aInventory backend. -# Runs first-run initialization then starts the application server. -# -# This script is the ENTRYPOINT defined in backend/Dockerfile. -# DATA_DIR and LOGS_DIR are set via docker-compose.yml environment section. +# [D-07] Backend entrypoint - loads config from /app/config/ (YAML format) +# Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables +# [D-06] Environment variables override YAML config (takes precedence) # ============================================================================= set -euo pipefail @@ -19,6 +17,14 @@ echo "🐳 [Docker] LOGS_DIR=${LOGS_DIR:-/app/logs}" export DATA_DIR="${DATA_DIR:-/app/data}" export LOGS_DIR="${LOGS_DIR:-/app/logs}" +# Verify config is accessible +if [ ! -f "/app/config/backend.yaml" ]; then + echo "❌ [Docker] ERROR: /app/config/backend.yaml not found!" + echo "❌ [Docker] Config must be mounted from host at /app/config/ (read-only)" + echo "❌ [Docker] See config/README.md for setup instructions" + exit 1 +fi + # Run shared first-run initialization echo "🐳 [Docker] Running data initialization..." bash /app/scripts/init_data.sh diff --git a/backend/main.py b/backend/main.py index d25b27e7..df0e42e8 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,6 +1,6 @@ import os from ipaddress import ip_address, ip_network, AddressValueError -from . import config_loader # This triggers the automatic environment loading +from .config_loader import get_config from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware from fastapi.staticfiles import StaticFiles @@ -25,18 +25,20 @@ app = FastAPI(title="TFM aInventory API", version="1.1.0") log.info("TFM aInventory API process started.") # [SECURITY FIX M-01] CORS Configuration with Subnet Support -# We dynamically build allowed origins from environment variables to simplify deployment. -_raw_origins = os.environ.get("ALLOWED_ORIGINS", "") +# We dynamically build allowed origins from configuration to simplify deployment. +config = get_config() +app_config = config.get("application", {}) +_raw_origins = app_config.get("cors_origins", "") ALLOWED_ORIGINS = [o.strip() for o in _raw_origins.split(",") if o.strip()] # Allowed subnets for subnet-based CORS validation (e.g., VPN, Tailscale) ALLOWED_SUBNETS = [] -# Automatically add origins based on network_config.env variables if present -server_ip = os.environ.get("SERVER_IP") -front_port = os.environ.get("FRONTEND_PORT", "8917") -front_ssl_port = os.environ.get("FRONTEND_SSL_PORT", "8919") -back_ssl_port = os.environ.get("BACKEND_SSL_PORT", "8918") +# Automatically add origins based on network config if present +server_ip = app_config.get("server_ip") +front_port = app_config.get("frontend_port", "8917") +front_ssl_port = app_config.get("frontend_ssl_port", "8919") +back_ssl_port = app_config.get("backend_ssl_port", "8918") # Always allow localhost defaults = [ @@ -60,7 +62,7 @@ if server_ip and server_ip != "localhost": ALLOWED_ORIGINS.append(ip_o) # [NEW] Add Extra Allowed Origins (Tailscale, VPN, etc.) with Subnet Support -extra_origins_raw = os.environ.get("EXTRA_ALLOWED_ORIGINS", "") +extra_origins_raw = app_config.get("cors_origins", "") if extra_origins_raw: for extra_item in [o.strip() for o in extra_origins_raw.split(",") if o.strip()]: # Check if it's a subnet (contains /) or individual IP diff --git a/backend/requirements.txt b/backend/requirements.txt index 11412865..0f19c89c 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -5,7 +5,7 @@ pydantic>=2.0.0 pydantic-settings>=2.0.0 google-genai>=0.1.0 anthropic>=0.40.0 -python-dotenv>=1.0.0 +PyYAML>=6.0.1 Pillow>=10.0.0 python-multipart>=0.0.9 ldap3>=2.9.1 diff --git a/config/README.md b/config/README.md new file mode 100644 index 00000000..afbb150d --- /dev/null +++ b/config/README.md @@ -0,0 +1,155 @@ +# TFM aInventory Configuration Management (v1.12.0) + +This directory contains the centralized configuration for the TFM aInventory system. Starting from Phase 7, the application has moved away from scattered `.env` files and hardcoded values towards a structured, domain-specific YAML configuration approach. + +## Table of Contents + +1. [Overview](#overview) +2. [Quick Start](#quick-start) +3. [Configuration Files](#configuration-files) + - [backend.yaml](#backendyaml) + - [frontend.yaml](#frontendyaml) + - [network.yaml](#networkyaml) + - [docker.yaml](#dockeryaml) + - [secrets.yaml](#secretsyaml) +4. [Environment Variable Overrides](#environment-variable-overrides) +5. [Load Order](#load-order) +6. [Security Best Practices](#security-best-practices) +7. [Troubleshooting](#troubleshooting) + +--- + +## Overview + +The `config/` directory is the **single source of truth** for all application settings. By using YAML, we achieve: +- **Structure:** Hierarchical settings grouped by domain. +- **Documentation:** Inline comments explaining every variable. +- **Flexibility:** Easy overrides via environment variables. +- **Safety:** Clear separation between non-sensitive config and secrets. + +## Quick Start + +To set up your configuration for a new installation: + +1. **Clone the examples:** + ```bash + cp config/backend.yaml.example config/backend.yaml + cp config/frontend.yaml.example config/frontend.yaml + cp config/network.yaml.example config/network.yaml + cp config/docker.yaml.example config/docker.yaml + cp config/secrets.yaml.example config/secrets.yaml + ``` + +2. **Generate Secrets:** + Open `config/secrets.yaml` and fill in your API keys. Generate a strong JWT secret: + ```bash + python3 -c "import secrets; print(secrets.token_urlsafe(64))" + ``` + +3. **Verify YAML Syntax:** + Ensure your changes are valid YAML: + ```bash + python3 -c "import yaml; [yaml.safe_load(open(f)) for f in ['config/backend.yaml', 'config/frontend.yaml', 'config/network.yaml', 'config/docker.yaml', 'config/secrets.yaml']]" + ``` + +## Configuration Files + +### backend.yaml +Controls the FastAPI backend, database, AI integration, and logging. + +| Section | Variable | Description | Default | +|---------|----------|-------------|---------| +| database | `sqlite_path` | Path to the SQLite DB file | `data/inventory.db` | +| database | `wal_mode` | Enable Write-Ahead Logging | `true` | +| ai | `primary_ai_provider` | `gemini` or `claude` | `gemini` | +| auth | `jwt_secret_key` | Secret for JWT (use `secrets.yaml`) | - | +| logging | `log_level` | `DEBUG`, `INFO`, `WARNING`, `ERROR` | `INFO` | + +### frontend.yaml +Controls the Next.js frontend application behavior and PWA settings. + +| Variable | Description | Default | +|----------|-------------|---------| +| `api.backend_url` | Full URL of the backend API | `http://localhost:8916` | +| `features.offline_enabled` | Enable service worker caching | `true` | +| `pwa.app_name` | Display name of the PWA | `TFM aInventory` | + +### network.yaml +Defines host-side port mappings and SSL/CORS policies. + +| Variable | Description | Default | +|----------|-------------|---------| +| `ports.backend_port` | Host port for backend | `8916` | +| `ports.frontend_port` | Host port for frontend | `8917` | +| `ssl.ssl_enabled` | Enable HTTPS via Caddy | `true` | +| `cors.allowed_origins` | List of allowed origins | `*` | + +### docker.yaml +Resource limits and container orchestration settings. + +| Variable | Description | Default | +|----------|-------------|---------| +| `resources.backend_cpu_limit` | Max CPU cores for backend | `1.0` | +| `resources.backend_memory_limit` | Max RAM for backend | `1G` | +| `volumes.use_named_volumes` | Use named volumes vs bind | `true` | + +### secrets.yaml +**CRITICAL:** This file contains sensitive data. It is ignored by Git and must NEVER be committed. +It contains: +- `JWT_SECRET_KEY` +- `GEMINI_API_KEY` +- `CLAUDE_API_KEY` +- `DATABASE_PASSWORD` (optional) +- `LDAP_PASSWORD` (optional) + +## Environment Variable Overrides + +Any value in the YAML files can be overridden by a system environment variable. The naming convention is: +`DOMAIN_SECTION_VARIABLE` (all uppercase). + +**Examples:** +- `backend.yaml`: `database.sqlite_path` -> `BACKEND_DATABASE_SQLITE_PATH` +- `network.yaml`: `ports.backend_port` -> `NETWORK_PORTS_BACKEND_PORT` +- `secrets.yaml`: `GEMINI_API_KEY` -> `GEMINI_API_KEY` (Directly mapped for common secrets) + +Environment variables take **precedence** over YAML files. This is useful for Docker deployments where secrets are injected at runtime. + +## Load Order + +The application loads configuration in the following priority: +1. **System Environment Variables** (Highest) +2. **secrets.yaml** +3. **Domain YAML files** (`backend.yaml`, etc.) +4. **Code Defaults** (Lowest) + +## Security Best Practices + +1. **Permissions:** Set strict permissions on `secrets.yaml`: + ```bash + chmod 600 config/secrets.yaml + ``` +2. **Rotation:** Rotate your `JWT_SECRET_KEY` and API keys every 90 days. +3. **CORS:** In production, never use `allowed_origins: "*"`. List specific IPs or FQDNs. +4. **Volume Mounts:** In Docker, mount the `config/` directory as **read-only** (`:ro`) except for the `secrets.yaml` if needed by a management tool. + +## Troubleshooting + +### Invalid YAML Syntax +If the application fails to start with a configuration error: +- Check for tabs instead of spaces (YAML requires spaces). +- Check for missing colons or incorrect indentation. +- Use a linter: `python3 -m yaml.scanner config/backend.yaml`. + +### Configuration Not Applying +- Verify the variable name matches exactly (case-sensitive in YAML). +- Check if an environment variable is overriding the YAML value. +- Ensure the file is in the correct directory: `/app/config/` inside the container. + +### Secret Exposure +If you accidentally commit a YAML file containing secrets: +1. Delete the file from the repository. +2. **Immediately** rotate all exposed keys and secrets. +3. Purge the secret from Git history using `git-filter-repo` or BFG Repo-Cleaner. + +--- +*TFM aInventory - Centralized Configuration Management System* diff --git a/config/backend.yaml.example b/config/backend.yaml.example new file mode 100644 index 00000000..4b9bebee --- /dev/null +++ b/config/backend.yaml.example @@ -0,0 +1,110 @@ +# ============================================================================= +# TFM aInventory - Backend Configuration Schema (v1.12.0) +# ============================================================================= +# Purpose: Configuration for the FastAPI backend, database, AI, and logging. +# Load order: system environment variables > config/backend.yaml > code defaults. +# Required/Optional: Most are optional; defaults in code provide sensible behavior. +# ============================================================================= + +# --- Database & Storage --- +database: + # Path to SQLite database file (relative to backend/ folder) + # Default: data/inventory.db + # Environment: BACKEND_DATABASE_SQLITE_PATH + sqlite_path: "data/inventory.db" + + # Keep database logs for performance auditing? + # Default: true + # Environment: BACKEND_DATABASE_WAL_MODE + wal_mode: true + + # How many days of audit logs to keep? + # Default: 30 + # Environment: BACKEND_DATABASE_LOG_RETENTION_DAYS + log_retention_days: 30 + +# --- AI & Vision Services --- +ai: + # Primary provider for image OCR and classification (gemini|claude) + # Default: gemini + # Environment: BACKEND_AI_PRIMARY_AI_PROVIDER + primary_ai_provider: "gemini" + + # Fallback provider if primary fails + # Default: claude + # Environment: BACKEND_AI_FALLBACK_PROVIDER + fallback_provider: "claude" + + # Gemini API Key (obtain from https://aistudio.google.com/) + # Environment: BACKEND_AI_GEMINI_API_KEY or GEMINI_API_KEY + gemini_api_key: "your-gemini-api-key" + + # Claude API Key (obtain from Anthropic Console) + # Environment: BACKEND_AI_CLAUDE_API_KEY or CLAUDE_API_KEY + claude_api_key: "your-claude-api-key" + +# --- Authentication & LDAP --- +auth: + # JWT Secret Key (min 32 chars) + # Generate with: openssl rand -hex 32 + # Environment: BACKEND_AUTH_JWT_SECRET_KEY or JWT_SECRET_KEY + jwt_secret_key: "change_me_in_production" + + # LDAP Server address (optional) + # Example: "ldap://192.168.1.100" + # Environment: BACKEND_AUTH_LDAP_SERVER + ldap_server: "" + + # LDAP Base DN for user search + # Example: "dc=example,dc=com" + # Environment: BACKEND_AUTH_LDAP_BASE_DN + ldap_base_dn: "" + + # Path to store hashed passwords for offline use + # Environment: BACKEND_AUTH_PASSWORD_CACHE_PATH + password_cache_path: "data/.passwords" + +# --- Logging & Diagnostics --- +logging: + # Log level (DEBUG|INFO|WARNING|ERROR) + # Default: INFO + # Environment: BACKEND_LOGGING_LOG_LEVEL or LOG_LEVEL + log_level: "INFO" + + # Log file rotation size in megabytes + # Default: 10 + # Environment: BACKEND_LOGGING_LOG_ROTATION_SIZE_MB + log_rotation_size_mb: 10 + + # Number of rotated log files to keep + # Default: 5 + # Environment: BACKEND_LOGGING_LOG_ROTATION_COUNT + log_rotation_count: 5 + +# --- Application --- +application: + # Directory for persistent data + # Environment: BACKEND_APPLICATION_DATA_DIR or DATA_DIR + data_dir: "./data" + + # Directory for log files + # Environment: BACKEND_APPLICATION_LOGS_DIR or LOGS_DIR + logs_dir: "./logs" + + # Comma-separated list of extra allowed CORS origins (IPs/FQDNs) + # Environment: BACKEND_APPLICATION_CORS_ORIGINS or EXTRA_ALLOWED_ORIGINS + cors_origins: "http://localhost:8917" + +# --- Feature Flags --- +features: + # Enable AI image extraction and OCR? + # Default: true + ai_extraction_enabled: true + + # Enable offline synchronization features? + # Default: true + offline_sync_enabled: true + + # Enable detailed audit logging for each API request? + # Default: true + audit_logging_enabled: true diff --git a/config/docker.yaml.example b/config/docker.yaml.example new file mode 100644 index 00000000..4b38b7e9 --- /dev/null +++ b/config/docker.yaml.example @@ -0,0 +1,61 @@ +# ============================================================================= +# TFM aInventory - Docker Configuration Schema (v1.12.0) +# ============================================================================= +# Purpose: Resource limits, image tags, and volume management for Docker. +# ============================================================================= + +# --- Image Management --- +images: + # Backend container image name and tag + # Default: backend:latest + backend_image: "backend:latest" + + # Frontend container image name and tag + # Default: frontend:latest + frontend_image: "frontend:latest" + + # Proxy container image name and tag + # Default: caddy:latest + proxy_image: "caddy:latest" + +# --- Resource Constraints --- +resources: + # CPU limit for the backend container + # Default: 1.0 (1 core) + backend_cpu_limit: "1.0" + + # Memory limit for the backend container + # Default: 1G + backend_memory_limit: "1G" + + # CPU limit for the frontend container + # Default: 0.5 (0.5 core) + frontend_cpu_limit: "0.5" + + # Memory limit for the frontend container + # Default: 512M + frontend_memory_limit: "512M" + +# --- Volume Management --- +volumes: + # Driver to use for data volume + # Default: local + data_volume_driver: "local" + + # Driver to use for logs volume + # Default: local + logs_volume_driver: "local" + + # Use named volumes instead of bind mounts for persistence? + # Default: true + use_named_volumes: true + +# --- Networking --- +network: + # Internal Docker network name + # Default: inventory-net + network_name: "inventory-net" + + # Internal Docker network driver + # Default: bridge + network_driver: "bridge" diff --git a/config/frontend.yaml.example b/config/frontend.yaml.example new file mode 100644 index 00000000..a4de88a9 --- /dev/null +++ b/config/frontend.yaml.example @@ -0,0 +1,63 @@ +# ============================================================================= +# TFM aInventory - Frontend Configuration Schema (v1.12.0) +# ============================================================================= +# Purpose: Configuration for the Next.js frontend application. +# ============================================================================= + +# --- API Connection --- +api: + # Base URL of the backend API + # Default: http://localhost:8916 + # Environment: FRONTEND_API_BACKEND_URL or BACKEND_URL + backend_url: "http://localhost:8916" + + # API timeout in milliseconds + # Default: 30000 (30 seconds) + # Environment: FRONTEND_API_TIMEOUT_MS + timeout_ms: 30000 + +# --- Feature Flags & PWA --- +features: + # Enable PWA service worker? + # Default: true + service_worker_enabled: true + + # Enable offline functionality? + # Default: true + offline_enabled: true + + # Enable AI image extraction UI features? + # Default: true + ai_extraction_ui_enabled: true + +# --- PWA Branding --- +pwa: + # Application long name + # Default: TFM aInventory + app_name: "TFM aInventory" + + # Application short name for home screen + # Default: aInventory + short_name: "aInventory" + + # Initial URL to open on launch + # Default: / + start_url: "/" + + # Display mode (standalone|browser|minimal-ui|fullscreen) + # Default: standalone + display_mode: "standalone" + +# --- Feature Toggles --- +toggles: + # Enable built-in QR code scanner? + # Default: true + enable_qr_scanner: true + + # Enable built-in barcode scanner? + # Default: true + enable_barcode_scanner: true + + # Enable batch import functionality? + # Default: true + enable_batch_import: true diff --git a/config/network.yaml.example b/config/network.yaml.example new file mode 100644 index 00000000..c3543fff --- /dev/null +++ b/config/network.yaml.example @@ -0,0 +1,73 @@ +# ============================================================================= +# TFM aInventory - Network Configuration Schema (v1.12.0) +# ============================================================================= +# Purpose: Network ports, SSL settings, and proxy configuration. +# ============================================================================= + +# --- Port Assignments --- +ports: + # Host-side port for HTTP backend + # Default: 8916 + # Environment: NETWORK_PORTS_BACKEND_PORT or BACKEND_PORT + backend_port: 8916 + + # Host-side port for HTTP frontend + # Default: 8917 + # Environment: NETWORK_PORTS_FRONTEND_PORT or FRONTEND_PORT + frontend_port: 8917 + + # Host-side port for HTTPS backend (via Caddy) + # Default: 8918 + # Environment: NETWORK_PORTS_BACKEND_SSL_PORT or BACKEND_SSL_PORT + backend_ssl_port: 8918 + + # Host-side port for HTTPS frontend (via Caddy) + # Default: 8919 + # Environment: NETWORK_PORTS_FRONTEND_SSL_PORT or FRONTEND_SSL_PORT + frontend_ssl_port: 8919 + +# --- SSL & Security --- +ssl: + # Enable SSL/TLS termination via reverse proxy? + # Default: true + # Environment: NETWORK_SSL_ENABLED + ssl_enabled: true + + # Path to SSL certificate (if not using Caddy's auto-HTTPS) + # Environment: NETWORK_SSL_CERTIFICATE_PATH + certificate_path: "" + + # Path to SSL private key + # Environment: NETWORK_SSL_KEY_PATH + key_path: "" + +# --- Proxy (Caddy) Configuration --- +proxy: + # Caddy log level (debug|info|warn|error) + # Default: info + # Environment: NETWORK_PROXY_CADDY_LOG_LEVEL + caddy_log_level: "info" + + # Maximum timeout for reading requests in seconds + # Default: 60 + # Environment: NETWORK_PROXY_READ_TIMEOUT_S + proxy_read_timeout_s: 60 + + # Maximum request body size in MB + # Default: 10 + # Environment: NETWORK_PROXY_MAX_REQUEST_SIZE_MB + max_request_size_mb: 10 + +# --- CORS Policies --- +cors: + # Comma-separated list of allowed origins + # Environment: NETWORK_CORS_ALLOWED_ORIGINS or EXTRA_ALLOWED_ORIGINS + allowed_origins: "*" + + # Allowed HTTP methods + # Environment: NETWORK_CORS_ALLOWED_METHODS + allowed_methods: "GET,POST,PUT,DELETE,OPTIONS" + + # Allowed HTTP headers + # Environment: NETWORK_CORS_ALLOWED_HEADERS + allowed_headers: "Authorization,Content-Type,Accept" diff --git a/config/secrets.yaml.example b/config/secrets.yaml.example new file mode 100644 index 00000000..4c08b906 --- /dev/null +++ b/config/secrets.yaml.example @@ -0,0 +1,34 @@ +# ============================================================================= +# TFM aInventory - Secrets Configuration Template (v1.12.0) +# ============================================================================= +# Purpose: Sensitive configuration (API keys, passwords, secret keys). +# Security: 'config/secrets.yaml' is ignored by Git. Do NOT commit actual secrets. +# Setup: Copy this file to 'config/secrets.yaml' and fill in actual values. +# ============================================================================= + +# --- JWT Secrets --- +# Secret key used for signing JSON Web Tokens. +# Generate a strong random key for production use: +# python3 -c "import secrets; print(secrets.token_urlsafe(64))" +# Environment override: BACKEND_AUTH_JWT_SECRET_KEY or JWT_SECRET_KEY +JWT_SECRET_KEY: "CHANGE_ME_IN_PRODUCTION_MIN_32_CHARS" + +# --- AI API Keys --- +# Google Gemini API Key (Required for AI image processing) +# Get one at: https://aistudio.google.com/ +# Environment override: BACKEND_AI_GEMINI_API_KEY or GEMINI_API_KEY +GEMINI_API_KEY: "your-gemini-api-key" + +# Anthropic Claude API Key (Required for fallback/enhanced processing) +# Get one at: https://console.anthropic.com/ +# Environment override: BACKEND_AI_CLAUDE_API_KEY or CLAUDE_API_KEY +CLAUDE_API_KEY: "your-claude-api-key" + +# --- External Services --- +# Database password (if using an external DB instead of SQLite) +# Environment override: BACKEND_DATABASE_PASSWORD +DATABASE_PASSWORD: "" + +# LDAP password for the service account +# Environment override: BACKEND_AUTH_LDAP_PASSWORD +LDAP_PASSWORD: "" diff --git a/deploy.sh b/deploy.sh deleted file mode 100755 index beb001cb..00000000 --- a/deploy.sh +++ /dev/null @@ -1,222 +0,0 @@ -#!/bin/bash -set -euo pipefail - -# Phase 6, Plan 1, Task 4: Automated Deployment Script -# Usage: ./deploy.sh [production|staging|development] [--rebuild] -# Purpose: Single-command deployment with Docker Compose, pre-flight checks, and health validation - -DEPLOYMENT_ENV="${1:-production}" -REBUILD_FLAG="${2:---no-rebuild}" - -# Color output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Logging functions -log_info() { echo -e "${BLUE}[INFO]${NC} $1"; } -log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; } -log_warn() { echo -e "${YELLOW}[WARN]${NC} $1"; } -log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; } - -log_info "=== TFM aInventory Deployment Script ===" -log_info "Environment: $DEPLOYMENT_ENV" -log_info "Rebuild: $REBUILD_FLAG" - -# ============================================================================ -# STEP 1: Pre-flight checks -# ============================================================================ -log_info "Step 1/10: Running pre-flight checks..." - -command -v docker &> /dev/null || log_error "Docker not installed. Please install Docker 24.0+" -log_success " ✓ Docker is installed" - -command -v docker-compose &> /dev/null || log_error "Docker Compose not installed. Please install Docker Compose 2.0+" -log_success " ✓ Docker Compose is installed" - -[[ -f "docker-compose.yml" ]] || log_error "docker-compose.yml not found in current directory" -log_success " ✓ docker-compose.yml found" - -[[ -f "inventory.env" ]] || log_warn "inventory.env not found; attempting to create from template..." -if [[ ! -f "inventory.env" ]] && [[ -f "inventory.env.template" ]]; then - cp inventory.env.template inventory.env - log_success " ✓ inventory.env created from template (review and customize)" -elif [[ ! -f "inventory.env" ]]; then - log_error "inventory.env not found and no template available. Create inventory.env before deploying." -fi - -# ============================================================================ -# STEP 2: Validate environment file -# ============================================================================ -log_info "Step 2/10: Validating inventory.env..." - -if [[ ! -f ".env.validation.sh" ]]; then - log_warn " .env.validation.sh not found; skipping validation" -else - bash .env.validation.sh || log_error "Environment validation failed" -fi - -log_success " ✓ Environment variables validated" - -# ============================================================================ -# STEP 3: Check port availability -# ============================================================================ -log_info "Step 3/10: Checking port availability..." - -# Source inventory.env to get port values -source inventory.env - -BACKEND_PORT=${BACKEND_PORT:-8000} -FRONTEND_PORT=${FRONTEND_PORT:-3000} -BACKEND_SSL_PORT=${BACKEND_SSL_PORT:-8918} -FRONTEND_SSL_PORT=${FRONTEND_SSL_PORT:-8919} - -for port in "$BACKEND_PORT" "$FRONTEND_PORT" "$BACKEND_SSL_PORT" "$FRONTEND_SSL_PORT"; do - if command -v netstat &> /dev/null; then - if netstat -tuln 2>/dev/null | grep -q ":$port "; then - log_error "Port $port is already in use. Choose a different port in inventory.env" - fi - else - log_warn " netstat not available; skipping port check (ensure ports are available)" - fi -done - -log_success " ✓ All required ports are available" - -# ============================================================================ -# STEP 4: Check disk space -# ============================================================================ -log_info "Step 4/10: Checking disk space..." - -AVAILABLE_SPACE=$(df . | awk 'NR==2 {print $4}') -REQUIRED_SPACE=$((10 * 1024 * 1024)) # 10GB in KB - -if [[ $AVAILABLE_SPACE -lt $REQUIRED_SPACE ]]; then - log_warn " Available space: $(($AVAILABLE_SPACE / 1024 / 1024))GB (recommended: 10GB+)" -else - log_success " ✓ Sufficient disk space available ($(($AVAILABLE_SPACE / 1024 / 1024))GB)" -fi - -# ============================================================================ -# STEP 5: Build or pull images -# ============================================================================ -log_info "Step 5/10: Building Docker images..." - -if [[ "$REBUILD_FLAG" == "--rebuild" ]]; then - log_info " Building with --no-cache (full rebuild)..." - docker-compose build --no-cache || log_error "Docker build failed" -else - log_info " Building with layer cache (incremental)..." - docker-compose build || log_error "Docker build failed" -fi - -log_success " ✓ Docker images built successfully" - -# ============================================================================ -# STEP 6: Create data directories -# ============================================================================ -log_info "Step 6/10: Preparing data directories..." - -mkdir -p data logs config -mkdir -p data/caddy_data data/caddy_config -chmod -R 777 data logs config - -log_success " ✓ Data directories created with proper permissions" - -# ============================================================================ -# STEP 7: Initialize database -# ============================================================================ -log_info "Step 7/10: Checking database initialization..." - -if [[ ! -f "data/inventory.db" ]]; then - log_info " Database not found; will be initialized on first backend startup" - log_success " ✓ Database initialization scheduled" -else - log_success " ✓ Existing database found; reusing" -fi - -# ============================================================================ -# STEP 8: Start services -# ============================================================================ -log_info "Step 8/10: Starting Docker services..." - -docker-compose up -d || log_error "Failed to start Docker services" -log_success " ✓ Services started in background" - -# ============================================================================ -# STEP 9: Wait for health checks -# ============================================================================ -log_info "Step 9/10: Waiting for services to become healthy (max 60 seconds)..." - -max_attempts=30 -attempt=0 -all_healthy=false - -while [[ $attempt -lt $max_attempts ]]; do - # Check if all 3 services are healthy - if docker-compose ps | grep -q "healthy.*healthy.*healthy"; then - all_healthy=true - break - fi - - attempt=$((attempt + 1)) - remaining=$((max_attempts - attempt)) - log_info " Waiting... ($remaining attempts remaining)" - sleep 2 -done - -if [[ "$all_healthy" == true ]]; then - log_success " ✓ All services are healthy" -else - log_warn "Services did not become healthy within timeout. Checking logs..." - docker-compose logs --tail=50 || true - log_error "Service health check timeout. Review logs above." -fi - -# ============================================================================ -# STEP 10: Verify connectivity -# ============================================================================ -log_info "Step 10/10: Verifying service connectivity..." - -if curl -sf "http://localhost:${BACKEND_PORT}/health" &> /dev/null; then - log_success " ✓ Backend API responding at http://localhost:${BACKEND_PORT}/health" -else - log_warn " Backend health check failed; services may still be initializing" -fi - -if curl -sf "http://localhost:${FRONTEND_PORT}/" &> /dev/null; then - log_success " ✓ Frontend responding at http://localhost:${FRONTEND_PORT}" -else - log_warn " Frontend check failed; container may still be initializing" -fi - -# ============================================================================ -# Summary -# ============================================================================ -echo "" -echo -e "${GREEN}╔════════════════════════════════════════════════════════════╗${NC}" -echo -e "${GREEN}║${NC} Deployment completed successfully! ${GREEN}║${NC}" -echo -e "${GREEN}╚════════════════════════════════════════════════════════════╝${NC}" -echo "" -echo "Access points:" -echo " Frontend (HTTP): http://localhost:${FRONTEND_PORT}" -echo " Backend (HTTP): http://localhost:${BACKEND_PORT}" -echo " API Docs: http://localhost:${BACKEND_PORT}/docs" -echo " Frontend (HTTPS): https://localhost:${FRONTEND_SSL_PORT}" -echo " Backend (HTTPS): https://localhost:${BACKEND_SSL_PORT}" -echo "" -echo "Useful commands:" -echo " View logs: docker-compose logs -f" -echo " Stop services: docker-compose down" -echo " Restart: docker-compose restart" -echo " Status: docker-compose ps" -echo "" -echo "For deployment in production ($DEPLOYMENT_ENV):" -echo " • Review and update JWT_SECRET_KEY in inventory.env" -echo " • Configure firewall to expose only required ports" -echo " • Set up automated backups (see docs/DEPLOYMENT_QUICKSTART.md)" -echo " • Monitor logs regularly: docker-compose logs -f" -echo "" -log_success "Deployment ready!" diff --git a/dev_docs/SESSION_STATE.md b/dev_docs/SESSION_STATE.md index a5245663..81e34499 100644 --- a/dev_docs/SESSION_STATE.md +++ b/dev_docs/SESSION_STATE.md @@ -1,51 +1,69 @@ # CURRENT AI WORKING SESSION — HANDOVER -**Active AI:** Gemini CLI (Antigravity) +**Active AI:** Claude Haiku 4.5 (Claude Code) **Last Updated:** 2026-04-23 -**Current Version:** v1.14.6 (Cleanup Branch: `maintenance/codebase-cleanup`) -**Status**: 🟢 CLEAN & CONSOLIDATED +**Current Version:** v1.14.6 +**Status**: 🟢 PHASE 7 PLANNED --- -## SESSION 39 EXECUTION — Major Codebase Cleanup & Doc Consolidation +## SESSION 40 EXECUTION — Phase 7 Config Consolidation Planning ### Work Completed This Session -**1. Documentation Consolidation:** -- ✅ Created **`DEPLOYMENT.md`**: Unified guide combining 7+ operational/setup files. -- ✅ Created **`dev_docs/PLAN.md`**: Consolidated active roadmap, requirements, and decisions from `.planning/`. -- ✅ Updated **`AI_RULES.md`**: Integrated refactoring and testing rules from `AGENTS.md`. -- ✅ Updated **`PROJECT_ARCHITECTURE.md`**: Added tech stack details and mobile constraints. -- ✅ Refined **`README.md`** and **`USER_GUIDE.md`**: Removed redundant technical info and simplified for end-users. +**1. Phase 7 Planning (Config Consolidation):** +- ✅ Executed `/gsd-plan-phase 7 --skip-research` +- ✅ Replanned Phase 7 from scratch (existing 18-task .env plan was misaligned with CONTEXT decisions) +- ✅ Created 4 new comprehensive plans aligned with YAML + Python decisions: + - **Plan 07-01**: Create config/ folder structure with YAML files + examples + documentation + - **Plan 07-02**: Refactor backend config_loader.py for YAML parsing + deprecate inventory.env + - **Plan 07-03**: Convert 4 bash deployment scripts to Python (deploy.py, run_standalone.py, install_service.py, export_prod.py) + - **Plan 07-04**: Update Docker/Compose, Dockerfile, .gitignore, documentation +- ✅ All plans verified and passed gsd-plan-checker (17 tasks, 31 files, all decisions covered) -**2. Workspace Cleanup:** -- ✅ Deleted 10+ obsolete root markdown files (audits, spacing reports, old guides). -- ✅ Deleted 3 historical reports from `dev_docs/`. -- ✅ Deleted redundant directories: `docs/` and `.planning/`. -- ✅ Removed temporary artifacts: `.coverage`, `.AGENTS.md.swp`, `.env.validation.sh`. +**2. Phase 7 Decisions Implemented:** +- ✅ **D-01**: YAML format standardization (backend.yaml, frontend.yaml, network.yaml, docker.yaml) +- ✅ **D-02**: Secrets management (config/secrets.yaml, git-ignored, .example files) +- ✅ **D-03**: Example files tracked in git (*.yaml.example) +- ✅ **D-04**: Complete inventory.env deprecation (no fallback) +- ✅ **D-05**: Bash → Python script conversion for deployment +- ✅ **D-06**: Backend config load order: env vars > YAML > defaults +- ✅ **D-07**: Docker Compose updated for config/ structure +- ✅ **D-08**: Comprehensive documentation (DEPLOYMENT.md, README.md, config/README.md) -**3. Branching:** -- All work performed on `maintenance/codebase-cleanup`. -- No changes made to production code logic, only documentation and workspace organization. +**3. Plan Structure:** +- **Wave 1**: Foundation (config/ structure creation) +- **Wave 2**: Backend refactoring + Python scripts (parallel execution) +- **Wave 3**: Docker integration + final documentation (depends on Waves 1-2) -### Final Workspace State -- **SSOT Core**: `AI_RULES.md`, `PROJECT_ARCHITECTURE.md`, `CLAUDE.md`, `GEMINI.md`. -- **Primary Guides**: `DEPLOYMENT.md`, `README.md`, `USER_GUIDE.md`. -- **Planning**: `dev_docs/PLAN.md`. -- **Logs**: `dev_docs/ARCHIVE_LOGS.md`, `dev_docs/SESSION_STATE.md`. +### Phase 7 Artifact Status +- **Plans**: `.planning/phases/07-config-consolidation/07-{01,02,03,04}-PLAN.md` ✓ Created +- **Context**: `.planning/phases/07-config-consolidation/07-CONTEXT.md` ✓ Exists (locked decisions) +- **Verification**: All 4 plans passed gsd-plan-checker ✓ --- ## NEXT STEPS -1. **Review & Merge**: - - Inspect changes in `maintenance/codebase-cleanup`. - - Merge to `dev` if the new documentation structure is satisfactory. +1. **Execute Phase 7**: + ```bash + /gsd-execute-phase 7 + ``` + - Wave 1: Create config/ folder with YAML files and examples + - Wave 2 (parallel): Refactor backend config loader + create Python deployment scripts + - Wave 3: Update Docker, documentation, and .gitignore -2. **Resume Phase 5/6**: - - Continue with the technical tasks defined in `dev_docs/PLAN.md`. - - Phase 6 focus: scale testing and production hardening. +2. **After Phase 7 Completion**: + - All configuration consolidated in config/ folder (YAML format) + - inventory.env deprecated and removed + - Deployment scripts converted to Python + - Docker and standalone deployments fully working with new structure + - Comprehensive documentation in place + +3. **Future Phases**: + - Phase 8+ (as defined in ROADMAP) + - Consider advanced config features (hot-reload, KMS encryption, config versioning) for future phases --- -✓ Done. +✓ Phase 7 planned and ready for execution. diff --git a/docker-compose.yml b/docker-compose.yml index 12dd6694..6eaaa6fb 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,15 +10,18 @@ services: - inventory_net ports: - ${BACKEND_PORT:-8000}:8000 - env_file: - - inventory.env + # [D-04] inventory.env deprecated — see config/ folder instead + # env_file: + # - inventory.env volumes: + # [D-07] New config/ structure — YAML config mounted read-only # Named volumes for data persistence - backend_data:/app/data - backend_logs:/app/logs - ./config:/app/config:ro - ./scripts:/app/scripts:ro environment: + # [D-06] Environment variables override config/backend.yaml load order - DATA_DIR=/app/data - LOGS_DIR=/app/logs # [C-01] JWT secret key — GENERATE A SECURE VALUE FOR PRODUCTION! @@ -47,10 +50,13 @@ services: - inventory_net ports: - ${FRONTEND_PORT:-3000}:3000 - env_file: - - inventory.env + # [D-04] inventory.env deprecated — see config/ folder instead + # env_file: + # - inventory.env volumes: - frontend_logs:/app/logs + # [D-07] New config/ structure — YAML config mounted read-only + - ./config:/app/config:ro # Write Next.js logs to both stdout (docker logs) and file (mapped volume) command: sh -c "mkdir -p /app/logs && node server.js 2>&1 | tee -a /app/logs/frontend.log" healthcheck: @@ -82,10 +88,13 @@ services: ports: - ${BACKEND_SSL_PORT:-8918}:444 - ${FRONTEND_SSL_PORT:-8919}:443 - env_file: - - inventory.env + # [D-04] inventory.env deprecated — see config/ folder instead + # env_file: + # - inventory.env volumes: - ./config/Caddyfile:/etc/caddy/Caddyfile:ro + # [D-07] New config/ structure — YAML config mounted read-only + - ./config:/app/config:ro # Persist the internal Caddy certificates so users don't get new certificate warnings constantly - caddy_data:/data - caddy_config:/config diff --git a/export_prod.sh b/export_prod.sh deleted file mode 100755 index ae8a11ff..00000000 --- a/export_prod.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash -# export_prod.sh - Generates a clean production bundle for distribution - -echo "📦 Preparing TFM aInventory Production Bundle..." - -# Extract version from frontend/VERSION.json -VERSION=$(grep '"version"' frontend/VERSION.json | head -n 1 | awk -F '"' '{print $4}') -PROD_DIR="aInventory-PROD-v${VERSION}" - -# Clean previous run if it exists -rm -rf "$PROD_DIR" -rm -f "${PROD_DIR}.zip" - -mkdir -p "$PROD_DIR" - -echo "📂 Copying application components (excluding dev artifacts)..." -# Core application -rsync -a --exclude 'node_modules' --exclude '.next' frontend/ "$PROD_DIR/frontend/" -rsync -a --exclude '__pycache__' --exclude '.pytest_cache' --exclude '.venv' --exclude 'tests' backend/ "$PROD_DIR/backend/" - -# Orchestration, Config & Scripts -mkdir -p "$PROD_DIR/config" "$PROD_DIR/scripts" -cp docker-compose.yml "$PROD_DIR/" -rsync -a config/ "$PROD_DIR/config/" -rsync -a scripts/ "$PROD_DIR/scripts/" -cp start_server.sh "$PROD_DIR/" -cp run_standalone.sh "$PROD_DIR/" -cp install_service.sh "$PROD_DIR/" -cp inventory.service.template "$PROD_DIR/" -cp USER_GUIDE.md "$PROD_DIR/" -cp README.md "$PROD_DIR/INSTALLATION_GUIDE.md" -cp inventory.env "$PROD_DIR/" -cp deploy.sh "$PROD_DIR/" -cp .git_path "$PROD_DIR/" 2>/dev/null || true -cp frontend/VERSION.json "$PROD_DIR/" -cp frontend/VERSION.json "$PROD_DIR/frontend/" - -# Setup persistent volume skeleton -mkdir -p "$PROD_DIR/data" -mkdir -p "$PROD_DIR/logs" -# Place a README in the root of the release -cat < "$PROD_DIR/README.txt" -TFM aInventory - v${VERSION} -============================= - -This is a clean production build, free of development or AI-agent constraints. - -TO RUN VIA DOCKER (Recommended): -1. Install Docker Desktop or Docker Engine. -2. Run: docker-compose build -3. Run: docker-compose up -d -4. Access via https://:8909 (Accept the internal security warning). - -TO INSTALL AS A LINUX SYSTEM SERVICE (Optional): -1. sudo ./install_service.sh -2. sudo systemctl start inventory - -TO RUN BARE-METAL (No Docker): -1. Install Python 3.12+ and Node.js 20+. -2. Ensure you have network access for npm installs. -3. Run: ./start_server.sh -4. Access via https://:8909 - -Note: Database and Logs will persist in the /data and /logs directories. -EOF - -echo "🗜️ Zipping the final bundle..." -zip -r -q "${PROD_DIR}.zip" "$PROD_DIR" - -# Optional: cleanup the directory to leave just the zip -# rm -rf "$PROD_DIR" - -echo "✅ SUCCESS: The clean production archive is ready: ${PROD_DIR}.zip" diff --git a/frontend/VERSION.json b/frontend/VERSION.json index 592064e9..36109769 100644 --- a/frontend/VERSION.json +++ b/frontend/VERSION.json @@ -1,6 +1,6 @@ { - "version": "1.13.0", - "last_build": "2026-04-21-1205", + "version": "1.13.1", + "last_build": "2026-04-23-1256", "codename": "PhotoUI", - "commit": "ca68aeae" + "commit": "d9dffdc3" } \ No newline at end of file diff --git a/install_service.sh b/install_service.sh deleted file mode 100755 index 99eaea79..00000000 --- a/install_service.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/bash -# install_service.sh - Installs the inventory system as a standalone systemd service - -if [[ $EUID -ne 0 ]]; then - echo "🚫 This script must be run as root (use sudo)" - exit 1 -fi - -echo "⚙️ Installing TFM aInventory as a Standalone Linux service..." - -# Detect working directory -WORKING_DIR="$(pwd)" - -# 1. Dependency Checks -echo "🔍 Checking dependencies..." -for cmd in python3 node npm; do - if ! command -v $cmd &> /dev/null; then - echo "❌ $cmd not found! Please install it before proceeding." - exit 1 - fi -done - -# 2. Setup Backend Environment -echo "🐍 Setting up Python Virtual Environment..." -if [ ! -d ".venv" ]; then - python3 -m venv .venv -fi -source .venv/bin/activate -pip install -q --upgrade pip -pip install -q -r backend/requirements.txt - -# 3. Setup Frontend Environment -echo "📦 Installing Node dependencies (this may take a minute)..." -cd frontend -npm install --quiet -echo "🏗️ Building Frontend for Production (Next.js build)..." -npm run build -cd .. - -# 4. Create the final service file from template -TEMPLATE="inventory.service.template" -TARGET="/etc/systemd/system/inventory.service" - -if [ ! -f "$TEMPLATE" ]; then - echo "❌ Template file $TEMPLATE not found in current directory!" - exit 1 -fi - -sed "s|__WORKING_DIR__|$WORKING_DIR|g" "$TEMPLATE" > "$TARGET" - -echo "📝 Service file created at $TARGET" - -# 5. Reload and enable -systemctl daemon-reload -systemctl enable inventory.service - -# 6. Ensure scripts are executable -chmod +x run_standalone.sh -chmod +x start_server.sh - -echo "" -echo "🚀 TFM aInventory Standalone service installed and enabled!" -echo " Commands:" -echo " 👉 sudo systemctl start inventory" -echo " 👉 sudo systemctl status inventory" -echo " 👉 sudo systemctl stop inventory" -echo "" -echo "Note: The service is currently ENABLED to start on boot, but NOT started." -echo " Run 'sudo systemctl start inventory' to launch it now." diff --git a/inventory.env b/inventory.env deleted file mode 100644 index ab70c29e..00000000 --- a/inventory.env +++ /dev/null @@ -1,32 +0,0 @@ -# ============================================================================= -# TFM aInventory - Docker Compose Environment -# ============================================================================= -# This file is used by Docker Compose for host-side port mapping. -# It should match the values in config/network_config.env -# ============================================================================= - -SERVER_IP=192.168.84.131 - -# Backend Ports -BACKEND_PORT=8916 -BACKEND_SSL_PORT=8918 - -# Frontend Ports -FRONTEND_PORT=8917 -FRONTEND_SSL_PORT=8919 - -# Security -JWT_SECRET_KEY=change_me_in_production - -# AI -GEMINI_API_KEY=AIzaSyAajthWG2agpDLyJHY11U5qFLP4WnV5z0w -CLAUDE_API_KEY=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 - -# External Access (CORS) -# Comma-separated list of extra IPs or FQDNs allowed to connect (e.g. Tailscale, VPN) -EXTRA_ALLOWED_ORIGINS=100.78.182.0/24 - -# Data and Logging (for standalone deployment) -DATA_DIR=./data -LOGS_DIR=./logs -LOG_LEVEL=INFO diff --git a/inventory.env.example b/inventory.env.example deleted file mode 100644 index 153ec8a5..00000000 --- a/inventory.env.example +++ /dev/null @@ -1,37 +0,0 @@ -# ============================================================================= -# TFM aInventory — Master Environment Configuration (v1.10.15) -# ============================================================================= -# 1. Copy this file to 'inventory.env' -# 2. Fill in your real values -# 3. 'inventory.env' is ignored by Git to protect your secrets. -# -# SYSTEM REQUIREMENTS (before running ./start_server.sh): -# - Node.js v20+ (required for frontend build) -# - Python 3.12+ with python3.12-venv package -# On Debian/Ubuntu: sudo apt install python3.12-venv -# ============================================================================= - -# --- Network & Identity --- -# Use your LAN IP (e.g., 192.168.1.10) for mobile access. -SERVER_IP=localhost - -# --- AI API Keys --- -# Google Gemini API Key (Required for AI label OCR onboarding) -# Get one at: https://aistudio.google.com/ -GEMINI_API_KEY=your_gemini_api_key_here - -# --- Security --- -# JWT secret key — generate a strong random value for production: -# python3 -c "import secrets; print(secrets.token_urlsafe(64))" -JWT_SECRET_KEY=change_me_in_production - -# --- Infrastructure Ports (Host-side mapping) --- -BACKEND_PORT=8916 -BACKEND_SSL_PORT=8918 -FRONTEND_PORT=8917 -FRONTEND_SSL_PORT=8919 - -# --- External Access (CORS) --- -# Comma-separated list of extra IPs or FQDNs allowed (e.g. Tailscale, VPN) -# Example: EXTRA_ALLOWED_ORIGINS=100.78.182.27,inventory.my-domain.com -EXTRA_ALLOWED_ORIGINS= diff --git a/run_standalone.sh b/run_standalone.sh deleted file mode 100755 index 89eb4e95..00000000 --- a/run_standalone.sh +++ /dev/null @@ -1,70 +0,0 @@ -#!/bin/bash -# run_standalone.sh - Headless production launcher for TFM aInventory -# manages Backend, Frontend, and SSL Proxies in a single process group. - -echo "🚀 Starting TFM aInventory in Standalone Mode..." - -# Trapping termination signals to clean up child processes -trap "echo 'Stopping all processes...'; kill 0" SIGINT SIGTERM EXIT - -# --- CONFIGURATION (Default values, overridden by network_configinventory.env) --- -BACKEND_PORT=8000 -FRONTEND_PORT=3001 -BACKEND_SSL_PORT=3002 -FRONTEND_SSL_PORT=3003 -SERVER_IP="localhost" - -# Load Configuration from file if it exists -CONFIG_PATH="$(cd "$(dirname "$0")" && pwd)/inventory.env" -if [ -f "$CONFIG_PATH" ]; then - echo "⚙️ Loading network configuration from $CONFIG_PATH..." - export $(grep -v '^#' "$CONFIG_PATH" | xargs) -fi - -# 1. Activate Environment -if [ -d ".venv" ]; then - source .venv/bin/activate -fi - -# 1.5 Sync Network Config to Frontend -echo "🔌 Syncing network configuration to frontend..." -mkdir -p frontend/public -cat < frontend/public/network.json -{ - "SERVER_IP": "$SERVER_IP", - "BACKEND_PORT": $BACKEND_PORT, - "BACKEND_SSL_PORT": $BACKEND_SSL_PORT, - "FRONTEND_PORT": $FRONTEND_PORT, - "FRONTEND_SSL_PORT": $FRONTEND_SSL_PORT -} -EOF - -# 2. Start Backend (No Reload for Prod) -echo "🔥 Starting Backend (Uvicorn)..." -python3 -m uvicorn backend.main:app --host 0.0.0.0 --port $BACKEND_PORT & - -# 3. Start Frontend (Production Start) -echo "💻 Starting Frontend (Next.js Prod)..." -cd frontend -npm run start -- -p $FRONTEND_PORT & -cd .. - -# 4. Start Proxies (via npx) -echo "🛡️ Starting HTTPS Proxies..." -npx local-ssl-proxy --source $BACKEND_SSL_PORT --target $BACKEND_PORT --hostname 0.0.0.0 > /dev/null 2>&1 & -npx local-ssl-proxy --source $FRONTEND_SSL_PORT --target $FRONTEND_PORT --hostname 0.0.0.0 > /dev/null 2>&1 & - -# 5. Detection of IP for logs -if [[ "$OSTYPE" == "darwin"* ]]; then - LOCAL_IP=$(ipconfig getifaddr en0 || ipconfig getifaddr en1 || echo "localhost") -else - LOCAL_IP=$(hostname -I | awk '{print $1}') -fi - -echo "" -echo "✅ TFM aInventory is active at https://$LOCAL_IP:$FRONTEND_SSL_PORT" -echo " Processes are running in background. Monitoring logs..." -echo "" - -# Wait for children -wait diff --git a/scripts/deploy.py b/scripts/deploy.py new file mode 100755 index 00000000..c572d80a --- /dev/null +++ b/scripts/deploy.py @@ -0,0 +1,324 @@ +#!/usr/bin/env python3 +""" +TFM aInventory - Docker Deployment Script (v1.12.0) +Converted from deploy.sh to Python per Decision D-05. +""" + +import os +import sys +import yaml +import argparse +import subprocess +import time +import socket +import logging +from typing import Dict, Any, List, Optional + +# Color codes for terminal output +class Colors: + RED = '\033[0;31m' + GREEN = '\033[0;32m' + YELLOW = '\033[1;33m' + BLUE = '\033[0;34m' + CYAN = '\033[0;36m' + NC = '\033[0m' # No Color + +class ColoredFormatter(logging.Formatter): + format_str = "%(asctime)s - %(levelname)s - %(message)s" + + FORMATS = { + logging.DEBUG: Colors.CYAN + format_str + Colors.NC, + logging.INFO: Colors.BLUE + format_str + Colors.NC, + logging.WARNING: Colors.YELLOW + format_str + Colors.NC, + logging.ERROR: Colors.RED + format_str + Colors.NC, + logging.CRITICAL: Colors.RED + format_str + Colors.NC + } + + def format(self, record): + log_fmt = self.FORMATS.get(record.levelno) + formatter = logging.Formatter(log_fmt, datefmt='%H:%M:%S') + return formatter.format(record) + +# Setup logging +logger = logging.getLogger("deploy") +logger.setLevel(logging.INFO) +ch = logging.StreamHandler() +ch.setFormatter(ColoredFormatter()) +logger.addHandler(ch) + +def run_command(cmd: List[str], capture_output: bool = False, env: Optional[Dict[str, str]] = None) -> subprocess.CompletedProcess: + """Run a system command securely with shell=False.""" + try: + return subprocess.run( + cmd, + check=True, + capture_output=capture_output, + text=True, + shell=False, + env=env or os.environ.copy() + ) + except subprocess.CalledProcessError as e: + logger.error(f"Command failed: {' '.join(cmd)}") + if e.stdout: logger.error(f"STDOUT: {e.stdout}") + if e.stderr: logger.error(f"STDERR: {e.stderr}") + raise + +def load_yaml(file_path: str) -> Dict[str, Any]: + """Load and parse a YAML file.""" + if not os.path.exists(file_path): + logger.warning(f"Config file {file_path} not found. Using empty defaults.") + return {} + try: + with open(file_path, 'r') as f: + return yaml.safe_load(f) or {} + except Exception as e: + logger.error(f"Failed to parse {file_path}: {e}") + return {} + +def is_port_in_use(port: int) -> bool: + """Check if a port is in use on localhost.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(('localhost', port)) == 0 + +def check_disk_space(path: str = ".", required_gb: int = 10) -> bool: + """Check if there is sufficient disk space.""" + stat = os.statvfs(path) + free_gb = (stat.f_bavail * stat.f_frsize) / (1024**3) + if free_gb < required_gb: + logger.warning(f"Low disk space: {free_gb:.2f} GB available (recommended: {required_gb} GB)") + return False + logger.info(f"Sufficient disk space: {free_gb:.2f} GB available") + return True + +def pre_flight_checks(): + """Step 1-5: System and config checks.""" + logger.info("Step 1/12: Running pre-flight checks...") + + # Check Docker + try: + run_command(["docker", "--version"], capture_output=True) + logger.info(" ✓ Docker is installed") + except (subprocess.CalledProcessError, FileNotFoundError): + logger.critical("Docker not installed. Please install Docker 24.0+") + sys.exit(1) + + # Check Docker Compose + try: + # Try 'docker compose' first (v2 plugin) + run_command(["docker", "compose", "version"], capture_output=True) + docker_compose_cmd = ["docker", "compose"] + logger.info(" ✓ Docker Compose (v2) is installed") + except (subprocess.CalledProcessError, FileNotFoundError): + try: + # Fallback to 'docker-compose' (v1) + run_command(["docker-compose", "--version"], capture_output=True) + docker_compose_cmd = ["docker-compose"] + logger.info(" ✓ docker-compose (v1) is installed") + except (subprocess.CalledProcessError, FileNotFoundError): + logger.critical("Docker Compose not installed. Please install Docker Compose 2.0+") + sys.exit(1) + + # Check docker-compose.yml + if not os.path.exists("docker-compose.yml"): + logger.critical("docker-compose.yml not found in current directory") + sys.exit(1) + logger.info(" ✓ docker-compose.yml found") + + # Check configs + required_configs = ["config/backend.yaml", "config/network.yaml"] + for cfg in required_configs: + if not os.path.exists(cfg): + logger.error(f"Required config file {cfg} missing.") + example = cfg + ".example" + if os.path.exists(example): + logger.info(f"Suggestion: Copy {example} to {cfg} and customize it.") + sys.exit(1) + logger.info(f" ✓ {cfg} found") + + return docker_compose_cmd + +def main(): + parser = argparse.ArgumentParser(description="TFM aInventory Docker Deployment Script") + parser.add_argument("environment", nargs="?", default="production", + choices=["production", "staging", "development"], + help="Deployment environment (default: production)") + parser.add_argument("--rebuild", action="store_true", help="Force rebuild of Docker images") + parser.add_argument("--verbose", action="store_true", help="Enable debug logging") + + args = parser.parse_args() + + if args.verbose: + logger.setLevel(logging.DEBUG) + + logger.info("=== TFM aInventory Deployment Script ===") + logger.info(f"Environment: {args.environment}") + logger.info(f"Rebuild: {'Yes' if args.rebuild else 'No'}") + + # 1-5 Pre-flight + docker_compose_cmd = pre_flight_checks() + + # Load Configs + docker_cfg = load_yaml("config/docker.yaml") + network_cfg = load_yaml("config/network.yaml") + backend_cfg = load_yaml("config/backend.yaml") + + # 6. Port availability check + logger.info("Step 6/12: Checking port availability...") + ports_cfg = network_cfg.get("ports", {}) + ports_to_check = { + "Backend HTTP": ports_cfg.get("backend_port", 8916), + "Frontend HTTP": ports_cfg.get("frontend_port", 8917), + "Backend HTTPS": ports_cfg.get("backend_ssl_port", 8918), + "Frontend HTTPS": ports_cfg.get("frontend_ssl_port", 8919), + } + + ports_busy = False + for name, port in ports_to_check.items(): + if is_port_in_use(port): + logger.error(f"Port {port} ({name}) is already in use!") + ports_busy = True + else: + logger.debug(f"Port {port} ({name}) is available") + + if ports_busy: + logger.critical("Some required ports are occupied. Please free them or change config/network.yaml") + sys.exit(1) + logger.info(" ✓ All required ports are available") + + # 7. Environment validation + logger.info("Step 7/12: Validating backend configuration...") + auth_cfg = backend_cfg.get("auth", {}) + jwt_secret = auth_cfg.get("jwt_secret_key") + if not jwt_secret or jwt_secret == "change_me_in_production": + if args.environment == "production": + logger.error("JWT_SECRET_KEY is missing or insecure in config/backend.yaml") + logger.info("Generate one with: openssl rand -hex 32") + sys.exit(1) + else: + logger.warning("Using insecure JWT_SECRET_KEY (acceptable for non-production)") + + ai_cfg = backend_cfg.get("ai", {}) + if ai_cfg.get("gemini_api_key") == "your-gemini-api-key": + logger.warning("Gemini API key is still the default placeholder") + if ai_cfg.get("claude_api_key") == "your-claude-api-key": + logger.warning("Claude API key is still the default placeholder") + + logger.info(" ✓ Backend configuration validated") + + # 4 (Disk space - extra check) + check_disk_space() + + # Prepare environment variables for Docker Compose + # We map YAML values to the environment variables expected by docker-compose.yml + env = os.environ.copy() + env["BACKEND_PORT"] = str(ports_to_check["Backend HTTP"]) + env["FRONTEND_PORT"] = str(ports_to_check["Frontend HTTP"]) + env["BACKEND_SSL_PORT"] = str(ports_to_check["Backend HTTPS"]) + env["FRONTEND_SSL_PORT"] = str(ports_to_check["Frontend HTTPS"]) + env["JWT_SECRET_KEY"] = jwt_secret or "change_me_in_production" + + # 8. Docker Compose Build + logger.info("Step 8/12: Building/Pulling Docker images...") + build_cmd = docker_compose_cmd + ["build"] + if args.rebuild: + build_cmd.append("--no-cache") + + try: + run_command(build_cmd, env=env) + logger.info(" ✓ Docker images prepared") + except Exception: + logger.critical("Docker build failed") + sys.exit(1) + + # 9. Preparing data directories + logger.info("Step 9/12: Preparing data directories...") + dirs = ["data", "logs", "config", "data/caddy_data", "data/caddy_config"] + for d in dirs: + os.makedirs(d, exist_ok=True) + # Note: chmod -R 777 is used in bash script, though slightly insecure, + # we'll keep it if it's necessary for the containers to write. + # subprocess.run(["chmod", "-R", "777", "data", "logs", "config"]) + logger.info(" ✓ Data directories ready") + + # 10. Start Services + logger.info("Step 10/12: Starting Docker services...") + up_cmd = docker_compose_cmd + ["up", "-d"] + try: + run_command(up_cmd, env=env) + logger.info(" ✓ Services started in background") + except Exception: + logger.critical("Failed to start Docker services") + sys.exit(1) + + # 11. Health checks + logger.info("Step 11/12: Waiting for services to become healthy...") + max_attempts = 30 + all_healthy = False + + for attempt in range(1, max_attempts + 1): + try: + ps_out = run_command(docker_compose_cmd + ["ps"], capture_output=True, env=env).stdout + # Simple check for health status in 'docker compose ps' output + # Usually it says '(healthy)' + if ps_out.count("(healthy)") >= 3: + all_healthy = True + break + # Newer docker compose versions might just show 'Running' or 'Up' but with health status + # If ps doesn't show health clearly, we can try curl + except Exception: + pass + + logger.info(f" Waiting... ({attempt}/{max_attempts})") + time.sleep(4) + + if all_healthy: + logger.info(" ✓ All services are healthy") + else: + logger.warning("Services did not report healthy via Docker. Checking via curl...") + # Fallback to manual connectivity check + + # 12. Connectivity & Report + logger.info("Step 12/12: Verifying service connectivity...") + + backend_url = f"http://localhost:{ports_to_check['Backend HTTP']}" + frontend_url = f"http://localhost:{ports_to_check['Frontend HTTP']}" + + backend_ok = False + for _ in range(5): + try: + # We use subprocess curl for simplicity as requested, avoiding external libs like 'requests' + res = run_command(["curl", "-sf", f"{backend_url}/health"], capture_output=True) + if res.returncode == 0: + backend_ok = True + break + except: + time.sleep(2) + + if backend_ok: + logger.info(f" ✓ Backend API responding at {backend_url}/health") + else: + logger.warning(f" Backend health check failed at {backend_url}/health") + + # Report + print(f"\n{Colors.GREEN}╔════════════════════════════════════════════════════════════╗{Colors.NC}") + print(f"{Colors.GREEN}║{Colors.NC} Deployment completed successfully! {Colors.GREEN}║{Colors.NC}") + print(f"{Colors.GREEN}╚════════════════════════════════════════════════════════════╝{Colors.NC}\n") + + print("Access points:") + print(f" Frontend (HTTP): {frontend_url}") + print(f" Backend (HTTP): {backend_url}") + print(f" API Docs: {backend_url}/docs") + print(f" Frontend (HTTPS): https://localhost:{ports_to_check['Frontend HTTPS']}") + print(f" Backend (HTTPS): https://localhost:{ports_to_check['Backend HTTPS']}") + print("\nUseful commands:") + print(f" View logs: {' '.join(docker_compose_cmd)} logs -f") + print(f" Stop services: {' '.join(docker_compose_cmd)} down") + print(f" Status: {' '.join(docker_compose_cmd)} ps") + + if args.environment == "production": + print(f"\n{Colors.YELLOW}Production Notes:{Colors.NC}") + print(" • Ensure firewall allows only required ports") + print(" • Set up automated backups using scripts/export_prod.py") + +if __name__ == "__main__": + main() diff --git a/scripts/export_prod.py b/scripts/export_prod.py new file mode 100755 index 00000000..ef9cd598 --- /dev/null +++ b/scripts/export_prod.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python3 +""" +TFM aInventory - Production Export/Backup Script (v1.12.0) +Converted from export_prod.sh (as a backup tool) to Python per Decision D-05. +""" + +import os +import sys +import yaml +import argparse +import tarfile +import logging +from datetime import datetime +from typing import Dict, Any, List + +# Color codes +class Colors: + RED = '\033[0;31m' + GREEN = '\033[0;32m' + YELLOW = '\033[1;33m' + BLUE = '\033[0;34m' + NC = '\033[0m' + +# Setup logging +logging.basicConfig(level=logging.INFO, format=f"{Colors.BLUE}[INFO]{Colors.NC} %(message)s") +logger = logging.getLogger("export_prod") + +def load_yaml(file_path: str) -> Dict[str, Any]: + if not os.path.exists(file_path): + return {} + try: + with open(file_path, 'r') as f: + return yaml.safe_load(f) or {} + except Exception as e: + logger.error(f"Failed to parse {file_path}: {e}") + return {} + +def main(): + parser = argparse.ArgumentParser(description="TFM aInventory Production Export/Backup Tool") + parser.add_argument("--output", help="Path to the output tar.gz file") + parser.add_argument("--include-logs", action="store_true", help="Include log files in the backup") + + args = parser.parse_args() + + # Load config to find data and logs directories + backend_cfg = load_yaml("config/backend.yaml") + app_cfg = backend_cfg.get("application", {}) + + data_dir = app_cfg.get("data_dir", "./data") + logs_dir = app_cfg.get("logs_dir", "./logs") + + timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S") + + if args.output: + output_path = args.output + else: + os.makedirs("backups", exist_ok=True) + output_path = f"backups/ainventory_{timestamp}.tar.gz" + + logger.info(f"Creating production backup: {output_path}") + + # Files and directories to include + to_include = [] + + # Data directory (contains database) + if os.path.exists(data_dir): + to_include.append(data_dir) + else: + logger.warning(f"Data directory {data_dir} not found. Skipping.") + + # Config files (specific ones) + config_files = [ + "config/backend.yaml", + "config/frontend.yaml", + "config/network.yaml", + "config/backend.yaml.example", + "config/frontend.yaml.example", + "config/network.yaml.example", + "config/docker.yaml.example", + "config/secrets.yaml.example" + ] + for cf in config_files: + if os.path.exists(cf): + to_include.append(cf) + + # Optional logs + if args.include_logs and os.path.exists(logs_dir): + to_include.append(logs_dir) + + # Exclude patterns + exclude_files = ["config/secrets.yaml"] + exclude_dirs = ["node_modules", "__pycache__", ".git", ".venv", ".next", ".pytest_cache"] + + def filter_tar(tarinfo): + # Exclude specific files + if tarinfo.name in exclude_files: + logger.info(f" Excluding sensitive file: {tarinfo.name}") + return None + + # Exclude directories by name + for d in exclude_dirs: + if f"/{d}/" in f"/{tarinfo.name}/": + return None + + return tarinfo + + try: + with tarfile.open(output_path, "w:gz") as tar: + for item in to_include: + logger.info(f" Adding {item}...") + tar.add(item, filter=filter_tar) + + # Check size + size_bytes = os.path.getsize(output_path) + size_mb = size_bytes / (1024 * 1024) + + logger.info(f"{Colors.GREEN}✓ Backup created successfully!{Colors.NC}") + logger.info(f" Archive: {output_path}") + logger.info(f" Size: {size_mb:.2f} MB") + logger.info(f" Contents: Data and config (secrets excluded)") + + except Exception as e: + logger.error(f"Failed to create backup: {e}") + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/scripts/install_service.py b/scripts/install_service.py new file mode 100755 index 00000000..86d2139a --- /dev/null +++ b/scripts/install_service.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python3 +""" +TFM aInventory - Systemd Service Installer (v1.12.0) +Converted from install_service.sh to Python per Decision D-05. +""" + +import os +import sys +import yaml +import argparse +import subprocess +import logging +import pwd +import grp +from typing import Dict, Any + +# Color codes +class Colors: + RED = '\033[0;31m' + GREEN = '\033[0;32m' + YELLOW = '\033[1;33m' + BLUE = '\033[0;34m' + NC = '\033[0m' + +# Setup logging +logging.basicConfig(level=logging.INFO, format=f"{Colors.BLUE}[INFO]{Colors.NC} %(message)s") +logger = logging.getLogger("install_service") + +SERVICE_NAME = "ainventory.service" +SERVICE_PATH = f"/etc/systemd/system/{SERVICE_NAME}" + +# Inline template if file missing +DEFAULT_TEMPLATE = """[Unit] +Description=TFM aInventory Service +After=network.target + +[Service] +Type=simple +User={SERVICE_USER} +Group={SERVICE_USER} +WorkingDirectory={PROJECT_DIR} +Environment=PATH={PROJECT_DIR}/.venv/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin +ExecStart={PYTHON_PATH} {PROJECT_DIR}/scripts/run_standalone.py --backend-only +Restart=on-failure +RestartSec=10 +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target +""" + +def load_yaml(file_path: str) -> Dict[str, Any]: + if not os.path.exists(file_path): + return {} + try: + with open(file_path, 'r') as f: + return yaml.safe_load(f) or {} + except Exception as e: + logger.error(f"Failed to parse {file_path}: {e}") + return {} + +def check_user_exists(username: str) -> bool: + try: + pwd.getpwnam(username) + return True + except KeyError: + return False + +def main(): + if os.getuid() != 0: + logger.error("This script must be run as root (use sudo)") + sys.exit(1) + + parser = argparse.ArgumentParser(description="TFM aInventory Systemd Service Installer") + parser.add_argument("--user", default="www-data", help="Service user (default: www-data)") + parser.add_argument("--force", action="store_true", help="Overwrite existing service file") + + args = parser.parse_args() + + project_dir = os.getcwd() + python_path = sys.executable + + # Load configs + backend_cfg = load_yaml("config/backend.yaml") + network_cfg = load_yaml("config/network.yaml") + + app_cfg = backend_cfg.get("application", {}) + ports_cfg = network_cfg.get("ports", {}) + + backend_port = ports_cfg.get("backend_port", 8916) + data_dir = os.path.abspath(app_cfg.get("data_dir", "./data")) + logs_dir = os.path.abspath(app_cfg.get("logs_dir", "./logs")) + + logger.info(f"Installing {SERVICE_NAME}...") + + if not check_user_exists(args.user): + logger.warning(f"User '{args.user}' does not exist. Suggestion: sudo useradd -r -s /bin/false {args.user}") + # We'll continue but warn, as user might want to create it later (though service won't start) + + if os.path.exists(SERVICE_PATH) and not args.force: + logger.error(f"Service file {SERVICE_PATH} already exists. Use --force to overwrite.") + sys.exit(1) + + # Read template if exists + template_content = DEFAULT_TEMPLATE + if os.path.exists("inventory.service.template"): + try: + with open("inventory.service.template", "r") as f: + content = f.read() + if content.strip(): + template_content = content + # If using old template with __WORKING_DIR__, replace it + template_content = template_content.replace("__WORKING_DIR__", "{PROJECT_DIR}") + except Exception as e: + logger.warning(f"Failed to read inventory.service.template: {e}. Using default.") + + # Format template + # Note: We need to handle keys that might not be in the template but are in our variables + # The plan mentions {PROJECT_DIR}, {SERVICE_USER}, {BACKEND_PORT}, {DATA_DIR}, {LOGS_DIR} + try: + # We'll use a safer way to replace placeholders to avoid KeyError if template doesn't have all of them + service_content = template_content.format( + PROJECT_DIR=project_dir, + SERVICE_USER=args.user, + BACKEND_PORT=backend_port, + DATA_DIR=data_dir, + LOGS_DIR=logs_dir, + PYTHON_PATH=python_path + ) + except KeyError as e: + # Fallback if template has unknown placeholders + logger.warning(f"Template contains unknown placeholder: {e}. Attempting simple replacement.") + service_content = template_content.replace("{PROJECT_DIR}", project_dir)\ + .replace("{SERVICE_USER}", args.user)\ + .replace("{BACKEND_PORT}", str(backend_port))\ + .replace("{DATA_DIR}", data_dir)\ + .replace("{LOGS_DIR}", logs_dir)\ + .replace("{PYTHON_PATH}", python_path) + + # Write service file + try: + with open(SERVICE_PATH, "w") as f: + f.write(service_content) + os.chmod(SERVICE_PATH, 0o644) + logger.info(f"✓ Service file created at {SERVICE_PATH}") + except Exception as e: + logger.error(f"Failed to write service file: {e}") + sys.exit(1) + + # Ensure data and logs directories exist and are owned by the service user + for d in [data_dir, logs_dir]: + os.makedirs(d, exist_ok=True) + try: + uid = pwd.getpwnam(args.user).pw_uid + gid = grp.getgrnam(args.user).gr_gid + os.chown(d, uid, gid) + # Also chown contents if any + for root, dirs, files in os.walk(d): + for momo in dirs: os.chown(os.path.join(root, momo), uid, gid) + for momo in files: os.chown(os.path.join(root, momo), uid, gid) + except Exception as e: + logger.warning(f"Could not set permissions for {d}: {e}") + + # Reload systemd + try: + subprocess.run(["systemctl", "daemon-reload"], check=True) + subprocess.run(["systemctl", "enable", SERVICE_NAME], check=True) + logger.info(f"✓ {SERVICE_NAME} enabled") + except subprocess.CalledProcessError as e: + logger.error(f"Failed to enable service: {e}") + sys.exit(1) + + print(f"\n{Colors.GREEN}🚀 TFM aInventory service installed successfully!{Colors.NC}") + print(f"Service user: {args.user}") + print(f"Project dir: {project_dir}") + print("\nNext steps:") + print(f" 👉 sudo systemctl start {SERVICE_NAME}") + print(f" 👉 sudo systemctl status {SERVICE_NAME}") + print(f" 👉 journalctl -u {SERVICE_NAME} -f") + +if __name__ == "__main__": + main() diff --git a/scripts/run_standalone.py b/scripts/run_standalone.py new file mode 100755 index 00000000..0b4714fc --- /dev/null +++ b/scripts/run_standalone.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +""" +TFM aInventory - Standalone Launcher (v1.12.0) +Converted from run_standalone.sh to Python per Decision D-05. +""" + +import os +import sys +import yaml +import argparse +import subprocess +import signal +import time +import threading +import logging +import socket +from typing import List, Optional, Dict, Any + +# Color codes +class Colors: + RED = '\033[0;31m' + GREEN = '\033[0;32m' + YELLOW = '\033[1;33m' + BLUE = '\033[0;34m' + MAGENTA = '\033[0;35m' + CYAN = '\033[0;36m' + NC = '\033[0m' + +# Setup logging +logging.basicConfig(level=logging.INFO, format=f"{Colors.BLUE}[INFO]{Colors.NC} %(message)s") +logger = logging.getLogger("run_standalone") + +processes: List[subprocess.Popen] = [] +shutdown_event = threading.Event() + +def signal_handler(sig, frame): + logger.info("Shutdown signal received. Stopping processes...") + shutdown_event.set() + for p in processes: + try: + p.terminate() + except: + pass + # Give them a moment to terminate gracefully + time.sleep(1) + for p in processes: + try: + p.kill() + except: + pass + sys.exit(0) + +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) + +def load_yaml(file_path: str) -> Dict[str, Any]: + if not os.path.exists(file_path): + logger.warning(f"Config file {file_path} not found. Using defaults.") + return {} + try: + with open(file_path, 'r') as f: + return yaml.safe_load(f) or {} + except Exception as e: + logger.error(f"Failed to parse {file_path}: {e}") + return {} + +def is_port_in_use(port: int) -> bool: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + return s.connect_ex(('localhost', port)) == 0 + +def log_stream(stream, prefix, color): + for line in iter(stream.readline, ''): + if shutdown_event.is_set(): + break + if line: + print(f"{color}[{prefix}]{Colors.NC} {line.strip()}") + stream.close() + +def start_backend(backend_port: int, backend_cfg: Dict[str, Any]): + logger.info(f"Starting Backend on port {backend_port}...") + + app_cfg = backend_cfg.get("application", {}) + env = os.environ.copy() + env["DATA_DIR"] = app_cfg.get("data_dir", "./data") + env["LOGS_DIR"] = app_cfg.get("logs_dir", "./logs") + env["LOG_LEVEL"] = backend_cfg.get("logging", {}).get("log_level", "INFO") + + # Ensure directories exist + os.makedirs(env["DATA_DIR"], exist_ok=True) + os.makedirs(env["LOGS_DIR"], exist_ok=True) + + cmd = [ + sys.executable, "-m", "uvicorn", + "backend.main:app", + "--host", "0.0.0.0", + "--port", str(backend_port) + ] + + # Add --reload if in dev mode (detectable via environment or flag, + # but the script is for 'run_standalone' which might be used for prod too) + # For now, let's keep it simple. + + try: + p = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + bufsize=1, + universal_newlines=True + ) + processes.append(p) + + threading.Thread(target=log_stream, args=(p.stdout, "BACKEND", Colors.GREEN), daemon=True).start() + threading.Thread(target=log_stream, args=(p.stderr, "BACKEND", Colors.GREEN), daemon=True).start() + + return p + except FileNotFoundError: + logger.error("uvicorn not found. Please install it with: pip install uvicorn") + sys.exit(1) + +def start_frontend(frontend_port: int, frontend_cfg: Dict[str, Any]): + logger.info(f"Starting Frontend on port {frontend_port}...") + + api_cfg = frontend_cfg.get("api", {}) + env = os.environ.copy() + env["NEXT_PUBLIC_API_URL"] = api_cfg.get("backend_url", "http://localhost:8916") + env["PORT"] = str(frontend_port) + + # Check if we should use 'npm run dev' or 'node server.js' + # For standalone, we might prefer 'npm run start' if built, or 'npm run dev' + # Bash script used 'npm run start' + + cmd = ["npm", "run", "dev"] + if os.path.exists("frontend/server.js"): + # If built for production + cmd = ["node", "frontend/server.js"] + + try: + p = subprocess.Popen( + cmd, + cwd="frontend" if os.path.isdir("frontend") else None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + env=env, + bufsize=1, + universal_newlines=True + ) + processes.append(p) + + threading.Thread(target=log_stream, args=(p.stdout, "FRONTEND", Colors.MAGENTA), daemon=True).start() + threading.Thread(target=log_stream, args=(p.stderr, "FRONTEND", Colors.MAGENTA), daemon=True).start() + + return p + except FileNotFoundError: + logger.error("npm or node not found. Please install Node.js") + sys.exit(1) + +def main(): + parser = argparse.ArgumentParser(description="TFM aInventory Standalone Launcher") + group = parser.add_mutually_exclusive_group() + group.add_argument("--backend-only", action="store_true", help="Start only the backend") + group.add_argument("--frontend-only", action="store_true", help="Start only the frontend") + + args = parser.parse_args() + + # Load all configs to find ports and settings + backend_cfg = load_yaml("config/backend.yaml") + frontend_cfg = load_yaml("config/frontend.yaml") + network_cfg = load_yaml("config/network.yaml") + + ports_cfg = network_cfg.get("ports", {}) + backend_port = ports_cfg.get("backend_port", 8916) + frontend_port = ports_cfg.get("frontend_port", 8917) + + if not args.frontend_only: + if is_port_in_use(backend_port): + logger.error(f"Port {backend_port} is already in use. Cannot start backend.") + if not args.backend_only: + # If we are supposed to start both, we fail. + sys.exit(1) + else: + start_backend(backend_port, backend_cfg) + + if not args.backend_only: + if is_port_in_use(frontend_port): + logger.error(f"Port {frontend_port} is already in use. Cannot start frontend.") + if not args.frontend_only: + sys.exit(1) + else: + start_frontend(frontend_port, frontend_cfg) + + if not processes: + logger.error("No processes started. Check configuration and port availability.") + sys.exit(1) + + logger.info("All services started. Press Ctrl+C to stop.") + + # Monitor processes + try: + while not shutdown_event.is_set(): + time.sleep(1) + for p in processes: + if p.poll() is not None: + logger.error(f"Process {p.pid} exited with code {p.returncode}") + shutdown_event.set() + break + except KeyboardInterrupt: + pass + finally: + signal_handler(None, None) + +if __name__ == "__main__": + main()