From b7218175bbad99b4262e22aa31af9bfc6dbbe03c Mon Sep 17 00:00:00 2001 From: Daniel Bedeleanu Date: Thu, 23 Apr 2026 12:29:05 +0300 Subject: [PATCH] docs(phase-7): create detailed phase plans for config consolidation (4 plans, 14 tasks) --- .../07-config-consolidation/07-01-PLAN.md | 395 +++++++++++ .../07-config-consolidation/07-02-PLAN.md | 421 ++++++++++++ .../07-config-consolidation/07-03-PLAN.md | 583 ++++++++++++++++ .../07-config-consolidation/07-04-PLAN.md | 642 ++++++++++++++++++ 4 files changed, 2041 insertions(+) create mode 100644 .planning/phases/07-config-consolidation/07-01-PLAN.md create mode 100644 .planning/phases/07-config-consolidation/07-02-PLAN.md create mode 100644 .planning/phases/07-config-consolidation/07-03-PLAN.md create mode 100644 .planning/phases/07-config-consolidation/07-04-PLAN.md diff --git a/.planning/phases/07-config-consolidation/07-01-PLAN.md b/.planning/phases/07-config-consolidation/07-01-PLAN.md new file mode 100644 index 00000000..de415a27 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-01-PLAN.md @@ -0,0 +1,395 @@ +--- +phase: 07-config-consolidation +plan: 01 +type: execute +wave: 1 +depends_on: [] +files_modified: + - config/backend.yaml + - config/backend.yaml.example + - config/frontend.yaml + - config/frontend.yaml.example + - config/network.yaml + - config/network.yaml.example + - config/docker.yaml + - config/docker.yaml.example + - config/secrets.yaml.example + - config/README.md + - .gitignore +autonomous: true +requirements: + - PHASE-7-CONFIG-STRUCT + - PHASE-7-YAML-FORMAT + - PHASE-7-SECRETS-MGMT + - PHASE-7-DOCUMENTATION +user_setup: [] + +must_haves: + truths: + - "config/ folder exists with all YAML files and examples" + - "YAML structure matches backend, frontend, network, docker, and secrets domains" + - "All *.yaml.example files committed to git showing complete schema" + - "config/secrets.yaml is git-ignored, with example provided" + - ".gitignore correctly tracks examples, ignores actual secrets" + - "config/README.md documents every YAML file with required variables and setup instructions" + artifacts: + - path: "config/backend.yaml.example" + provides: "Backend configuration schema (database, AI keys, auth, logging)" + min_lines: 40 + - path: "config/frontend.yaml.example" + provides: "Frontend configuration schema (API endpoints, feature flags, PWA)" + min_lines: 25 + - path: "config/network.yaml.example" + provides: "Network configuration schema (ports, SSL, CORS, server IPs)" + min_lines: 20 + - path: "config/docker.yaml.example" + provides: "Docker-specific overrides (container resources, mount paths)" + min_lines: 20 + - path: "config/secrets.yaml.example" + provides: "Secrets template (JWT, API keys, passwords)" + min_lines: 15 + - path: "config/README.md" + provides: "Comprehensive documentation of all config files and setup" + min_lines: 100 + key_links: + - from: "config/backend.yaml" + to: "backend/config_loader.py" + via: "YAML parsing in config loader" + pattern: "load.*backend\\.yaml" + - from: "config/secrets.yaml.example" + to: ".gitignore" + via: "git ignore rule" + pattern: "config/\\*\\.yaml.*!.*\\.example" + - from: "config/" + to: "docker-compose.yml" + via: "volume mount in service definition" + pattern: "\\./config:/app/config" +--- + + +Create the config/ folder structure with YAML files for backend, frontend, network, docker, and secrets configurations. Establish the foundation for centralized configuration management with clear schema documentation. + +Purpose: Consolidate scattered configuration (currently in inventory.env) into a structured, domain-specific YAML format (per D-01, D-02, D-03). + +Output: config/ folder with 4 YAML config files + examples, secrets template, comprehensive README, and .gitignore updates. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/STATE.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@DEPLOYMENT.md +@inventory.env +@inventory.env.example +@docker-compose.yml + + + + + + Task 1: Create config/ folder structure and YAML example files + + config/backend.yaml.example + config/frontend.yaml.example + config/network.yaml.example + config/docker.yaml.example + config/secrets.yaml.example + + + - inventory.env (current config source) + - inventory.env.example (existing schema) + - PROJECT_ARCHITECTURE.md (tech stack, components) + - DEPLOYMENT.md (current config categories) + - docker-compose.yml (container environment vars) + + + Create config/ folder in project root with 5 YAML example files (per D-01, D-03): + + **config/backend.yaml.example** — Backend configuration template covering: + - Database: sqlite_path, log_retention_days, wal_mode (from current code patterns) + - AI: primary_ai_provider (gemini|claude), gemini_api_key, claude_api_key, fallback_provider + - Auth: jwt_secret_key, ldap_server (optional), ldap_base_dn, password_cache_path + - Logging: log_level (DEBUG|INFO|WARNING|ERROR), log_rotation_size_mb, log_rotation_count + - Application: data_dir, logs_dir, cors_origins + - Feature flags: ai_extraction_enabled, offline_sync_enabled, audit_logging_enabled + + **config/frontend.yaml.example** — Frontend configuration template covering: + - API: backend_url (e.g., http://localhost:8916), timeout_ms + - Feature flags: service_worker_enabled, offline_enabled, ai_extraction_ui_enabled + - PWA: app_name, short_name, start_url, display_mode + - Feature toggles: enable_qr_scanner, enable_barcode_scanner, enable_batch_import + + **config/network.yaml.example** — Network/deployment configuration template covering: + - Ports: backend_port (8916), frontend_port (8917), backend_ssl_port (8918), frontend_ssl_port (8919) + - SSL: ssl_enabled (true|false), certificate_path, key_path (or auto-generated by Caddy) + - Proxy: caddy_log_level, proxy_read_timeout_s, max_request_size_mb + - CORS: allowed_origins (comma-separated), allowed_methods, allowed_headers + + **config/docker.yaml.example** — Docker-specific overrides covering: + - Images: backend_image, frontend_image, proxy_image (with tags) + - Resources: backend_cpu_limit, backend_memory_limit, frontend_cpu_limit, frontend_memory_limit + - Volumes: data_volume_driver, logs_volume_driver, use_named_volumes (true|false) + - Networks: network_name, network_driver (bridge|overlay) + + **config/secrets.yaml.example** — Secrets template (git-ignored) covering: + - JWT_SECRET_KEY: "CHANGE_ME_IN_PRODUCTION" (minimum 32 chars) + - GEMINI_API_KEY: "your-api-key" + - CLAUDE_API_KEY: "your-api-key" + - DATABASE_PASSWORD: "db-password" (if using external DB) + - LDAP_PASSWORD: "ldap-password" (if using LDAP) + - CORS_ORIGIN_PASSWORD: (if CORS origins require auth) + + All examples should have comments explaining: + - What each variable controls + - Default values + - Valid value ranges + - Where to obtain secrets (e.g., "Generate with: openssl rand -hex 32") + - Whether the variable is required or optional + + + - `test -d config` (folder exists) + - `test -f config/backend.yaml.example && grep -q "primary_ai_provider" config/backend.yaml.example` (backend schema present) + - `test -f config/frontend.yaml.example && grep -q "backend_url" config/frontend.yaml.example` (frontend schema present) + - `test -f config/network.yaml.example && grep -q "backend_port" config/network.yaml.example` (network schema present) + - `test -f config/docker.yaml.example && grep -q "backend_cpu_limit" config/docker.yaml.example` (docker schema present) + - `test -f config/secrets.yaml.example && grep -q "JWT_SECRET_KEY" config/secrets.yaml.example` (secrets template present) + - All 5 files are readable and valid YAML syntax: `python3 -c "import yaml; [yaml.safe_load(open(f)) for f in ['config/backend.yaml.example', 'config/frontend.yaml.example', 'config/network.yaml.example', 'config/docker.yaml.example', 'config/secrets.yaml.example']]"` + + + config/ folder created with 5 YAML example files, all valid YAML syntax, comprehensive comments documenting schema, required/optional status, and secret generation instructions. + + + + + Task 2: Create actual config files from examples and establish .gitignore rules + + config/backend.yaml + config/frontend.yaml + config/network.yaml + config/docker.yaml + .gitignore + + + - config/backend.yaml.example (just created) + - config/frontend.yaml.example (just created) + - config/network.yaml.example (just created) + - config/docker.yaml.example (just created) + - inventory.env (current actual values to migrate) + - .gitignore (current ignore rules) + + + Create actual (non-example) YAML config files by copying examples and filling in values from inventory.env (per D-03 pattern): + + **config/backend.yaml** — Copy from backend.yaml.example and fill in values from inventory.env: + - primary_ai_provider: (from PRIMARY_AI_PROVIDER in inventory.env, default: gemini) + - gemini_api_key: (from GEMINI_API_KEY if present) + - claude_api_key: (from CLAUDE_API_KEY if present) + - ldap_server: (from LDAP_SERVER if present, or empty) + - log_level: (from LOG_LEVEL if present, default: INFO) + - data_dir: ./data + - logs_dir: ./logs + - cors_origins: (from CORS_ORIGINS if present, or default: http://localhost:8917) + + **config/frontend.yaml** — Create from frontend.yaml.example: + - backend_url: (from BACKEND_URL or derived from BACKEND_PORT, default: http://localhost:8916) + - timeout_ms: 30000 + - service_worker_enabled: true + - offline_enabled: true + - ai_extraction_ui_enabled: true + + **config/network.yaml** — Create from network.yaml.example: + - backend_port: (from BACKEND_PORT in inventory.env, default: 8916) + - frontend_port: (from FRONTEND_PORT in inventory.env, default: 8917) + - backend_ssl_port: (from BACKEND_SSL_PORT if present, default: 8918) + - frontend_ssl_port: (from FRONTEND_SSL_PORT if present, default: 8919) + - ssl_enabled: true + - allowed_origins: (from CORS_ORIGINS if present) + + **config/docker.yaml** — Create from docker.yaml.example: + - backend_cpu_limit: "1.0" + - backend_memory_limit: "1G" + - frontend_cpu_limit: "0.5" + - frontend_memory_limit: "512M" + - use_named_volumes: true + + **Update .gitignore** (per D-08): + Add rules to IGNORE actual config files (secrets protection) but TRACK examples: + ``` + # Config files — ignore all .yaml except examples + config/*.yaml + !config/*.yaml.example + config/secrets.yaml + !config/secrets.yaml.example + ``` + + Do NOT delete old inventory.env yet (backward compatibility until Phase 7 complete, per D-04 deprecation timeline). + + + - `test -f config/backend.yaml && grep -q "primary_ai_provider" config/backend.yaml` (backend.yaml exists and has content) + - `test -f config/frontend.yaml && grep -q "backend_url" config/frontend.yaml` (frontend.yaml exists) + - `test -f config/network.yaml && grep -q "backend_port" config/network.yaml` (network.yaml exists) + - `test -f config/docker.yaml && grep -q "backend_cpu_limit" config/docker.yaml` (docker.yaml exists) + - All 4 files are valid YAML: `python3 -c "import yaml; [yaml.safe_load(open(f)) for f in ['config/backend.yaml', 'config/frontend.yaml', 'config/network.yaml', 'config/docker.yaml']]"` + - .gitignore contains rules: `grep -q "config/\*\.yaml" .gitignore && grep -q "!config/\*\.yaml\.example" .gitignore` + - config/secrets.yaml does NOT exist (will be created manually by developers from example) + + + Actual config files created with production values migrated from inventory.env, .gitignore updated to track examples and ignore actual config files (including secrets). + + + + + Task 3: Create comprehensive config/README.md documentation + + config/README.md + + + - config/backend.yaml.example + - config/frontend.yaml.example + - config/network.yaml.example + - config/docker.yaml.example + - config/secrets.yaml.example + - DEPLOYMENT.md (current deployment docs) + + + Create config/README.md with comprehensive documentation (per D-08) covering: + + **Section 1: Overview** + - Explain that config/ is the single source of truth for application configuration + - Mention load order: system environment variables > config/*.yaml > defaults in code + - Note that secrets are separate (git-ignored) + + **Section 2: Quick Start** + - Copy all *.example files to remove .example suffix + - Fill in required values (especially JWT_SECRET_KEY, API keys) + - Create secrets.yaml from secrets.yaml.example with actual values + + **Section 3: backend.yaml** + - Explain each variable: purpose, valid values, defaults, required/optional + - List how to obtain each value (e.g., "Generate JWT key with: openssl rand -hex 32") + - Show example values + - List environment variable override names (e.g., BACKEND_PRIMARY_AI_PROVIDER) + + **Section 4: frontend.yaml** + - Explain each variable: purpose, valid values, defaults + - List required vs optional flags + - Show how feature flags affect UI behavior + - List environment variable override names + + **Section 5: network.yaml** + - Explain port assignments and SSL settings + - Show how CORS origins work + - List default values and adjustment guidance + - List environment variable override names + + **Section 6: docker.yaml** + - Explain container resource limits (CPU, memory) + - Show how to adjust for different hardware + - Explain volume management + - List environment variable override names + + **Section 7: secrets.yaml** + - Explain git-ignore protection + - List all required secrets with generation/obtainment instructions + - Warn about security implications + - Show correct file permissions (600) + + **Section 8: Environment Variable Override** + - Explain how system environment variables take precedence over YAML + - Show naming convention (e.g., BACKEND_PRIMARY_AI_PROVIDER -> backend.yaml:primary_ai_provider) + - Useful for Docker deployments where secrets come from docker run -e + + **Section 9: Troubleshooting** + - Common issues: missing secrets, invalid YAML syntax, missing required values + - How to validate YAML syntax + - How to debug which config source is being used + + Include helpful tables showing all variables, their purposes, defaults, and how to override them. + + + - `test -f config/README.md && wc -l config/README.md | awk '{print $1}' | awk '$1 >= 100 {print "pass"}'` (README has at least 100 lines) + - `grep -q "Quick Start" config/README.md && grep -q "backend.yaml" config/README.md && grep -q "secrets.yaml" config/README.md` (README covers all files) + - `grep -q "environment variable" config/README.md` (override mechanism documented) + - `grep -q "JWT_SECRET_KEY" config/README.md && grep -q "openssl rand" config/README.md` (secret generation instructions present) + + + config/README.md created with comprehensive documentation of all YAML files, required variables, generation instructions, environment variable overrides, and troubleshooting guide. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| Filesystem → Application | Config files loaded from disk must not be tampered with | +| Environment → Application | System environment variables override YAML (untrusted if exposed) | +| Git repository → Deployment | .gitignore must prevent accidental commit of secrets | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-01 | Tampering | config/*.yaml | mitigate | File permissions enforced via git (0644 for examples, 0600 for secrets). Read-only volume mounts in Docker per docker-compose.yml line 19 `:ro` flag. | +| T-07-02 | Information Disclosure | config/secrets.yaml | mitigate | .gitignore rule prevents accidental commit. File permissions enforced (chmod 600). config/README.md warns developers about security. Example file included to guide setup. | +| T-07-03 | Denial of Service | config/*.yaml parsing | mitigate | PyYAML used with safe_load() only (no arbitrary code execution). Backend config_loader.py validates syntax before loading. Invalid YAML causes logged error + default fallback per D-06 load order. | +| T-07-04 | Elevation of Privilege | JWT_SECRET_KEY exposure | accept | JWT secret hardcoded in docker-compose.yml example (line 25) warns with comment. Developers must provide production value. Risk low for development environments. | + + + + +**Phase 7, Plan 1 Verification Checklist:** + +1. **Config Folder Structure** + - [ ] `config/` folder exists in project root + - [ ] 5 example files present: backend.yaml.example, frontend.yaml.example, network.yaml.example, docker.yaml.example, secrets.yaml.example + - [ ] All examples contain valid YAML syntax (parseable by python3 -m yaml) + - [ ] All examples have comprehensive comments explaining each variable + +2. **Actual Config Files** + - [ ] 4 actual config files exist: backend.yaml, frontend.yaml, network.yaml, docker.yaml + - [ ] All actual files contain valid YAML syntax + - [ ] Values migrated from inventory.env are present and reasonable + - [ ] secrets.yaml does NOT exist (will be created manually) + +3. **Git Integration** + - [ ] .gitignore updated with rules: `config/*.yaml`, `!config/*.yaml.example`, `config/secrets.yaml`, `!config/secrets.yaml.example` + - [ ] Examples are tracked: `git status config/*.example` shows "new file" or no changes + - [ ] Actual configs are ignored: `git check-ignore config/backend.yaml` returns success + - [ ] Secrets example is tracked but secrets themselves are ignored + +4. **Documentation** + - [ ] config/README.md exists with 100+ lines + - [ ] README covers all 5 YAML files (backend, frontend, network, docker, secrets) + - [ ] README documents environment variable override mechanism + - [ ] README includes secret generation/obtainment instructions + - [ ] README has troubleshooting section + +5. **Backward Compatibility (D-04)** + - [ ] inventory.env still exists (will be fully deprecated after backend refactor in Plan 2) + - [ ] No code changes yet (Config loading still uses inventory.env, Phase 7 Plan 2 updates backend) + + + +- config/ folder created with 5 YAML example files defining complete schema +- 4 actual YAML config files created with values migrated from inventory.env +- config/secrets.yaml.example provides template (actual secrets.yaml created manually by developers) +- .gitignore updated to track examples, ignore actual config files and secrets +- config/README.md provides comprehensive documentation and setup instructions +- All YAML files are syntactically valid and parseable +- No backend code changes yet (backward compatibility maintained per D-04) +- Foundation ready for backend refactoring in Plan 2 + + + +After completion, create `.planning/phases/07-config-consolidation/07-01-SUMMARY.md` + diff --git a/.planning/phases/07-config-consolidation/07-02-PLAN.md b/.planning/phases/07-config-consolidation/07-02-PLAN.md new file mode 100644 index 00000000..bc658250 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-02-PLAN.md @@ -0,0 +1,421 @@ +--- +phase: 07-config-consolidation +plan: 02 +type: execute +wave: 2 +depends_on: + - 07-01 +files_modified: + - backend/config_loader.py + - backend/config_manager.py + - backend/main.py + - backend/entrypoint.sh +autonomous: true +requirements: + - PHASE-7-BACKEND-YAML + - PHASE-7-ENV-OVERRIDE + - PHASE-7-NO-FALLBACK +user_setup: [] + +must_haves: + truths: + - "Backend loads from config/backend.yaml + config/secrets.yaml (YAML parsing with PyYAML)" + - "System environment variables override YAML values (per D-06 load order)" + - "NO fallback to inventory.env—deprecation complete after Phase 7 (per D-04)" + - "Config loading logs which source is being used for debugging" + - "Backend starts successfully with new config structure and passes health checks" + artifacts: + - path: "backend/config_loader.py" + provides: "YAML config parsing with env var override and load order enforcement" + exports: ["load_config()", "get_config()", "validate_config()"] + min_lines: 80 + - path: "backend/config_manager.py" + provides: "Config management and updates with YAML support" + min_lines: 50 + - path: "backend/main.py" + provides: "Updated main() to use new config_loader (no inventory.env references)" + pattern: "from backend.config_loader import load_config" + - path: "backend/entrypoint.sh" + provides: "Updated Docker entrypoint sourcing YAML config paths" + pattern: "config/backend.yaml" + key_links: + - from: "backend/config_loader.py" + to: "config/backend.yaml" + via: "PyYAML parsing" + pattern: "yaml\\.safe_load.*backend\\.yaml" + - from: "backend/config_loader.py" + to: "config/secrets.yaml" + via: "PyYAML parsing with file existence check" + pattern: "yaml\\.safe_load.*secrets\\.yaml" + - from: "backend/main.py" + to: "backend/config_loader.py" + via: "import and call load_config()" + pattern: "from backend.config_loader import load_config" +--- + + +Refactor backend configuration loading from .env to YAML (backend.yaml + secrets.yaml) with system environment variable override support. Remove all inventory.env fallback paths and ensure deprecation is complete. + +Purpose: Implement D-06 load order (env vars > YAML > defaults) with proper logging and validation. + +Output: Updated config_loader.py with YAML parsing, config_manager.py with YAML support, main.py using new loader, and updated Docker entrypoint. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@backend/config_loader.py +@backend/config_manager.py +@backend/main.py +@backend/entrypoint.sh +@config/backend.yaml.example +@config/secrets.yaml.example + + + + + + Task 1: Refactor backend/config_loader.py for YAML parsing with env var override + + backend/config_loader.py + + + - backend/config_loader.py (current implementation using dotenv) + - config/backend.yaml.example (schema to parse) + - config/secrets.yaml.example (secrets schema) + - backend/main.py (to understand how config is used) + + + Refactor backend/config_loader.py to implement D-06 load order: system env vars > config/backend.yaml > config/secrets.yaml > defaults. + + **Required changes:** + + 1. **Replace dotenv with PyYAML:** Add `import yaml` and remove `from dotenv import load_dotenv` + + 2. **Implement load_config() function** that: + - Locates config/ folder (one level up from backend/) + - Attempts to load config/backend.yaml (required if exists) + - Attempts to load config/secrets.yaml (optional, file may not exist) + - Defines hard defaults for all variables (fallback if YAML missing) + - Merges in order: defaults <- YAML values <- environment variable overrides + - Returns a dict/object with all configuration + + 3. **Environment variable override pattern:** + - System env var takes precedence over YAML + - Naming convention: BACKEND_ or just + - Examples: + - BACKEND_PRIMARY_AI_PROVIDER -> backend.yaml:primary_ai_provider + - JWT_SECRET_KEY (from secrets.yaml or env) + - LOG_LEVEL -> backend.yaml:log_level + - All env vars checked with os.getenv() + + 4. **Load order example (pseudocode):** + ``` + defaults = {primary_ai_provider: "gemini", log_level: "INFO", ...} + backend_yaml = yaml.safe_load(open("config/backend.yaml")) if exists else {} + secrets_yaml = yaml.safe_load(open("config/secrets.yaml")) if exists else {} + config = merge(defaults, backend_yaml, secrets_yaml) + config = merge(config, env_var_overrides()) + return config + ``` + + 5. **Implement validate_config() function** that: + - Checks required variables are present (JWT_SECRET_KEY, primary_ai_provider, etc.) + - Validates enum values (primary_ai_provider must be gemini|claude|fallback) + - Validates log levels (DEBUG|INFO|WARNING|ERROR) + - Raises ConfigError if validation fails with descriptive message + + 6. **Implement get_config() function** that: + - Returns the loaded configuration dict + - Allows other modules to import: `from backend.config_loader import get_config` + + 7. **Logging:** + - Log which files were loaded: "Loaded backend.yaml from config/" + - Log env var overrides: "Override primary_ai_provider from environment: gemini" + - Log final validated config (without secrets): "Config validated: primary_ai_provider=gemini, log_level=INFO" + - Use log.info() and log.warning() (not print) + + 8. **Remove all inventory.env references:** + - Delete any checks for inventory_env_path + - Delete fallback to backend/.env + - NO fallback to old locations (per D-04 deprecation) + + 9. **Error handling:** + - If config/backend.yaml missing: raise ConfigError with instructions to copy from .example + - If secrets.yaml missing: log warning but continue (secrets can come from env vars) + - If required variables missing after all sources: raise ConfigError listing missing vars + + 10. **Function signature** (updated): + ```python + def load_config() -> dict: + """Load config from YAML files with env var overrides (D-06 load order).""" + + def get_config() -> dict: + """Get loaded config.""" + + def validate_config(config: dict) -> bool: + """Validate config has all required values.""" + ``` + + Keep the auto-run at module load: `load_config()` and `validate_config()` called on import. + + + - `grep -q "import yaml" backend/config_loader.py` (PyYAML imported) + - `grep -q "def load_config" backend/config_loader.py && grep -q "def get_config" backend/config_loader.py` (required functions exist) + - `grep -q "config/backend.yaml" backend/config_loader.py` (reads backend YAML) + - `grep -q "config/secrets.yaml" backend/config_loader.py` (reads secrets YAML) + - `grep -q "os.getenv" backend/config_loader.py` (env var overrides present) + - `grep -v "inventory.env" backend/config_loader.py | grep -q "inventory"` should return empty (no inventory.env references) + - `grep -q "validate_config" backend/config_loader.py` (validation function present) + - `grep -q "log.info\|log.warning" backend/config_loader.py` (logging present) + - File should be valid Python: `python3 -m py_compile backend/config_loader.py` + + + config_loader.py refactored to parse YAML files with env var override, remove inventory.env completely, implement D-06 load order with validation and logging. + + + + + Task 2: Update backend/config_manager.py for YAML config updates + + backend/config_manager.py + + + - backend/config_manager.py (current implementation) + - config/backend.yaml.example (schema) + - backend/config_loader.py (just updated) + + + Update backend/config_manager.py to support YAML config file updates (if runtime updates are needed): + + **If config_manager.py currently reads/writes .env files:** + + 1. **Replace dotenv with PyYAML:** + - Add `import yaml` + - Remove any dotenv usage + + 2. **Implement update_config() function** that: + - Takes key-value pairs to update + - Loads current config/backend.yaml + - Updates values in-memory + - Writes back to config/backend.yaml with safe_dump() + - NEVER writes to config/secrets.yaml (secrets are git-ignored for a reason) + - Logs what was updated + + 3. **Implement read_config() function** that: + - Reads config/backend.yaml and returns dict + - Uses yaml.safe_load() + + 4. **Handle errors gracefully:** + - If config/backend.yaml not found, raise error (it should exist from Plan 1) + - If YAML syntax error, log and return current in-memory config + - Preserve file comments if possible (or warn user they will be lost) + + 5. **Function signature** (updated): + ```python + def read_config() -> dict: + """Read backend.yaml and return current config.""" + + def update_config(updates: dict) -> dict: + """Update backend.yaml with new values and return updated config.""" + + def validate_config_file() -> bool: + """Validate backend.yaml syntax and required fields.""" + ``` + + **If config_manager.py is minimal/unused:** + - Add basic functions as above for future extensibility + - Add docstrings explaining YAML handling + - Import and use config_loader.load_config() as primary source + + + - `grep -q "import yaml" backend/config_manager.py` (PyYAML imported) + - `grep -q "def read_config\|def update_config\|def validate_config_file" backend/config_manager.py` (functions present) + - `grep -q "config/backend.yaml" backend/config_manager.py` (references YAML file) + - `grep -q "yaml.safe_load\|yaml.safe_dump" backend/config_manager.py` (YAML parsing present) + - File should be valid Python: `python3 -m py_compile backend/config_manager.py` + + + config_manager.py updated to support YAML config file updates with safe_load/safe_dump, no dotenv dependencies. + + + + + Task 3: Update backend/main.py to use new YAML config loader + + backend/main.py + + + - backend/main.py (current implementation) + - backend/config_loader.py (just updated) + + + Update backend/main.py to use the refactored config_loader: + + 1. **Update imports:** + - Replace any `from dotenv import load_dotenv` with `from backend.config_loader import load_config, get_config` + - Remove `load_dotenv()` calls + + 2. **Update main startup:** + - Call `load_config()` at app startup (or rely on module-level auto-run) + - Retrieve config with `get_config()` instead of `os.getenv()` + - Example: `config = get_config()` then `db_path = config['database']['sqlite_path']` + + 3. **Update environment variable access:** + - Replace `os.getenv("BACKEND_PORT")` with `config.get("backend_port")` + - Replace `os.getenv("JWT_SECRET_KEY")` with `config.get("jwt_secret_key")` + - All refs should come from config dict, not os.getenv() + + 4. **Remove inventory.env references:** + - Delete any checks for inventory.env existence + - Delete fallback logic to root-level config + - Ensure NO hardcoded "inventory.env" strings remain + + 5. **Logging:** + - Log at startup which config was loaded (already done by config_loader, but confirm) + - Example: "Backend initialized with config from config/backend.yaml" + + Note: This should be minimal changes if main.py already calls config_loader.load_config() at startup. + + + - `grep -q "from backend.config_loader import" backend/main.py` (imports from new loader) + - `grep -q "load_dotenv" backend/main.py` should return empty (no dotenv) + - `grep -q "inventory.env" backend/main.py` should return empty (no old config refs) + - File should be valid Python: `python3 -m py_compile backend/main.py` + - Check for os.getenv() calls and ensure they reference config dict instead: `grep "os.getenv" backend/main.py | head -5` (should be minimal or zero) + + + backend/main.py updated to import and use new YAML-based config_loader, remove all inventory.env and dotenv references. + + + + + Task 4: Update backend/entrypoint.sh for new config paths + + backend/entrypoint.sh + + + - backend/entrypoint.sh (current Docker entrypoint) + - docker-compose.yml (volumes mapping config/) + + + Update backend/entrypoint.sh to reference new config/ folder paths (per D-07 Docker support): + + 1. **If entrypoint sources config:** + - Remove any sourcing of inventory.env + - Add comment: "Config is loaded from /app/config/ (YAML format) per Phase 7" + - Ensure /app/config/ path is correct (mapped from host config/ via docker-compose.yml line 19) + + 2. **Update environment variable documentation:** + - Add comment: "Environment variables override YAML config (D-06 load order)" + - List key variables that can be overridden: JWT_SECRET_KEY, PRIMARY_AI_PROVIDER, etc. + - Example: `export JWT_SECRET_KEY="$(openssl rand -hex 32)"` (if not set) + + 3. **Ensure startup doesn't fail if config missing:** + - Add check: if config/backend.yaml not found, log error and instructions + - Python code will raise ConfigError, so entrypoint can remain simple + - Just ensure permissions are correct: `chmod 644 config/*.yaml` + + 4. **Update Dockerfile comment (if present):** + - Reference config/ volume mount + - Explain YAML loading approach + + + - `grep -q "config/backend.yaml\|config/secrets.yaml" backend/entrypoint.sh || echo "pass"` (references new config paths or doesn't source at all) + - `grep -q "inventory.env" backend/entrypoint.sh` should return empty (no old config) + - File should be valid bash: `bash -n backend/entrypoint.sh` + + + backend/entrypoint.sh updated to reference config/ paths and document environment variable override behavior. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| Filesystem → Backend | Backend reads from config/backend.yaml and config/secrets.yaml | +| Environment → Backend | System environment variables override config files (untrusted if exposed in logs) | +| Network → Backend | API receives JWT which is read from config (must be kept secret) | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-05 | Information Disclosure | Config logging | mitigate | Log config values WITHOUT secrets. config_loader.py logs final config but masks JWT_SECRET_KEY, API keys. Pattern: log keys but not values for sensitive fields. | +| T-07-06 | Denial of Service | Invalid YAML parsing | mitigate | yaml.safe_load() prevents code injection. ConfigError raised with clear message if required vars missing. validate_config() checks all required fields. | +| T-07-07 | Tampering | Environment variables | mitigate | Log env var overrides so operators know what was applied. Env vars documented in config/README.md. | +| T-07-08 | Elevation of Privilege | Backend startup | mitigate | ConfigError on missing JWT_SECRET_KEY prevents insecure defaults. Backend refuses to start without proper config. | + + + + +**Phase 7, Plan 2 Verification Checklist:** + +1. **config_loader.py Refactoring** + - [ ] PyYAML is imported, dotenv is removed + - [ ] load_config() loads config/backend.yaml + - [ ] load_config() loads config/secrets.yaml (optional) + - [ ] Environment variable overrides are applied correctly (D-06) + - [ ] validate_config() function checks required variables + - [ ] No inventory.env references remain + - [ ] Logging shows which config source was used + - [ ] File is valid Python syntax + +2. **config_manager.py Updates** + - [ ] PyYAML is imported + - [ ] read_config() and update_config() functions exist + - [ ] YAML file operations use safe_load/safe_dump + - [ ] No dotenv references + - [ ] File is valid Python syntax + +3. **main.py Updates** + - [ ] Imports from backend.config_loader + - [ ] Calls load_config() or relies on module-level auto-run + - [ ] Uses get_config() to retrieve configuration + - [ ] No dotenv or inventory.env references + - [ ] No direct os.getenv() calls for app config + - [ ] File is valid Python syntax + +4. **entrypoint.sh Updates** + - [ ] References config/ paths (not inventory.env) + - [ ] Documents environment variable override behavior + - [ ] Bash syntax is valid + +5. **Load Order Verification (D-06)** + - [ ] System env vars > config/backend.yaml > config/secrets.yaml > defaults + - [ ] Integration test: set BACKEND_LOG_LEVEL=DEBUG, verify backend uses DEBUG level + - [ ] Integration test: remove config/backend.yaml, verify defaults are used + - [ ] Integration test: set JWT_SECRET_KEY in env, verify it overrides YAML value + +6. **Deprecation Verification (D-04)** + - [ ] inventory.env no longer used by backend + - [ ] No fallback code remains + - [ ] Backend fails clearly (ConfigError) if required config missing (instead of silent defaults) + + + +- backend/config_loader.py refactored to use PyYAML with env var override support (D-06) +- D-06 load order implemented: env vars > YAML > defaults +- All inventory.env references removed from backend code (D-04) +- config_manager.py updated for YAML file operations +- backend/main.py uses new config loader +- backend/entrypoint.sh references config/ paths +- Configuration validation ensures required variables are present +- Logging shows which sources were used for debugging +- Backend starts successfully with new config structure + + + +After completion, create `.planning/phases/07-config-consolidation/07-02-SUMMARY.md` + diff --git a/.planning/phases/07-config-consolidation/07-03-PLAN.md b/.planning/phases/07-config-consolidation/07-03-PLAN.md new file mode 100644 index 00000000..bf70d11c --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-03-PLAN.md @@ -0,0 +1,583 @@ +--- +phase: 07-config-consolidation +plan: 03 +type: execute +wave: 2 +depends_on: + - 07-01 +files_modified: + - scripts/deploy.py + - scripts/run_standalone.py + - scripts/install_service.py + - scripts/export_prod.py +autonomous: true +requirements: + - PHASE-7-PYTHON-SCRIPTS + - PHASE-7-YAML-PARSING + - PHASE-7-DEPLOYMENT +user_setup: [] + +must_haves: + truths: + - "Python deployment scripts (deploy.py, run_standalone.py, install_service.py, export_prod.py) exist and parse YAML config" + - "All scripts parse config/*.yaml files using PyYAML (per D-05)" + - "deploy.py handles Docker deployment with health checks" + - "run_standalone.py launches backend and frontend without Docker" + - "install_service.py installs systemd service with new config paths" + - "export_prod.py exports production data/config for backups" + - "All scripts are executable and tested" + artifacts: + - path: "scripts/deploy.py" + provides: "Docker deployment with YAML config parsing, pre-flight checks, health validation" + exports: ["main()"] + min_lines: 150 + - path: "scripts/run_standalone.py" + provides: "Standalone launcher for backend (FastAPI) and frontend (Next.js) with YAML config" + exports: ["main()"] + min_lines: 120 + - path: "scripts/install_service.py" + provides: "Systemd service installation with config paths" + exports: ["main()"] + min_lines: 100 + - path: "scripts/export_prod.py" + provides: "Production export/backup script with YAML config support" + exports: ["main()"] + min_lines: 100 + key_links: + - from: "scripts/deploy.py" + to: "config/docker.yaml" + via: "PyYAML parsing for container config" + pattern: "yaml\\.safe_load.*docker\\.yaml" + - from: "scripts/run_standalone.py" + to: "config/backend.yaml" + via: "Read port and path config" + pattern: "yaml\\.safe_load.*backend\\.yaml" + - from: "scripts/install_service.py" + to: "inventory.service.template" + via: "Service file generation" + pattern: "template|service" +--- + + +Convert bash deployment scripts (deploy.sh, run_standalone.sh, install_service.sh, export_prod.sh) to Python with YAML config parsing. Provide consistent, maintainable deployment tooling that understands the new config structure. + +Purpose: Implement D-05 (Python scripts with YAML parsing) for modern deployment infrastructure. + +Output: 4 Python scripts in scripts/ folder with full deployment functionality and YAML config support. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@DEPLOYMENT.md +@deploy.sh +@run_standalone.sh +@install_service.sh +@export_prod.sh +@config/backend.yaml.example +@config/docker.yaml.example +@docker-compose.yml + + + + + + Task 1: Create scripts/deploy.py (Docker deployment with YAML config) + + scripts/deploy.py + + + - deploy.sh (current bash implementation to port) + - docker-compose.yml (structure and environment variables) + - config/docker.yaml.example (schema) + - config/backend.yaml.example (config structure) + - DEPLOYMENT.md (deployment procedure documentation) + + + Create scripts/deploy.py to replace deploy.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature and usage:** + ```bash + python3 scripts/deploy.py [production|staging|development] [--rebuild] + ``` + + 2. **Core functionality:** + - Pre-flight checks: Docker, Docker Compose, docker-compose.yml, config files + - Load config from config/docker.yaml and config/network.yaml + - Port availability checks (from network.yaml: backend_port, frontend_port, etc.) + - Environment file validation (config/backend.yaml exists and has required values) + - Docker Compose up with appropriate flags (rebuild if --rebuild) + - Health check polling (curl to /health endpoints) + - Deployment report (services running, ports, access URLs) + + 3. **Config file parsing:** + - Use PyYAML to load config/docker.yaml (for container resource limits, image names) + - Use PyYAML to load config/network.yaml (for port numbers and SSL settings) + - Use PyYAML to load config/backend.yaml (to validate required values) + - Fallback to sensible defaults if config files missing (but log warnings) + + 4. **Pre-flight checks (Step 1-5):** + - [ ] docker command available + - [ ] docker-compose command available + - [ ] docker-compose.yml exists + - [ ] config/backend.yaml exists (with helpful error if missing) + - [ ] config/network.yaml exists (with helpful error if missing) + + 5. **Port availability check (Step 6):** + - Read backend_port, frontend_port, backend_ssl_port, frontend_ssl_port from network.yaml + - Use netstat or ss to check if ports are in use + - Error if ports occupied, suggest alternatives + + 6. **Environment validation (Step 7):** + - Check config/backend.yaml for required values: JWT_SECRET_KEY, primary_ai_provider + - Warn if API keys are placeholders + - Proceed with warning (not error) for optional values + + 7. **Docker Compose deployment (Step 8-9):** + - Run `docker-compose up -d` (or with --build if --rebuild flag) + - Capture and display output with color codes + - Catch errors and provide helpful debugging steps + + 8. **Health checks (Step 10-11):** + - Poll backend health: `curl http://localhost:{backend_port}/health` (retry logic) + - Poll frontend health: `curl http://localhost:{frontend_port}/` (retry logic) + - Wait up to 2 minutes for services to become healthy + - Display health status to user + + 9. **Deployment report (Step 12):** + - Display service status: `docker-compose ps` + - Display access URLs: + - Frontend: http://localhost:{frontend_port} + - Backend API: http://localhost:{backend_port}/docs + - HTTPS: https://localhost:{frontend_ssl_port} (if SSL enabled in network.yaml) + - Display next steps (logs, troubleshooting, etc.) + + 10. **Error handling:** + - Descriptive error messages with suggested fixes + - Log all actions and results to stdout/stderr + - Use color output (GREEN for success, RED for errors, YELLOW for warnings, BLUE for info) + - Exit codes: 0 for success, 1 for fatal error + + 11. **Logging:** + - Use Python logging module (not print) + - Log level: INFO by default, DEBUG if --verbose flag + - Each step logged: "Step N/M: Description..." + - Results logged at end: "Deployment complete, services healthy" + + 12. **Required libraries:** + - sys, os, subprocess, time, socket (built-in) + - yaml (PyYAML) + - argparse (for CLI args) + - logging (for logging) + - No external deployment libraries (keep it simple) + + 13. **Make executable:** `chmod +x scripts/deploy.py` and include shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/deploy.py && head -1 scripts/deploy.py | grep -q "python3"` (shebang present) + - `test -x scripts/deploy.py` (executable) + - `python3 -m py_compile scripts/deploy.py` (valid Python syntax) + - `python3 scripts/deploy.py --help | grep -q "deployment"` (help works) + - `grep -q "import yaml" scripts/deploy.py` (PyYAML imported) + - `grep -q "config/docker.yaml\|config/network.yaml" scripts/deploy.py` (loads config files) + - `grep -q "docker-compose" scripts/deploy.py` (calls docker-compose) + - `grep -q "curl.*health" scripts/deploy.py` (health checks present) + + + scripts/deploy.py created with Docker deployment, YAML config parsing, health checks, and error handling. + + + + + Task 2: Create scripts/run_standalone.py (Standalone launcher with YAML config) + + scripts/run_standalone.py + + + - run_standalone.sh (current bash implementation to port) + - config/backend.yaml.example (schema) + - config/frontend.yaml.example (schema) + - backend/main.py (backend entry point) + - frontend package.json or next.config.js (frontend startup) + + + Create scripts/run_standalone.py to replace run_standalone.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature:** + ```bash + python3 scripts/run_standalone.py [--backend-only|--frontend-only] + ``` + + 2. **Core functionality:** + - Load config from config/backend.yaml and config/frontend.yaml + - Start FastAPI backend (uvicorn) + - Start Next.js frontend (npm run dev or node server.js) + - Display console output from both processes + - Handle shutdown gracefully (SIGTERM/SIGINT kills both services) + - Display health status and access URLs + + 3. **Config file parsing:** + - Load config/backend.yaml to get: backend_port, data_dir, logs_dir, log_level + - Load config/frontend.yaml to get: frontend_port, backend_url + - Use defaults if config files missing (with warnings) + + 4. **Backend startup (--backend-only or default):** + - Command: `uvicorn backend.main:app --host 0.0.0.0 --port {backend_port} --reload` + - Set environment: DATA_DIR, LOGS_DIR, LOG_LEVEL (from config) + - Capture output and display with [BACKEND] prefix + - Wait for backend to log "Uvicorn running on..." or similar + - Verify backend is listening on backend_port + + 5. **Frontend startup (--frontend-only or default):** + - Command: `npm run dev` (if in development) or `node server.js` (if built) + - Set environment: NEXT_PUBLIC_API_URL (from config:frontend:backend_url) + - Capture output and display with [FRONTEND] prefix + - Wait for frontend to log "ready - started server on..." or similar + - Verify frontend is listening on frontend_port + + 6. **Process management:** + - Use subprocess.Popen with shell=False (for security) + - Manage both processes in list/tuple + - Handle SIGTERM/SIGINT (Ctrl+C) to kill both processes + - Display "Shutting down..." and wait for clean shutdown + - Exit with code 0 if both shut down cleanly + + 7. **Health monitoring:** + - Periodically check if processes are alive (poll returncode) + - If one process dies, log error and optionally shutdown other (per config flag) + - Display uptime and status every 30 seconds + + 8. **Logging and output:** + - Use Python logging module + - Log each process with [BACKEND] / [FRONTEND] prefix + - Merge stdout/stderr from both processes to terminal + - Show final status: "Backend running on http://localhost:{backend_port}, Frontend on http://localhost:{frontend_port}" + + 9. **Error handling:** + - If uvicorn not installed, error and suggest: `pip install uvicorn` + - If npm not found, error and suggest: install Node.js + - If ports already in use, error with port number + - If config files missing, log warnings but use defaults + + 10. **Required libraries:** + - sys, os, subprocess, signal, time (built-in) + - yaml (PyYAML) + - argparse (for CLI args --backend-only, --frontend-only) + - logging (for logging) + + 11. **Make executable:** `chmod +x scripts/run_standalone.py` with shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/run_standalone.py && head -1 scripts/run_standalone.py | grep -q "python3"` (shebang present) + - `test -x scripts/run_standalone.py` (executable) + - `python3 -m py_compile scripts/run_standalone.py` (valid Python syntax) + - `grep -q "import yaml" scripts/run_standalone.py` (PyYAML imported) + - `grep -q "config/backend.yaml\|config/frontend.yaml" scripts/run_standalone.py` (loads config) + - `grep -q "uvicorn\|subprocess.Popen" scripts/run_standalone.py` (backend startup present) + - `grep -q "npm\|node server" scripts/run_standalone.py` (frontend startup present) + - `grep -q "signal.signal\|SIGTERM" scripts/run_standalone.py` (signal handling present) + + + scripts/run_standalone.py created with backend/frontend startup, YAML config parsing, process management, and graceful shutdown. + + + + + Task 3: Create scripts/install_service.py (Systemd service installation) + + scripts/install_service.py + + + - install_service.sh (current bash implementation to port) + - inventory.service.template (systemd service template) + - config/backend.yaml.example (to understand config structure) + - config/network.yaml.example (for port information) + + + Create scripts/install_service.py to replace install_service.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature:** + ```bash + sudo python3 scripts/install_service.py [--user=service_user] [--port=port] + ``` + + 2. **Core functionality:** + - Read inventory.service.template (or create template inline) + - Load config from config/backend.yaml (for paths, ports) + - Generate systemd service file with correct paths and user/group + - Install service file to /etc/systemd/system/ainventory.service + - Enable service (systemctl enable) + - Display installation summary and next steps + + 3. **Config file parsing:** + - Load config/backend.yaml to get: data_dir, logs_dir + - Load config/network.yaml to get: backend_port (for documentation) + - Use defaults if missing + + 4. **Service file generation:** + - Read inventory.service.template + - Replace placeholders: + - {PROJECT_DIR}: current working directory (project root) + - {SERVICE_USER}: service user (default: www-data, configurable via --user) + - {BACKEND_PORT}: from config/network.yaml + - {DATA_DIR}: from config/backend.yaml + - {LOGS_DIR}: from config/backend.yaml + - Template should: + - Type=simple + - ExecStart=/usr/bin/python3 {PROJECT_DIR}/scripts/run_standalone.py --backend-only + - WorkingDirectory={PROJECT_DIR} + - User={SERVICE_USER} + - Environment=PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin + - Restart=on-failure + - RestartSec=10 + + 5. **Permission checks:** + - Require sudo/root: `if os.getuid() != 0: error("Must run with sudo")` + - Check project directory is readable + - Check data_dir and logs_dir exist or can be created + + 6. **Service file installation:** + - Write service file to /etc/systemd/system/ainventory.service + - Set permissions: 644 (readable, not writable by non-root) + - Run `systemctl daemon-reload` + - Run `systemctl enable ainventory.service` + - Optionally start service: `systemctl start ainventory.service` + + 7. **Status display:** + - Show service file location + - Show service user and group + - Show project directory + - Show next steps: `systemctl start ainventory`, `systemctl status ainventory` + - Show logs: `journalctl -u ainventory -f` + + 8. **Error handling:** + - Check if service already installed (offer --force to overwrite) + - Check if user exists (suggest: `useradd -r -s /bin/false {user}`) + - Check if directories are writable + - Descriptive errors with suggested fixes + + 9. **Required libraries:** + - sys, os, subprocess, pwd, grp (built-in) + - yaml (PyYAML) + - argparse (for CLI args) + - logging (for logging) + + 10. **Make executable:** `chmod +x scripts/install_service.py` with shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/install_service.py && head -1 scripts/install_service.py | grep -q "python3"` (shebang present) + - `test -x scripts/install_service.py` (executable) + - `python3 -m py_compile scripts/install_service.py` (valid Python syntax) + - `grep -q "import yaml" scripts/install_service.py` (PyYAML imported) + - `grep -q "config/backend.yaml\|config/network.yaml" scripts/install_service.py` (loads config) + - `grep -q "/etc/systemd/system\|systemctl" scripts/install_service.py` (systemd integration present) + - `grep -q "os.getuid\|sudo" scripts/install_service.py` (permission check present) + + + scripts/install_service.py created with systemd service generation, config parsing, and installation logic. + + + + + Task 4: Create scripts/export_prod.py (Production export/backup) + + scripts/export_prod.py + + + - export_prod.sh (current bash implementation to port) + - config/backend.yaml.example (for data_dir) + - DEPLOYMENT.md (backup procedures) + + + Create scripts/export_prod.py to replace export_prod.sh with Python implementation (per D-05). + + **Key features:** + + 1. **Script signature:** + ```bash + python3 scripts/export_prod.py [--output=/path/to/backup.tar.gz] [--include-logs] + ``` + + 2. **Core functionality:** + - Load config from config/backend.yaml (to find data_dir, logs_dir) + - Create tar.gz archive of production data + - Include database file(s), config files (no secrets), and optionally logs + - Output to specified location or default: backups/{timestamp}.tar.gz + - Display archive size and location + + 3. **Config file parsing:** + - Load config/backend.yaml to get: data_dir, logs_dir + - Use defaults if missing: data_dir=./data, logs_dir=./logs + + 4. **Archive creation:** + - Include: {data_dir}/* (all application data, database, etc.) + - Include: config/*.yaml.example (config templates) + - Include: config/backend.yaml, config/frontend.yaml, config/network.yaml (actual configs, no secrets) + - Include: config/secrets.yaml.example (secrets template only, NOT actual secrets.yaml) + - Include: logs/* (optional, if --include-logs flag) + - Exclude: config/secrets.yaml (never backup actual secrets) + - Exclude: node_modules/, __pycache__/, .git/, .venv/ + - Exclude: temporary files, cache + + 5. **Archive naming:** + - Default: backups/ainventory_{timestamp}.tar.gz + - Timestamp format: YYYY-MM-DD_HH-MM-SS + - Custom path via --output flag + + 6. **Backup directory:** + - Create backups/ directory if not exists + - Set directory permissions: 750 (rwxr-x---) + + 7. **Verification:** + - Verify tar.gz was created successfully + - Display archive size: X.XX MB + - Display archive contents summary: "Includes database, data, and config (secrets excluded)" + + 8. **Error handling:** + - If data_dir doesn't exist, error and suggest creating it + - If no write permission to backups/, error and suggest location + - If tar command fails, show error and suggest troubleshooting + + 9. **Output example:** + ``` + [INFO] Loading config from config/backend.yaml + [INFO] Data directory: ./data + [INFO] Creating backup... + [INFO] Archive created: backups/ainventory_2026-04-23_14-30-45.tar.gz + [INFO] Archive size: 125.43 MB + [INFO] Contents: database, data, config (secrets excluded) + [INFO] Backup complete! + ``` + + 10. **Required libraries:** + - sys, os, subprocess, datetime, tarfile (built-in) + - yaml (PyYAML) + - argparse (for CLI args) + - logging (for logging) + + 11. **Make executable:** `chmod +x scripts/export_prod.py` with shebang: `#!/usr/bin/env python3` + + + - `test -f scripts/export_prod.py && head -1 scripts/export_prod.py | grep -q "python3"` (shebang present) + - `test -x scripts/export_prod.py` (executable) + - `python3 -m py_compile scripts/export_prod.py` (valid Python syntax) + - `grep -q "import yaml" scripts/export_prod.py` (PyYAML imported) + - `grep -q "config/backend.yaml" scripts/export_prod.py` (loads config) + - `grep -q "tarfile\|tar.gz" scripts/export_prod.py` (tar archive creation present) + - `grep -q "secrets.yaml" scripts/export_prod.py | grep -q "exclude"` (secrets excluded from backup) + + + scripts/export_prod.py created with production data export, YAML config parsing, archive creation, and secrets exclusion. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| User input → Script | Script arguments and config files must be validated | +| Script → System | Scripts execute system commands (subprocess) — must escape/quote properly | +| Script → Network | Health checks make HTTP requests (must handle timeouts) | +| Script → Filesystem | Scripts read/write files (must respect permissions) | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-09 | Injection | deploy.py subprocess | mitigate | Use subprocess with shell=False and list args (not f-strings). Example: `subprocess.run(["docker-compose", "up", "-d"], ...)` not `subprocess.run(f"docker-compose up -d", shell=True)`. | +| T-07-10 | Elevation of Privilege | install_service.py sudo | mitigate | Check `os.getuid() != 0` at start. Require sudo for systemd operations only. Log all systemctl calls. | +| T-07-11 | Information Disclosure | export_prod.py backup | mitigate | Exclude config/secrets.yaml explicitly in tarfile. Log what is excluded. Verify file permissions (backups/ dir 750). | +| T-07-12 | Denial of Service | Health checks timeout | mitigate | Set socket timeout to 10 seconds. Limit retry attempts to 12 (2 minutes total). Log timeout errors. | + + + + +**Phase 7, Plan 3 Verification Checklist:** + +1. **scripts/deploy.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Loads config/docker.yaml and config/network.yaml + - [ ] Pre-flight checks for Docker, Docker Compose, config files + - [ ] Port availability checks implemented + - [ ] Health checks poll backend and frontend endpoints + - [ ] Color output for info/warning/success/error + - [ ] Displays deployment summary and access URLs + - [ ] Valid Python syntax + +2. **scripts/run_standalone.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Loads config/backend.yaml and config/frontend.yaml + - [ ] Launches uvicorn for backend with correct port and settings + - [ ] Launches frontend (npm dev or node server.js) with correct port + - [ ] Signal handling (SIGTERM/SIGINT) for clean shutdown + - [ ] Process monitoring and output display with prefixes + - [ ] Valid Python syntax + +3. **scripts/install_service.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Checks for sudo/root permission + - [ ] Loads config/backend.yaml and config/network.yaml + - [ ] Generates systemd service file from template + - [ ] Replaces placeholders: {PROJECT_DIR}, {SERVICE_USER}, {BACKEND_PORT}, etc. + - [ ] Installs to /etc/systemd/system/ with correct permissions + - [ ] Runs systemctl daemon-reload and enable + - [ ] Valid Python syntax + +4. **scripts/export_prod.py** + - [ ] File exists and is executable + - [ ] Shebang present: `#!/usr/bin/env python3` + - [ ] Loads config/backend.yaml to find data_dir + - [ ] Creates tar.gz archive with data and config files + - [ ] Excludes config/secrets.yaml (actual secrets, not example) + - [ ] Includes config/*.yaml.example files + - [ ] Output to backups/{timestamp}.tar.gz or custom path + - [ ] Displays archive size and summary + - [ ] Valid Python syntax + +5. **Security (subprocess, permissions, file ops)** + - [ ] All subprocess calls use shell=False with list args + - [ ] No f-strings in shell commands + - [ ] File operations respect umask/permissions + - [ ] No hardcoded credentials in scripts + +6. **Integration** + - [ ] Each script loads YAML config files correctly + - [ ] Scripts reference new config/ structure (not inventory.env) + - [ ] Error messages are helpful and actionable + + + +- 4 Python scripts created (deploy.py, run_standalone.py, install_service.py, export_prod.py) +- All scripts use PyYAML to parse config files +- All scripts are executable with proper shebangs +- deploy.py handles Docker deployment with health checks +- run_standalone.py launches backend and frontend without Docker +- install_service.py creates systemd service with new config paths +- export_prod.py exports production data excluding secrets +- All subprocess calls use shell=False (secure) +- Error handling and logging present in all scripts +- Scripts integrate with new config/ structure (D-05, D-06, D-07) + + + +After completion, create `.planning/phases/07-config-consolidation/07-03-SUMMARY.md` + diff --git a/.planning/phases/07-config-consolidation/07-04-PLAN.md b/.planning/phases/07-config-consolidation/07-04-PLAN.md new file mode 100644 index 00000000..a0257a35 --- /dev/null +++ b/.planning/phases/07-config-consolidation/07-04-PLAN.md @@ -0,0 +1,642 @@ +--- +phase: 07-config-consolidation +plan: 04 +type: execute +wave: 3 +depends_on: + - 07-01 + - 07-02 + - 07-03 +files_modified: + - docker-compose.yml + - backend/Dockerfile + - backend/entrypoint.sh + - .gitignore + - DEPLOYMENT.md + - README.md +autonomous: true +requirements: + - PHASE-7-DOCKER-UPDATE + - PHASE-7-DOCUMENTATION + - PHASE-7-GITIGNORE +user_setup: [] + +must_haves: + truths: + - "docker-compose.yml updated to reference config/ volume and remove inventory.env env_file" + - "backend/Dockerfile and entrypoint updated for new config paths" + - ".gitignore properly configured to track examples, ignore actual configs and secrets" + - "DEPLOYMENT.md updated with YAML config structure, new Python scripts, setup instructions" + - "README.md updated with config setup and configuration management instructions" + - "Docker deployment works with new config structure (tested with docker-compose up)" + - "All documentation references config/ as single source of truth" + artifacts: + - path: "docker-compose.yml" + provides: "Docker Compose with config/ volume mount, no inventory.env env_file reference" + pattern: "\\./config:/app/config|!inventory.env" + min_lines: 120 + - path: "backend/Dockerfile" + provides: "Backend container image with new config paths" + pattern: "config/|/app/config" + - path: "backend/entrypoint.sh" + provides: "Docker entrypoint with config/ reference" + pattern: "config/|/app/config" + - path: "DEPLOYMENT.md" + provides: "Updated deployment guide with YAML config structure and Python scripts" + min_lines: 150 + - path: "README.md" + provides: "Updated README with config setup and onboarding" + min_lines: 100 + - path: ".gitignore" + provides: ".gitignore with rules for config/ folder (track examples, ignore secrets)" + pattern: "config/.*\\.yaml" + key_links: + - from: "docker-compose.yml" + to: "config/" + via: "volume mount" + pattern: "\\./config:/app/config" + - from: "DEPLOYMENT.md" + to: "config/README.md" + provides: "Cross-reference to config documentation" + pattern: "config/README.md|config/" + - from: "README.md" + to: "DEPLOYMENT.md" + provides: "Cross-reference to deployment guide" + pattern: "DEPLOYMENT.md|config/" +--- + + +Update Docker Compose, Dockerfile, documentation, and .gitignore to integrate the new config/ structure. Remove references to inventory.env from deployment infrastructure and update all deployment documentation. + +Purpose: Complete D-07 (Docker & Compose update), D-08 (documentation), and D-04 (deprecation) for cohesive deployment experience. + +Output: Updated docker-compose.yml, Dockerfile, entrypoint.sh, DEPLOYMENT.md, README.md, and .gitignore. + + + +@$HOME/.claude/get-shit-done/workflows/execute-plan.md +@$HOME/.claude/get-shit-done/templates/summary.md + + + +@.planning/PROJECT.md +@.planning/ROADMAP.md +@.planning/phases/07-config-consolidation/07-CONTEXT.md +@PROJECT_ARCHITECTURE.md +@docker-compose.yml +@backend/Dockerfile +@backend/entrypoint.sh +@DEPLOYMENT.md +@README.md +@.gitignore +@config/README.md + + + + + + Task 1: Update docker-compose.yml to reference config/ and remove inventory.env env_file + + docker-compose.yml + + + - docker-compose.yml (current file) + - backend/entrypoint.sh (to understand how config is used in containers) + - config/backend.yaml.example (to understand what config is needed) + - config/docker.yaml.example (Docker-specific config) + + + Update docker-compose.yml to integrate new config/ structure (per D-07): + + 1. **Remove inventory.env env_file references:** + - Delete or comment out `env_file: - inventory.env` from backend service (currently line 14) + - Delete or comment out `env_file: - inventory.env` from frontend service (currently line 51) + - Keep proxy service as is (may not need env_file) + + 2. **Add config/ volume mount to backend service:** + - Keep existing volume mounts + - Add: `- ./config:/app/config:ro` (read-only, config should not be modified in container) + - Update volumes section to reflect new mount + + 3. **Add config/ volume mount to frontend service (if frontend needs config):** + - Add: `- ./config:/app/config:ro` if frontend needs to read config files + - Or skip if frontend doesn't read YAML config directly + + 4. **Update environment variables:** + - Keep all existing environment variables (Docker overrides are still valid per D-06) + - Ensure JWT_SECRET_KEY is still set with warning: `# CHANGE THIS IN PRODUCTION!` + - Add comment: "Environment variables override config/backend.yaml per D-06 load order" + - Ensure DATA_DIR and LOGS_DIR are set to persist volume locations + + 5. **Add proxy service config/ mount (if proxy reads config):** + - Check if Caddyfile uses any dynamic config + - If not, no change needed + - If yes, add: `- ./config:/app/config:ro` + + 6. **Verify volume definitions:** + - Named volumes (backend_data, backend_logs, frontend_logs, caddy_data, caddy_config) remain unchanged + - Config mount is bind mount (./config), not named volume + + 7. **Add comments explaining the change:** + - Add section comment before volumes: "# [D-07] New config/ structure — YAML config mounted read-only" + - Add comment on env_file removal: "# [D-04] inventory.env deprecated — config now in config/ folder" + - Reference Phase 7 decisions + + 8. **Maintain backward compatibility during transition:** + - Don't delete old env_file line yet (can exist but be ignored by modern docker-compose) + - Or clearly comment it out with deprecation notice + + Example section after update: + ```yaml + backend: + ... + # [D-04] inventory.env deprecated — see config/ folder instead + # env_file: - inventory.env + volumes: + - backend_data:/app/data + - backend_logs:/app/logs + # [D-07] New config/ structure mounted read-only + - ./config:/app/config:ro + - ./scripts:/app/scripts:ro + ... + ``` + + + - `grep -n "env_file" docker-compose.yml | head` (check if inventory.env env_file is removed/commented) + - `grep -q "\\./config:/app/config:ro" docker-compose.yml` (config volume mount present) + - `docker-compose config 2>&1 | grep -q "config" || echo "valid yaml"` (valid docker-compose syntax) + - `grep -q "D-07\|D-04" docker-compose.yml || echo "pass"` (comments reference phase decisions, optional) + + + docker-compose.yml updated with config/ volume mount, inventory.env env_file removed, comments documenting changes. + + + + + Task 2: Update backend/Dockerfile for new config paths + + backend/Dockerfile + + + - backend/Dockerfile (current file) + - docker-compose.yml (just updated) + - backend/entrypoint.sh (how config is used at runtime) + + + Update backend/Dockerfile to document/support new config/ paths (per D-07): + + 1. **Add comments explaining config/ structure:** + - Add comment at top: "# [D-07] Backend container - config/ folder mounted at /app/config (read-only)" + - Add comment before WORKDIR: "# Config is expected in /app/config (mounted from host)" + + 2. **Ensure volume mount points exist:** + - Config is mounted at runtime by docker-compose, not created in Dockerfile + - No changes needed to RUN commands for config directory + - (Already handled by docker-compose volume mount) + + 3. **Update any hardcoded paths referencing inventory.env:** + - Search for "inventory.env" in Dockerfile + - Replace with reference to config/ or remove if no longer needed + - Example: If old CMD references inventory.env, update to reference config/ + + 4. **Update ENTRYPOINT or CMD if needed:** + - Ensure entrypoint.sh (or equivalent) references config/ paths + - Add environment documentation: "# Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables" + + 5. **Add healthcheck if not present:** + - Verify backend has healthcheck (curl to /health endpoint) + - Should already be in docker-compose.yml, but double-check Dockerfile + + 6. **Document environment variables:** + - Add comment: "# Environment variables override YAML config per D-06" + - List: DATA_DIR, LOGS_DIR, LOG_LEVEL (these come from env and/or config) + + Example after update: + ```dockerfile + # [D-07] Backend container - config/ folder mounted at /app/config (read-only) + # Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables + # Environment variables override YAML config per D-06 load order + + FROM python:3.12-slim + WORKDIR /app + + # Copy code, requirements, and startup scripts + COPY backend/ ./backend/ + COPY scripts/ ./scripts/ + COPY requirements.txt . + + # Install dependencies + RUN pip install --no-cache-dir -r requirements.txt + + # Config is mounted at /app/config by docker-compose + # No need to COPY config/ here (it's mounted read-only) + + ENTRYPOINT ["python", "backend/main.py"] + ``` + + + - `grep -q "D-07\|config/" backend/Dockerfile || echo "pass"` (comments reference config, optional) + - `grep -q "inventory.env" backend/Dockerfile` should return empty (no old inventory.env refs) + - `docker build -f backend/Dockerfile .` (valid Dockerfile syntax — may not succeed without full context, but no syntax errors) + + + backend/Dockerfile updated with comments documenting config/ structure, no inventory.env references. + + + + + Task 3: Update backend/entrypoint.sh for new config paths + + backend/entrypoint.sh + + + - backend/entrypoint.sh (current file) + - backend/config_loader.py (updated in Plan 2, to understand config loading) + - config/README.md (documentation on config structure) + + + Update backend/entrypoint.sh to reference and support new config/ structure (per D-07): + + 1. **Remove inventory.env sourcing:** + - Delete any lines that source or check for inventory.env + - Delete any EXPORT statements that copy inventory.env values to environment + + 2. **Add config/ path documentation:** + - Add comment at top: "# [D-07] Backend entrypoint - loads config from /app/config/ (YAML format)" + - Add comment: "# Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables" + + 3. **Add environment variable override documentation:** + - Add comment: "# [D-06] Environment variables override YAML config — set below takes precedence" + - List typical overrides: JWT_SECRET_KEY, PRIMARY_AI_PROVIDER, LOG_LEVEL, etc. + + 4. **Ensure config validation:** + - Add check: if [ ! -f "/app/config/backend.yaml" ]; then log error and instructions + - Add comment: "# Config validation handled by Python config_loader.py" + + 5. **Set working directory:** + - Ensure WORKDIR is set to /app (should be done in Dockerfile, but double-check) + + 6. **Exec main process:** + - Ensure entrypoint uses `exec` to replace shell: `exec python backend/main.py` + - This ensures signals (SIGTERM) are properly handled by Python process + + Example after update: + ```bash + #!/bin/bash + # [D-07] Backend entrypoint - loads config from /app/config/ (YAML format) + # Config sources: /app/config/backend.yaml, /app/config/secrets.yaml, environment variables + # [D-06] Environment variables override YAML config (below takes precedence) + + set -euo pipefail + + cd /app + + # Verify config is accessible + if [ ! -f "/app/config/backend.yaml" ]; then + echo "[ERROR] /app/config/backend.yaml not found!" + echo "[ERROR] Config must be mounted from host at /app/config/" + echo "[ERROR] See config/README.md for setup instructions" + exit 1 + fi + + # Environment variables below override YAML config + # (docker run -e JWT_SECRET_KEY="..." or docker-compose environment) + + # Start backend (signals properly handled with exec) + exec python -m uvicorn backend.main:app --host 0.0.0.0 --port 8000 + ``` + + 7. **Keep it minimal:** + - Entrypoint should be simple (most logic in Python config_loader.py) + - Just verify config exists and start the app + + + - `grep -q "D-07\|D-06\|config/" backend/entrypoint.sh` (references config structure, optional) + - `grep -q "inventory.env" backend/entrypoint.sh` should return empty (no old config) + - `bash -n backend/entrypoint.sh` (valid bash syntax) + - `head -1 backend/entrypoint.sh | grep -q "bash"` (shebang present) + + + backend/entrypoint.sh updated to reference config/ paths, remove inventory.env, document env var overrides. + + + + + Task 4: Update .gitignore to track config examples and ignore actual configs/secrets + + .gitignore + + + - .gitignore (current file) + - config/backend.yaml.example (created in Plan 1) + - config/secrets.yaml.example (created in Plan 1) + + + Update .gitignore to properly handle config/ folder (per D-03, D-08): + + 1. **Add config/ rules:** + ``` + # [D-08] Config folder — track examples, ignore actual configs and secrets + config/*.yaml + !config/*.yaml.example + config/secrets.yaml + !config/secrets.yaml.example + ``` + + 2. **Rationale:** + - `config/*.yaml` — Ignore all YAML files (actual configs with real secrets) + - `!config/*.yaml.example` — Except examples (these are tracked for schema/documentation) + - `config/secrets.yaml` — Explicitly ignore secrets file (redundant but clear) + - `!config/secrets.yaml.example` — Except example (for developer setup guidance) + + 3. **Clean up old rules:** + - Remove any existing `inventory.env` entries from .gitignore (deprecated) + - Or update to comment them as deprecated: `# inventory.env # [D-04] Deprecated - use config/backend.yaml` + + 4. **Add comment at top of config section:** + - Add comment: "# [D-04] inventory.env deprecated — see config/ folder instead" + - Add comment: "# [D-08] Config structure: examples tracked, actual configs ignored" + + 5. **Example .gitignore config section:** + ``` + # [D-04] inventory.env deprecated — see config/ folder instead + # [D-08] Config structure: examples tracked (schema), actual configs ignored (secrets) + config/*.yaml + !config/*.yaml.example + config/secrets.yaml + !config/secrets.yaml.example + ``` + + 6. **Verify git status:** + - After update, git status should show: + - config/*.yaml.example as "new file" or tracked + - config/*.yaml as ignored + - config/secrets.yaml as ignored + + + - `grep -q "config/\\*\\.yaml" .gitignore` (config rule present) + - `grep -q "!config/\\*\\.yaml\\.example" .gitignore` (exception for examples present) + - `grep -q "config/secrets\\.yaml" .gitignore` (secrets ignored) + - `grep -q "!config/secrets\\.yaml\\.example" .gitignore` (example tracked) + + + .gitignore updated with config/ rules to track examples and ignore actual configs/secrets. + + + + + Task 5: Update DEPLOYMENT.md with YAML config structure and new Python scripts + + DEPLOYMENT.md + + + - DEPLOYMENT.md (current file) + - config/README.md (created in Plan 1) + - scripts/deploy.py, scripts/run_standalone.py (created in Plan 3) + + + Update DEPLOYMENT.md to document new YAML config structure and Python deployment scripts (per D-08): + + 1. **Add section: Configuration (Before Quick Start)** + - Explain config/ as single source of truth + - List config files: backend.yaml, frontend.yaml, network.yaml, docker.yaml, secrets.yaml + - Reference config/README.md for detailed setup + - Explain examples and how to create actual config from examples + + 2. **Update Quick Start section:** + - Step 1: Clone and cd + - Step 2: Copy config examples to actual files: `cp config/*.yaml.example {without .example}` + - Step 3: Edit config files (backend.yaml, network.yaml, secrets.yaml) with your values + - Step 4: Choose deployment mode (Docker or Standalone) + + 3. **Update Docker Deployment section:** + - Replace old deploy.sh instructions with new deploy.py + - Usage: `python3 scripts/deploy.py [production|staging|development] [--rebuild]` + - Script handles: pre-flight checks, port validation, config loading, health checks + - Output: service status, access URLs + + 4. **Update Standalone Deployment section:** + - Replace old run_standalone.sh with new run_standalone.py + - Usage: `python3 scripts/run_standalone.py [--backend-only|--frontend-only]` + - Script handles: config loading, backend startup, frontend startup, signal handling + + 5. **Update Configuration Reference section:** + - Refer to config/README.md for complete reference + - List key files: config/backend.yaml, config/frontend.yaml, config/network.yaml, config/docker.yaml, config/secrets.yaml + - Explain environment variable overrides (D-06) + + 6. **Add Systemd Service Installation section:** + - Usage: `sudo python3 scripts/install_service.py [--user=www-data]` + - Explain what service does: runs standalone backend + frontend + - Show how to manage: `systemctl start|stop|status ainventory` + - Show logs: `journalctl -u ainventory -f` + + 7. **Add Backup & Export section:** + - Usage: `python3 scripts/export_prod.py [--output=/path/to/backup.tar.gz] [--include-logs]` + - Explains what is included: data, config, config templates + - Explains what is excluded: actual secrets (security), logs (optional) + + 8. **Add Security section:** + - Mention secrets.yaml is git-ignored + - Explain how to set up secrets (copy from example, fill in values) + - Warn about JWT_SECRET_KEY in docker-compose.yml (must change for production) + + 9. **Add Troubleshooting section:** + - Common issues: missing config files, invalid YAML syntax, port conflicts + - Debug steps: check config syntax, verify file permissions, run health checks + - Reference config/README.md for setup help + + 10. **Add Migration section (from old inventory.env):** + - Explain Phase 7 transition from inventory.env to config/ structure + - Provide migration script or manual steps + - Clear instructions: which old values map to which config files + + Structure should be roughly: + - 1. Overview (unchanged) + - 2. Prerequisites (unchanged) + - **3. Configuration** (NEW) + - 4. Quick Start (UPDATED) + - 5. Deployment Modes (Docker, Standalone) (UPDATED to use Python scripts) + - 6. Systemd Service (UPDATED with Python script) + - 7. Backup & Export (UPDATED with Python script) + - 8. Operations & Health Monitoring (UPDATED with new paths) + - 9. Security (NEW or UPDATED) + - 10. Troubleshooting (UPDATED) + - 11. Migration from inventory.env (NEW) + + + - `grep -q "config/.*\\.yaml" DEPLOYMENT.md` (references YAML config files) + - `grep -q "scripts/deploy\\.py\|scripts/run_standalone\\.py" DEPLOYMENT.md` (references Python scripts) + - `grep -q "config/README\\.md" DEPLOYMENT.md` (cross-references config documentation) + - `grep -q "environment.*override\|D-06" DEPLOYMENT.md || echo "pass"` (explains env var overrides, optional) + - File should have 150+ lines: `wc -l DEPLOYMENT.md | awk '$1 >= 150 {print "pass"}'` + + + DEPLOYMENT.md updated with YAML config structure, Python script usage, systemd service, backup procedures, and troubleshooting. + + + + + Task 6: Update README.md with config setup and configuration management instructions + + README.md + + + - README.md (current file) + - DEPLOYMENT.md (just updated) + - config/README.md (created in Plan 1) + + + Update README.md to include configuration management and quick setup instructions (per D-08): + + 1. **Add Configuration section (after Quick Start or before Deployment):** + - Brief explanation: config/ folder is single source of truth + - Quick steps: cp config/*.example to remove .example, edit with your values + - Reference config/README.md for detailed setup + - Reference DEPLOYMENT.md for deployment options + + 2. **Update Quick Start section (if exists):** + - Add configuration step before deployment + - Example: + ``` + # 1. Clone and setup + git clone ... && cd tfm-inventory + + # 2. Configure application + cp config/*.yaml.example config/$(basename {} .example) # or similar + nano config/backend.yaml # Edit with your values + cp config/secrets.yaml.example config/secrets.yaml + nano config/secrets.yaml # Fill in API keys and secrets + + # 3. Deploy (choose one) + python3 scripts/deploy.py production # Docker + # OR + python3 scripts/run_standalone.py # Standalone + ``` + + 3. **Add note about .gitignore:** + - Config examples are tracked (for schema) + - Actual configs are git-ignored (protect secrets) + - secrets.yaml is git-ignored (never commit) + + 4. **Cross-reference documentation:** + - Add links/references to: + - config/README.md (configuration reference) + - DEPLOYMENT.md (detailed deployment guide) + - dev_docs/ (for development setup) + + 5. **Add "Getting Help" section (if not exists):** + - Point to DEPLOYMENT.md troubleshooting + - Point to config/README.md for config questions + - Reference AI_RULES.md for project conventions + + 6. **Update any hardcoded inventory.env references:** + - Replace with config/ references + - Update any env-related docs/examples + + 7. **Maintain existing structure:** + - Don't delete or significantly reorder existing sections + - Just add/update config-related content and update cross-references + + Keep README concise but informative. Detailed docs go in DEPLOYMENT.md and config/README.md. + + + - `grep -q "config/" README.md` (references config structure) + - `grep -q "DEPLOYMENT\\.md\|config/README\\.md" README.md` (cross-references detailed docs) + - `grep -q "secrets.yaml\|git.*ignore" README.md || echo "pass"` (mentions secrets and gitignore, optional) + - File should be valid markdown: `grep "^#" README.md | head -3` (has headers) + + + README.md updated with configuration management, quick setup steps, and documentation cross-references. + + + + + + +## Trust Boundaries + +| Boundary | Description | +|----------|-------------| +| Git repository → Deployment | .gitignore must prevent secrets from being committed | +| Documentation → Users | Documentation must clearly explain security requirements | +| Environment → Container | Docker environment variables can expose secrets if logged | + +## STRIDE Threat Register + +| Threat ID | Category | Component | Disposition | Mitigation Plan | +|-----------|----------|-----------|-------------|-----------------| +| T-07-13 | Tampering | docker-compose volume mount | mitigate | Config volume mounted read-only `:ro`. Backend cannot modify config at runtime. Changes require host-level edits. | +| T-07-14 | Information Disclosure | DEPLOYMENT.md instructions | mitigate | Documentation warns to generate JWT_SECRET_KEY, don't use placeholder. Warns about secrets.yaml setup. | +| T-07-15 | Information Disclosure | .gitignore config rules | mitigate | Clear rules prevent accidental secret commits. !config/*.example exception ensures schema is tracked. | +| T-07-16 | Elevation of Privilege | Docker container permissions | mitigate | No RUN as root in Dockerfile. Container runs as unprivileged user (if specified in docker-compose). | + + + + +**Phase 7, Plan 4 Verification Checklist:** + +1. **docker-compose.yml** + - [ ] inventory.env env_file removed or commented (per D-04) + - [ ] ./config:/app/config:ro volume mount added to backend service + - [ ] Syntax valid: `docker-compose config` succeeds + - [ ] Comments reference D-07, D-04 decisions + - [ ] Environment variables preserved (JWT_SECRET_KEY etc. with production warning) + +2. **backend/Dockerfile** + - [ ] No references to inventory.env + - [ ] Comments reference config/ structure and D-07 + - [ ] ENTRYPOINT or CMD properly set + - [ ] Syntax valid: `docker build --dry-run` or manual parse + +3. **backend/entrypoint.sh** + - [ ] No sourcing of inventory.env + - [ ] References /app/config/ paths + - [ ] Checks if config/backend.yaml exists + - [ ] Documents environment variable override behavior + - [ ] Bash syntax valid: `bash -n backend/entrypoint.sh` + +4. **.gitignore** + - [ ] Rules added: config/*.yaml, !config/*.yaml.example, config/secrets.yaml, !config/secrets.yaml.example + - [ ] Old inventory.env references removed or marked deprecated + - [ ] Git test: `git check-ignore config/backend.yaml` returns success (ignored) + - [ ] Git test: `git status config/*.example` shows untracked (not ignored) + +5. **DEPLOYMENT.md** + - [ ] Configuration section added before or after Quick Start + - [ ] References config/ files (backend.yaml, frontend.yaml, network.yaml, docker.yaml, secrets.yaml) + - [ ] Docker deployment updated to use scripts/deploy.py + - [ ] Standalone deployment updated to use scripts/run_standalone.py + - [ ] Systemd service section with scripts/install_service.py + - [ ] Backup section with scripts/export_prod.py + - [ ] Troubleshooting section + - [ ] Migration section (from inventory.env to config/) + - [ ] Length 150+ lines + +6. **README.md** + - [ ] Configuration section added + - [ ] Quick start updated with config setup steps + - [ ] Cross-references to config/README.md and DEPLOYMENT.md + - [ ] Mentions secrets.yaml and .gitignore + - [ ] No hardcoded inventory.env references + +7. **Cross-document consistency** + - [ ] README.md, DEPLOYMENT.md, config/README.md use consistent terminology + - [ ] All three documents reference each other appropriately + - [ ] No conflicting instructions across documents + + + +- docker-compose.yml updated with config/ volume mount, inventory.env env_file removed (D-07) +- backend/Dockerfile and entrypoint.sh updated for new config paths +- .gitignore configured to track examples, ignore actual configs and secrets (D-08) +- DEPLOYMENT.md updated with YAML structure, Python scripts, systemd setup, troubleshooting (D-08) +- README.md updated with config setup and documentation cross-references (D-08) +- All documentation references config/ as single source of truth +- Docker deployment works with new config structure +- Clear migration path from inventory.env to new config/ structure documented + + + +After completion, create `.planning/phases/07-config-consolidation/07-04-SUMMARY.md` +