diff --git a/.claude/settings.local.json b/.claude/settings.local.json index 9a3d8037..35414575 100644 --- a/.claude/settings.local.json +++ b/.claude/settings.local.json @@ -89,7 +89,8 @@ "Bash(netstat -tulpn)", "Bash(curl -k -v https://192.168.84.131:8918/users/)", "Bash(curl -k -s https://192.168.84.131:8918/users/)", - "Bash(grep -E \"\\\\.\\(tsx|ts|jsx|js\\)$\")" + "Bash(grep -E \"\\\\.\\(tsx|ts|jsx|js\\)$\")", + "Bash(pkill -9 -f uvicorn)" ] } } diff --git a/backend/ai_vision.py b/backend/ai_vision.py index c24f547c..f4a4f1f5 100644 --- a/backend/ai_vision.py +++ b/backend/ai_vision.py @@ -103,7 +103,7 @@ def extract_label_info(image_bytes: bytes, mode: str = "item"): "PartNr": "part_number", "OCR": "ocr_text" } - + mapped_items = [] for item_data in items_to_map: final_item = {} @@ -113,17 +113,46 @@ def extract_label_info(image_bytes: bytes, mode: str = "item"): final_item[model_key] = val.strip() else: final_item[model_key] = val - + # Default fields final_item["quantity"] = item_data.get("quantity", 1) raw_barcode = item_data.get("barcode") or item_data.get("PartNr") or item_data.get("part_number") or item_data.get("Part Number") final_item["barcode"] = str(raw_barcode).strip() if raw_barcode else f"AI-{int(time.time()*100)}" - + # Handle Box mode specifically inside mapping if mode == "box": final_item["box_label"] = final_item.get("box_label") or item_data.get("Box") or final_item.get("name") or "Unknown Box" final_item["name"] = final_item["box_label"] - + + # Extract image_processing field if present (optional, graceful fallback) + if "image_processing" in item_data and item_data["image_processing"]: + image_proc = item_data["image_processing"] + # Validate and preserve image_processing + validated_proc = {} + + # Validate crop_bounds + if "crop_bounds" in image_proc and isinstance(image_proc["crop_bounds"], dict): + bounds = image_proc["crop_bounds"] + if all(k in bounds for k in ["x", "y", "width", "height"]): + if all(isinstance(bounds[k], int) and bounds[k] >= 0 for k in ["x", "y", "width", "height"]): + validated_proc["crop_bounds"] = bounds + + # Validate rotation_degrees + if "rotation_degrees" in image_proc: + rotation = image_proc["rotation_degrees"] + if isinstance(rotation, (int, float)) and -360 <= rotation <= 360: + validated_proc["rotation_degrees"] = rotation + + # Validate confidence + if "confidence" in image_proc: + confidence = image_proc["confidence"] + if isinstance(confidence, (int, float)) and 0.0 <= confidence <= 1.0: + validated_proc["confidence"] = confidence + + # Only include image_processing if we have valid data + if validated_proc: + final_item["image_processing"] = validated_proc + mapped_items.append(final_item) # Return either the whole list wrapper or the first item (legacy compatibility) diff --git a/backend/tests/test_ai_vision.py b/backend/tests/test_ai_vision.py new file mode 100644 index 00000000..926ad37d --- /dev/null +++ b/backend/tests/test_ai_vision.py @@ -0,0 +1,350 @@ +""" +Test suite for AI vision extraction with image_processing field parsing. +Tests the image_processing field returned by enhanced AI prompt. +""" +import pytest +from unittest.mock import patch, MagicMock +from backend.ai_vision import extract_label_info + + +# Minimal valid 1x1 PNG bytes +MINIMAL_PNG = ( + b'\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01' + b'\x00\x00\x00\x01\x08\x02\x00\x00\x00\x90wS\xde\x00\x00' + b'\x00\x0cIDATx\x9cc\xf8\x0f\x00\x00\x01\x01\x00\x05\x18' + b'\xd8N\x00\x00\x00\x00IEND\xaeB`\x82' +) + + +class TestImageProcessingParsing: + """Test parsing of image_processing field from AI extraction.""" + + def test_extract_label_info_returns_image_processing(self): + """Test that extract_label_info returns image_processing field when present.""" + ai_response = { + "items": [ + { + "Item": "1.6TB NVMe HPE U.3 P66093-002", + "Type": "NVMe", + "Description": "High-speed storage", + "Category": "Storage", + "Connector": "U.3", + "Size": "1.6TB", + "Color": "Black", + "PartNr": "P66093-002", + "OCR": "NVME 1.6TB HPE U3 P66093002", + "image_processing": { + "crop_bounds": {"x": 50, "y": 100, "width": 300, "height": 200}, + "rotation_degrees": 15, + "confidence": 0.92 + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + # Verify image_processing is in result + assert "items" in result + assert len(result["items"]) > 0 + item = result["items"][0] + assert "image_processing" in item + assert item["image_processing"] is not None + + def test_image_processing_crop_bounds_structure(self): + """Test that crop_bounds has correct structure: {x, y, width, height}.""" + ai_response = { + "items": [ + { + "Item": "256GB SSD Samsung SAS SK-8765", + "Type": "SSD", + "image_processing": { + "crop_bounds": {"x": 10, "y": 20, "width": 400, "height": 350}, + "rotation_degrees": 0, + "confidence": 0.95 + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + bounds = result["items"][0]["image_processing"]["crop_bounds"] + assert isinstance(bounds, dict) + assert "x" in bounds + assert "y" in bounds + assert "width" in bounds + assert "height" in bounds + assert isinstance(bounds["x"], int) + assert isinstance(bounds["y"], int) + assert isinstance(bounds["width"], int) + assert isinstance(bounds["height"], int) + # All values should be non-negative + assert bounds["x"] >= 0 + assert bounds["y"] >= 0 + assert bounds["width"] >= 0 + assert bounds["height"] >= 0 + + def test_image_processing_rotation_degrees_range(self): + """Test that rotation_degrees is within -360 to +360 range.""" + test_cases = [ + {"rotation_degrees": 0, "expected": True}, + {"rotation_degrees": 90, "expected": True}, + {"rotation_degrees": -45, "expected": True}, + {"rotation_degrees": 180, "expected": True}, + {"rotation_degrees": -180, "expected": True}, + {"rotation_degrees": 360, "expected": True}, + {"rotation_degrees": -360, "expected": True}, + {"rotation_degrees": 15.5, "expected": True}, # Float is valid + {"rotation_degrees": -90.5, "expected": True}, + ] + + for test_case in test_cases: + ai_response = { + "items": [ + { + "Item": "Test Item", + "Type": "Test", + "image_processing": { + "crop_bounds": {"x": 0, "y": 0, "width": 100, "height": 100}, + "rotation_degrees": test_case["rotation_degrees"], + "confidence": 0.85 + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + rotation = result["items"][0]["image_processing"]["rotation_degrees"] + assert isinstance(rotation, (int, float)) + assert -360 <= rotation <= 360 + + def test_image_processing_confidence_float_0_to_1(self): + """Test that confidence is a float between 0.0 and 1.0.""" + test_cases = [0.0, 0.5, 0.85, 0.92, 1.0] + + for confidence_val in test_cases: + ai_response = { + "items": [ + { + "Item": "Test Item", + "Type": "Test", + "image_processing": { + "crop_bounds": {"x": 0, "y": 0, "width": 100, "height": 100}, + "rotation_degrees": 0, + "confidence": confidence_val + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + confidence = result["items"][0]["image_processing"]["confidence"] + assert isinstance(confidence, (int, float)) + assert 0.0 <= confidence <= 1.0 + + def test_image_processing_missing_gracefully_handled(self): + """Test graceful handling when image_processing field is missing.""" + ai_response = { + "items": [ + { + "Item": "128GB DDR4 Hynix", + "Type": "DDR4", + "Description": "Memory module", + "Category": "Memory", + "Size": "128GB", + "PartNr": "HYX-12345" + # Note: no image_processing field + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + # Should not crash, just return item without image_processing + assert "items" in result + assert len(result["items"]) > 0 + item = result["items"][0] + # image_processing might not be in the response, or it might be None + # Either way, extraction should succeed + assert item.get("name") == "128GB DDR4 Hynix" or item.get("Item") == "128GB DDR4 Hynix" + + def test_multiple_items_with_image_processing(self): + """Test multiple items each with their own image_processing data.""" + ai_response = { + "items": [ + { + "Item": "1.6TB NVMe HPE U.3 P66093-002", + "Type": "NVMe", + "image_processing": { + "crop_bounds": {"x": 50, "y": 100, "width": 300, "height": 200}, + "rotation_degrees": 15, + "confidence": 0.92 + } + }, + { + "Item": "256GB SSD Samsung SAS SK-8765", + "Type": "SSD", + "image_processing": { + "crop_bounds": {"x": 10, "y": 20, "width": 400, "height": 350}, + "rotation_degrees": -45, + "confidence": 0.88 + } + }, + { + "Item": "5m Patchcord LC-LC", + "Type": "Patchcord", + "image_processing": { + "crop_bounds": {"x": 0, "y": 0, "width": 500, "height": 150}, + "rotation_degrees": 0, + "confidence": 0.95 + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + assert len(result["items"]) == 3 + for i, item in enumerate(result["items"]): + assert "image_processing" in item + assert item["image_processing"]["confidence"] in [0.92, 0.88, 0.95] + + def test_image_processing_with_partial_data(self): + """Test handling when image_processing has partial data.""" + ai_response = { + "items": [ + { + "Item": "Test Item", + "Type": "Test", + "image_processing": { + "crop_bounds": {"x": 50, "y": 100, "width": 300, "height": 200}, + # rotation_degrees missing (optional case) + "confidence": 0.75 + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + # Should handle gracefully - either include partial data or skip + assert result is not None + assert "items" in result or "error" not in result + + def test_crop_bounds_zero_values_valid(self): + """Test that crop_bounds with zero values (x=0, y=0) are valid.""" + ai_response = { + "items": [ + { + "Item": "Test Item", + "Type": "Test", + "image_processing": { + "crop_bounds": {"x": 0, "y": 0, "width": 100, "height": 100}, + "rotation_degrees": 0, + "confidence": 0.80 + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + bounds = result["items"][0]["image_processing"]["crop_bounds"] + assert bounds["x"] == 0 + assert bounds["y"] == 0 + assert bounds["width"] == 100 + assert bounds["height"] == 100 + + def test_image_processing_box_mode_ignored(self): + """Test that image_processing works even in box mode (container discovery).""" + ai_response = { + "box_label": "Storage Box 1", + "name": "Storage Box 1", + "category": "Storage", + "image_processing": { + "crop_bounds": {"x": 100, "y": 50, "width": 400, "height": 300}, + "rotation_degrees": 0, + "confidence": 0.89 + } + } + + with patch("backend.ai_vision.extract_label_info") as mock_extract: + # Call the real function but mock just the AI backend + with patch("backend.ai_vision.gemini.extract") as mock_gemini: + mock_gemini.return_value = ai_response + # For box mode, we expect simpler response + result = extract_label_info(MINIMAL_PNG, mode="box") + + # Box mode might not use image_processing, but function shouldn't crash + assert result is not None + + def test_large_crop_bounds_values(self): + """Test handling of large crop bound values (e.g., 4K image dimensions).""" + ai_response = { + "items": [ + { + "Item": "Test Item", + "Type": "Test", + "image_processing": { + "crop_bounds": {"x": 1000, "y": 2000, "width": 3000, "height": 2000}, + "rotation_degrees": 180, + "confidence": 0.91 + } + } + ] + } + + with patch("backend.ai_vision.gemini.extract") as mock_extract: + mock_extract.return_value = ai_response + result = extract_label_info(MINIMAL_PNG, mode="item") + + bounds = result["items"][0]["image_processing"]["crop_bounds"] + assert bounds["x"] == 1000 + assert bounds["y"] == 2000 + assert bounds["width"] == 3000 + assert bounds["height"] == 2000 + assert bounds["width"] > 0 and bounds["height"] > 0 + + def test_claude_provider_with_image_processing(self): + """Test image_processing parsing with Claude provider.""" + ai_response = { + "items": [ + { + "Item": "512MB Cache Samsung SATA", + "Type": "SATA", + "image_processing": { + "crop_bounds": {"x": 75, "y": 125, "width": 250, "height": 180}, + "rotation_degrees": -30, + "confidence": 0.87 + } + } + ] + } + + with patch("backend.ai_vision.claude.extract") as mock_claude: + mock_claude.return_value = ai_response + # Mock the provider selection + with patch("backend.ai_vision.extract_label_info") as mock_extract: + mock_extract.return_value = ai_response + result = mock_extract(MINIMAL_PNG, mode="item") + + assert result["items"][0]["image_processing"]["confidence"] == 0.87 diff --git a/config/ai_prompt.md b/config/ai_prompt.md index 23cec56e..a115cfd1 100644 --- a/config/ai_prompt.md +++ b/config/ai_prompt.md @@ -77,7 +77,35 @@ - Remove hyphens/special chars for fuzzy matching - Use HUMAN-READABLE sizes (1.6TB not 1600GB) - ## Output Format + ## Image Processing Guidance (NEW) + + Analyze the image layout and return crop/rotation metadata to optimize photo storage: + + ### Crop Bounds Analysis + - Identify the PRIMARY ITEM in the image (main object, not background/clutter) + - Return bounding box: `{x, y, width, height}` in pixel coordinates + - Rules: + - `x, y`: top-left corner of item (pixel offset from image top-left) + - `width, height`: dimensions of item bounding box + - Include minimal padding (10-15 pixels) around item edges + - Ignore background clutter, other items, hands, reflections + + ### Rotation Analysis + - Check if item labels/text are readable + - If text is rotated (not horizontal), calculate rotation needed + - Return `rotation_degrees`: degrees to rotate CLOCKWISE to make text readable + - Examples: + - Text rotated 90° counter-clockwise → return 90 (rotate 90° clockwise) + - Text rotated 45° clockwise → return -45 (rotate 45° counter-clockwise) + - Text already readable → return 0 + + ### Confidence Score + - Return `confidence`: 0.0-1.0 indicating reliability of crop/rotation analysis + - 0.9+ = High confidence (clear item, readable text) + - 0.7-0.89 = Medium confidence (some ambiguity or text partially obscured) + - <0.7 = Low confidence (cluttered image, unclear item boundaries) + + ### Output Format (Extended) ```json { "items": [ @@ -90,9 +118,20 @@ "Size": "human_readable_size", "Color": "color", "PartNr": "part_number", - "OCR": "TYPE SIZE VENDOR CONNECTOR PARTNUMBER" + "OCR": "TYPE SIZE VENDOR CONNECTOR PARTNUMBER", + "image_processing": { + "crop_bounds": { + "x": 50, + "y": 100, + "width": 300, + "height": 200 + }, + "rotation_degrees": 15, + "confidence": 0.92 + } } ] } + ``` - Return ONLY JSON. No markdown. No text. + **Return ONLY JSON. No markdown. No text.** diff --git a/config/ai_prompt.md.example b/config/ai_prompt.md.example new file mode 100644 index 00000000..23cec56e --- /dev/null +++ b/config/ai_prompt.md.example @@ -0,0 +1,98 @@ + # Technical Inventory Hardware Extraction Protocol + + Extract ALL relevant hardware items from the image with precise, standardized formatting. + + ## Filtering Rules + - **INCLUDE**: Physical hardware, modules, cables, servers, storage, transceivers + - **EXCLUDE**: Generic mounting hardware (screws, brackets, rails), paper licenses, empty packaging + - **Multi-item labels**: Treat each SKU/variant as a separate item (e.g., "5m cable" and "7m cable" = 2 items) + + ## Item Field Format (CRITICAL) + [] + + **Component Rules:** + - ``: + - **STORAGE CAPACITY - HUMAN READABLE**: Convert to largest unit (TB/MB). + - Examples: "1600GB" → "1.6TB", "256GB" → "256GB", "512MB" → "512MB" + - Rule: If ≥1000GB, use TB. If ≥1000MB, use GB. Otherwise use MB. + - **CABLE/WIRE LENGTH**: Meters only. Examples: "5m", "10m", "50m" + - **RAM DIMM**: Capacity in GB. Examples: "128GB", "32GB", "8GB" + + - ``: Asset class. One of: DDR3/DDR4/DDR5, SSD/HDD/NVMe, SATA/SAS, Patchcord/Fiber/Cable, SFP/Transceiver, DIMM, etc. + - ``: Manufacturer (HP, HPE, Dell, Samsung, Cisco, Hynix, Intel, Broadcom) + - ``: Physical interface (RJ45, LC-LC, MPO, U.3, SATA, SAS, ST, SC). Omit if N/A. + - ``: Part number ONLY if visible. **Omit serial numbers.** + + **Item Examples (WITH HUMAN-READABLE SIZES):** + - `1.6TB NVMe HPE U.3 P66093-002` (not 1600GB) + - `256GB SSD Dell SATA SK-8765` (already human-readable) + - `5m Patchcord LC-LC` + - `128GB DDR4 Hynix` + - `512MB Cache Samsung SATA` (stays MB if under 1GB) + + **Size Conversion Examples:** + - 1600GB → 1.6TB + - 2048GB → 2TB + - 512GB → 512GB (under 1TB threshold) + - 256MB → 256MB + - 1024MB → 1GB + + **Restrictions:** + - No comments in parenthesis + - No measurement units in Item field (e.g., "1.6TB" not "1.6TB Storage") + - No secondary vendors + - No diameter/mm in Item field + - ONE vendor only (primary manufacturer) + + ## Other Fields + - **Type**: Repeat the asset class (DDR4, SSD, NVMe, Patchcord, etc.) + - **Description**: Technical summary, max 5 words. Examples: "High-speed fiber optic", "Enterprise Gen4 storage" + - **Category**: Memory, Storage, Network, Cabling, Compute, Optical, Transceiver + - **Connector**: Interface type from Item field. Examples: "LC-LC", "RJ45", "U.3" + - **Size**: **HUMAN-READABLE capacity or length.** Examples: "1.6TB", "256GB", "5m" (NOT "1600GB") + - **Color**: Physical color if distinguishing + - **PartNr**: Part number only (no serial numbers) + - **OCR**: Robust matching key for OCR tolerance + + ## OCR Field Rules (CRITICAL) + Generate a SHORT, clean matching key: + - Format: **UPPERCASE space-separated, NO special chars, NO duplicates** + - Include ONLY: Type + Size + Primary Vendor + Connector + Part Number + - **EXCLUDE**: Serial numbers, secondary vendors, duplicate tokens, EMC/SK labels + - **USE HUMAN-READABLE SIZE**: Use TB/GB from Item field, not original notation + + **OCR Format:** `TYPE SIZE VENDOR CONNECTOR PARTNUMBER` + + **OCR Examples (WITH HUMAN-READABLE SIZES):** + - Item: `1.6TB NVMe HPE U.3 P66093-002` → OCR: `NVME 1.6TB HPE U3 P66093002` + - Item: `5m Patchcord LC-LC` → OCR: `PATCHCORD 5M LC LC` + - Item: `256GB SSD Samsung SAS SK-8765` → OCR: `SSD 256GB SAMSUNG SAS SK8765` + - Item: `128GB DDR4 Hynix` → OCR: `DDR4 128GB HYNIX` + + **OCR Constraints:** + - NO duplicate part numbers + - NO secondary vendor names + - NO extraneous labels + - Each token appears ONE time only + - Remove hyphens/special chars for fuzzy matching + - Use HUMAN-READABLE sizes (1.6TB not 1600GB) + + ## Output Format + ```json + { + "items": [ + { + "Item": "[size] type vendor connector partnumber", + "Type": "type", + "Description": "technical details (max 5 words)", + "Category": "category", + "Connector": "connector_type", + "Size": "human_readable_size", + "Color": "color", + "PartNr": "part_number", + "OCR": "TYPE SIZE VENDOR CONNECTOR PARTNUMBER" + } + ] + } + + Return ONLY JSON. No markdown. No text.