Files
tfm_ainventory/frontend/hooks/useAIExtraction.ts
Daniel Bedeleanu 8d47732de4 debug: add comprehensive logging for image adjustment flow
Added detailed console logging across entire image adjustment pipeline:

1. ImageAdjustmentModal.handleConfirm():
   - Original image dimensions
   - User inputs (rotation, zoom, pan, crop)
   - Canvas processing steps
   - Final blob size

2. AIOnboarding.handleImageAdjustmentConfirm():
   - Adjustments received from modal
   - Item being updated
   - extractedImageBlob status before/after

3. useAIExtraction.confirmSingleItem():
   - newItem being built
   - extractedImageBlob attached
   - imageProcessing attached

This will help identify where values are lost or incorrect in the flow.

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>
2026-04-22 13:53:05 +03:00

281 lines
9.5 KiB
TypeScript

import { useState, useRef, useEffect, useMemo } from 'react';
import { toast } from 'react-hot-toast';
import { inventoryApi } from '@/lib/api';
import { Item } from '@/lib/db';
export function useAIExtraction(inventory: Item[], onComplete: (itemData: any) => void) {
const [image, setImage] = useState<string | null>(null);
const [uploading, setUploading] = useState(false);
const [extractedItems, setExtractedItems] = useState<any[]>([]);
const [editingIndex, setEditingIndex] = useState<number | null>(null);
const [mode, setMode] = useState<'item' | 'box'>('item');
const [isLive, setIsLive] = useState(false);
const [extractedImageBlob, setExtractedImageBlob] = useState<Blob | null>(null);
const videoRef = useRef<HTMLVideoElement>(null);
const canvasRef = useRef<HTMLCanvasElement>(null);
const streamRef = useRef<MediaStream | null>(null);
const fileInputRef = useRef<HTMLInputElement>(null);
const existingTypes = useMemo(
() => Array.from(new Set(inventory.map(i => i.type).filter(Boolean))).sort() as string[],
[inventory]
);
const existingBoxes = useMemo(
() => Array.from(new Set(inventory.map(i => i.box_label).filter(Boolean))).sort() as string[],
[inventory]
);
const startLiveCamera = async () => {
try {
setIsLive(true);
const stream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: 'environment', width: { ideal: 1920 }, height: { ideal: 1080 } },
audio: false
});
if (videoRef.current) {
videoRef.current.srcObject = stream;
streamRef.current = stream;
}
} catch (err) {
console.error("Camera access error:", err);
toast.error("Could not access camera for live scan.");
setIsLive(false);
}
};
const stopLiveCamera = () => {
if (streamRef.current) {
streamRef.current.getTracks().forEach(track => track.stop());
streamRef.current = null;
}
setIsLive(false);
};
const captureSnapshot = () => {
if (videoRef.current && canvasRef.current) {
const video = videoRef.current;
const canvas = canvasRef.current;
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const ctx = canvas.getContext('2d');
if (ctx) {
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
const dataUrl = canvas.toDataURL('image/jpeg', 0.85);
setImage(dataUrl);
stopLiveCamera();
}
}
};
const processImage = async () => {
if (!image) return;
setUploading(true);
try {
const blob = await (await fetch(image)).blob();
setExtractedImageBlob(blob);
const formData = new FormData();
formData.append('file', blob, 'label.jpg');
const data = await inventoryApi.analyzeLabel(formData, mode);
if (data.error) {
toast.error(`AI Error: ${data.error}`);
setUploading(false);
return;
}
let parsedData = data;
if (typeof data === 'string') {
try { parsedData = JSON.parse(data); } catch (e) {}
}
const d = parsedData;
// Find ANY array in the response if it's not a direct array
let items: any[] = [];
if (Array.isArray(d)) {
items = d;
} else {
const potentialArrayKey = Object.keys(d).find(k => Array.isArray(d[k]));
if (potentialArrayKey) {
items = d[potentialArrayKey];
} else {
// Check for singular object (must have at least name or Item or PN)
const target = d.data || d;
if (target.name || target.Item || target.PartNr || target.part_number) {
items = [target];
}
}
}
if (!items || items.length === 0) {
toast.error("No relevant items detected. Try a closer photo.");
} else {
setExtractedItems(items);
if (items.length === 1) {
setEditingIndex(0);
toast.success("Item identified!");
} else {
toast.success(`Found ${items.length} items!`);
}
}
} catch (error) {
toast.error("Failed to process image with AI");
console.error(error);
} finally {
setUploading(false);
}
};
const confirmSingleItem = (index: number) => {
const data = extractedItems[index];
const skipPhoto = data._skipPhoto === true; // User explicitly rejected the photo
console.log('=== CONFIRM SINGLE ITEM (Hook) ===');
console.log('[Input] Item index:', index);
console.log('[Input] Item data:', data);
console.log('[Input] Skip photo flag:', skipPhoto);
console.log('[Input] extractedImageBlob size:', extractedImageBlob?.size || 'null', 'bytes');
console.log('[Input] image_processing from data:', data.image_processing);
const newItem = {
name: String(data.Item || data.name || "New AI Item"),
category: String(data.Category || data.category || "Uncategorized"),
type: data.Type || data.type ? String(data.Type || data.type) : null,
part_number: data.PartNr || data.part_number ? String(data.PartNr || data.part_number) : null,
color: data.Color || data.color ? String(data.Color || data.color) : null,
description: String(data.Description || data.description || ""),
connector: data.Connector || data.connector ? String(data.Connector || data.connector) : null,
size: data.Size || data.size ? String(data.Size || data.size) : null,
ocr_text: data.OCR || data.ocr_text ? String(data.OCR || data.ocr_text) : null,
specs: String(data.specs || ""),
barcode: String(data.barcode || data.PartNr || data.part_number || `AI-${Date.now()}-${index}`),
quantity: parseFloat(String(data.quantity || 1)),
min_quantity: 1.0,
box_label: data.box_label ? String(data.box_label) : null,
labels_data: JSON.stringify(data),
// Only pass image blob if user approved it
...(skipPhoto ? {} : {
extractedImageBlob,
imageProcessing: data.image_processing
})
};
console.log('[Output] newItem being sent to onComplete:', newItem);
console.log('[Output] extractedImageBlob attached:', !!newItem.extractedImageBlob);
console.log('[Output] imageProcessing attached:', !!newItem.imageProcessing);
console.log('=== END HOOK DEBUG ===');
onComplete(newItem);
if (extractedItems.length > 1) {
const remaining = [...extractedItems];
remaining.splice(index, 1);
setExtractedItems(remaining);
setEditingIndex(null);
} else {
setExtractedItems([]);
setEditingIndex(null);
}
};
const confirmAllItems = async () => {
const itemsToProcess = [...extractedItems];
setUploading(true);
const toastId = toast.loading(`Adding ${itemsToProcess.length} items...`);
try {
for (let i = 0; i < itemsToProcess.length; i++) {
const data = itemsToProcess[i];
const skipPhoto = data._skipPhoto === true;
const newItem = {
name: String(data.Item || data.name || "New AI Item"),
category: String(data.Category || data.category || "Uncategorized"),
type: data.Type || data.type ? String(data.Type || data.type) : null,
part_number: data.PartNr || data.part_number ? String(data.PartNr || data.part_number) : null,
color: data.Color || data.color ? String(data.Color || data.color) : null,
description: String(data.Description || data.description || ""),
connector: data.Connector || data.connector ? String(data.Connector || data.connector) : null,
size: data.Size || data.size ? String(data.Size || data.size) : null,
ocr_text: data.OCR || data.ocr_text ? String(data.OCR || data.ocr_text) : null,
specs: String(data.specs || ""),
barcode: String(data.barcode || data.PartNr || data.part_number || `AI-${Date.now()}-${i}`),
quantity: parseFloat(String(data.quantity || 1)),
min_quantity: 1.0,
box_label: data.box_label ? String(data.box_label) : null,
labels_data: JSON.stringify(data),
// Only pass image blob if user approved it
...(skipPhoto ? {} : {
extractedImageBlob,
imageProcessing: data.image_processing
})
};
await onComplete(newItem);
}
toast.success(`Successfully added ${itemsToProcess.length} items`, { id: toastId });
setExtractedItems([]);
} catch (err) {
toast.error("Error during batch add", { id: toastId });
} finally {
setUploading(false);
}
};
const updateEditingItem = (fields: any) => {
if (editingIndex === null) return;
const newItems = [...extractedItems];
newItems[editingIndex] = { ...newItems[editingIndex], ...fields };
setExtractedItems(newItems);
};
const handleFileChange = (e: React.ChangeEvent<HTMLInputElement>) => {
const file = e.target.files?.[0];
if (file) {
const reader = new FileReader();
reader.onload = () => setImage(reader.result as string);
reader.readAsDataURL(file);
}
};
useEffect(() => {
return () => {
if (streamRef.current) {
streamRef.current.getTracks().forEach(track => track.stop());
}
};
}, []);
return {
image,
setImage,
uploading,
extractedItems,
setExtractedItems,
editingIndex,
setEditingIndex,
mode,
setMode,
isLive,
videoRef,
canvasRef,
fileInputRef,
existingTypes,
existingBoxes,
extractedImageBlob,
setExtractedImageBlob,
startLiveCamera,
stopLiveCamera,
captureSnapshot,
processImage,
confirmSingleItem,
confirmAllItems,
updateEditingItem,
handleFileChange
};
}