diff --git a/frontend/hooks/useAIExtraction.ts b/frontend/hooks/useAIExtraction.ts index 534a4ed7..ed3b95e3 100644 --- a/frontend/hooks/useAIExtraction.ts +++ b/frontend/hooks/useAIExtraction.ts @@ -10,6 +10,7 @@ export function useAIExtraction(inventory: Item[], onComplete: (itemData: any) = const [editingIndex, setEditingIndex] = useState(null); const [mode, setMode] = useState<'item' | 'box'>('item'); const [isLive, setIsLive] = useState(false); + const [extractedImageBlob, setExtractedImageBlob] = useState(null); const videoRef = useRef(null); const canvasRef = useRef(null); @@ -74,6 +75,8 @@ export function useAIExtraction(inventory: Item[], onComplete: (itemData: any) = try { const blob = await (await fetch(image)).blob(); + setExtractedImageBlob(blob); + const formData = new FormData(); formData.append('file', blob, 'label.jpg'); @@ -236,6 +239,8 @@ export function useAIExtraction(inventory: Item[], onComplete: (itemData: any) = fileInputRef, existingTypes, existingBoxes, + extractedImageBlob, + setExtractedImageBlob, startLiveCamera, stopLiveCamera, captureSnapshot, diff --git a/frontend/tests/hooks/useAIExtraction.test.ts b/frontend/tests/hooks/useAIExtraction.test.ts new file mode 100644 index 00000000..57b2c45e --- /dev/null +++ b/frontend/tests/hooks/useAIExtraction.test.ts @@ -0,0 +1,455 @@ +import { renderHook, act } from '@testing-library/react'; +import { useAIExtraction } from '@/hooks/useAIExtraction'; +import * as api from '@/lib/api'; +import { toast } from 'react-hot-toast'; + +jest.mock('@/lib/api'); +jest.mock('react-hot-toast'); + +describe('useAIExtraction', () => { + const mockInventory = [ + { id: 1, name: 'Item 1', type: 'Type A', box_label: 'Box 1' }, + { id: 2, name: 'Item 2', type: 'Type B', box_label: 'Box 2' } + ]; + + const mockOnComplete = jest.fn(); + + beforeEach(() => { + jest.clearAllMocks(); + (api.inventoryApi.analyzeLabel as jest.Mock).mockResolvedValue({ + items: [ + { + name: 'Test Item', + Item: 'Test Item', + category: 'Electronics', + Category: 'Electronics', + type: 'Resistor', + Type: 'Resistor', + part_number: 'R-001', + PartNr: 'R-001', + image_processing: { + crop_bounds: { x: 10, y: 20, width: 100, height: 100 }, + rotation_degrees: 0, + confidence: 0.95 + } + } + ] + }); + }); + + describe('extractedImageBlob state', () => { + it('should initialize extractedImageBlob as null', () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + expect(result.current.extractedImageBlob).toBeNull(); + }); + + it('should store blob after processImage fetches from data URL', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['fake image data'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,abc123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedImageBlob).toBe(mockBlob); + }); + + it('should allow manual setExtractedImageBlob', () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const testBlob = new Blob(['test data'], { type: 'image/jpeg' }); + + act(() => { + result.current.setExtractedImageBlob(testBlob); + }); + + expect(result.current.extractedImageBlob).toBe(testBlob); + }); + + it('should allow clearing extractedImageBlob by setting to null', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['test'], { type: 'image/jpeg' }); + + act(() => { + result.current.setExtractedImageBlob(mockBlob); + }); + + expect(result.current.extractedImageBlob).toBe(mockBlob); + + act(() => { + result.current.setExtractedImageBlob(null); + }); + + expect(result.current.extractedImageBlob).toBeNull(); + }); + }); + + describe('extractedItems with image_processing metadata', () => { + it('should store extractedItems with image_processing from AI response', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['image'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,abc123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedItems).toHaveLength(1); + expect(result.current.extractedItems[0]).toMatchObject({ + name: 'Test Item', + category: 'Electronics', + type: 'Resistor', + part_number: 'R-001', + image_processing: { + crop_bounds: { x: 10, y: 20, width: 100, height: 100 }, + rotation_degrees: 0, + confidence: 0.95 + } + }); + }); + + it('should preserve image_processing when handling wrapped AI responses', async () => { + (api.inventoryApi.analyzeLabel as jest.Mock).mockResolvedValue({ + data: { + items: [ + { + name: 'Wrapped Item', + Item: 'Wrapped Item', + image_processing: { + crop_bounds: { x: 5, y: 15, width: 200, height: 150 }, + rotation_degrees: 90, + confidence: 0.87 + } + } + ] + } + }); + + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['image'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,xyz789'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedItems[0].image_processing).toEqual({ + crop_bounds: { x: 5, y: 15, width: 200, height: 150 }, + rotation_degrees: 90, + confidence: 0.87 + }); + }); + + it('should handle multiple items each with independent image_processing', async () => { + (api.inventoryApi.analyzeLabel as jest.Mock).mockResolvedValue({ + items: [ + { + name: 'Item 1', + Item: 'Item 1', + image_processing: { + crop_bounds: { x: 0, y: 0, width: 100, height: 100 }, + rotation_degrees: 0, + confidence: 0.9 + } + }, + { + name: 'Item 2', + Item: 'Item 2', + image_processing: { + crop_bounds: { x: 110, y: 0, width: 100, height: 100 }, + rotation_degrees: 45, + confidence: 0.85 + } + } + ] + }); + + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['image'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,multi123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedItems).toHaveLength(2); + expect(result.current.extractedItems[0].image_processing.crop_bounds).toEqual({ + x: 0, + y: 0, + width: 100, + height: 100 + }); + expect(result.current.extractedItems[1].image_processing.crop_bounds).toEqual({ + x: 110, + y: 0, + width: 100, + height: 100 + }); + }); + }); + + describe('blob and metadata together', () => { + it('should store both blob and image_processing for use in photo upload', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['image data'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,together123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + // Both blob and metadata should be available + expect(result.current.extractedImageBlob).toBe(mockBlob); + expect(result.current.extractedItems).toHaveLength(1); + + const item = result.current.extractedItems[0]; + expect(item.image_processing).toBeDefined(); + expect(item.image_processing.crop_bounds).toBeDefined(); + }); + + it('should maintain blob when extractedItems are updated', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['image'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,maintain123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + const originalBlob = result.current.extractedImageBlob; + + act(() => { + result.current.updateEditingItem({ name: 'Updated Name' }); + }); + + expect(result.current.extractedImageBlob).toBe(originalBlob); + }); + }); + + describe('cleanup and reset', () => { + it('should clear extractedImageBlob when resetting extracted items', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['image'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,reset123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedImageBlob).toBe(mockBlob); + + act(() => { + result.current.setExtractedItems([]); + result.current.setExtractedImageBlob(null); + }); + + expect(result.current.extractedItems).toHaveLength(0); + expect(result.current.extractedImageBlob).toBeNull(); + }); + + it('should allow resetting image without affecting blob storage', () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['image'], { type: 'image/jpeg' }); + + act(() => { + result.current.setExtractedImageBlob(mockBlob); + result.current.setImage('data:image/jpeg;base64,somedata'); + }); + + expect(result.current.extractedImageBlob).toBe(mockBlob); + expect(result.current.image).toBe('data:image/jpeg;base64,somedata'); + + act(() => { + result.current.setImage(null); + }); + + expect(result.current.extractedImageBlob).toBe(mockBlob); + expect(result.current.image).toBeNull(); + }); + }); + + describe('error handling', () => { + it('should not set extractedImageBlob if fetch fails', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const dataURL = 'data:image/jpeg;base64,error123'; + global.fetch = jest.fn().mockRejectedValue(new Error('Fetch failed')); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedImageBlob).toBeNull(); + }); + + it('should not set extractedImageBlob if blob conversion fails', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const dataURL = 'data:image/jpeg;base64,blobfail123'; + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockRejectedValue(new Error('Blob conversion failed')) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedImageBlob).toBeNull(); + }); + }); + + describe('accessibility for photo upload', () => { + it('should provide extractedImageBlob as FormData-ready Blob for later photo upload', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const mockBlob = new Blob(['fake jpeg data'], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,formdata123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + // Blob should be usable in FormData + const formData = new FormData(); + formData.append('file', result.current.extractedImageBlob!, 'photo.jpg'); + + expect(formData.get('file')).toBe(mockBlob); + }); + + it('should preserve blob size and type for upload validation', async () => { + const { result } = renderHook(() => + useAIExtraction(mockInventory, mockOnComplete) + ); + + const blobData = new Uint8Array(5000); // 5KB blob + const mockBlob = new Blob([blobData], { type: 'image/jpeg' }); + const dataURL = 'data:image/jpeg;base64,size123'; + + global.fetch = jest.fn().mockResolvedValue({ + blob: jest.fn().mockResolvedValue(mockBlob) + }); + + act(() => { + result.current.setImage(dataURL); + }); + + await act(async () => { + await result.current.processImage(); + }); + + expect(result.current.extractedImageBlob?.size).toBe(5000); + expect(result.current.extractedImageBlob?.type).toBe('image/jpeg'); + }); + }); +}); diff --git a/images/electronics/itemwithphoto_5875b86f_thumb.jpg b/images/electronics/itemwithphoto_5875b86f_thumb.jpg new file mode 100644 index 00000000..d2349a6d Binary files /dev/null and b/images/electronics/itemwithphoto_5875b86f_thumb.jpg differ diff --git a/images/electronics/itemwithphoto_fe5cc7b1_original.jpg b/images/electronics/itemwithphoto_fe5cc7b1_original.jpg new file mode 100644 index 00000000..d3016c2a Binary files /dev/null and b/images/electronics/itemwithphoto_fe5cc7b1_original.jpg differ