feat: store extracted image blob and image_processing metadata in useAIExtraction hook

frontend/hooks/useAIExtraction.ts

@@ -10,6 +10,7 @@ export function useAIExtraction(inventory: Item[], onComplete: (itemData: any) =
   const [editingIndex, setEditingIndex] = useState<number | null>(null);
   const [mode, setMode] = useState<'item' | 'box'>('item');
   const [isLive, setIsLive] = useState(false);
+  const [extractedImageBlob, setExtractedImageBlob] = useState<Blob | null>(null);
 
   const videoRef = useRef<HTMLVideoElement>(null);
   const canvasRef = useRef<HTMLCanvasElement>(null);
@@ -74,6 +75,8 @@ export function useAIExtraction(inventory: Item[], onComplete: (itemData: any) =
 
     try {
       const blob = await (await fetch(image)).blob();
+      setExtractedImageBlob(blob);
+
       const formData = new FormData();
       formData.append('file', blob, 'label.jpg');
 
@@ -236,6 +239,8 @@ export function useAIExtraction(inventory: Item[], onComplete: (itemData: any) =
     fileInputRef,
     existingTypes,
     existingBoxes,
+    extractedImageBlob,
+    setExtractedImageBlob,
     startLiveCamera,
     stopLiveCamera,
     captureSnapshot,

frontend/tests/hooks/useAIExtraction.test.ts

@@ -0,0 +1,455 @@
+import { renderHook, act } from '@testing-library/react';
+import { useAIExtraction } from '@/hooks/useAIExtraction';
+import * as api from '@/lib/api';
+import { toast } from 'react-hot-toast';
+
+jest.mock('@/lib/api');
+jest.mock('react-hot-toast');
+
+describe('useAIExtraction', () => {
+  const mockInventory = [
+    { id: 1, name: 'Item 1', type: 'Type A', box_label: 'Box 1' },
+    { id: 2, name: 'Item 2', type: 'Type B', box_label: 'Box 2' }
+  ];
+
+  const mockOnComplete = jest.fn();
+
+  beforeEach(() => {
+    jest.clearAllMocks();
+    (api.inventoryApi.analyzeLabel as jest.Mock).mockResolvedValue({
+      items: [
+        {
+          name: 'Test Item',
+          Item: 'Test Item',
+          category: 'Electronics',
+          Category: 'Electronics',
+          type: 'Resistor',
+          Type: 'Resistor',
+          part_number: 'R-001',
+          PartNr: 'R-001',
+          image_processing: {
+            crop_bounds: { x: 10, y: 20, width: 100, height: 100 },
+            rotation_degrees: 0,
+            confidence: 0.95
+          }
+        }
+      ]
+    });
+  });
+
+  describe('extractedImageBlob state', () => {
+    it('should initialize extractedImageBlob as null', () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      expect(result.current.extractedImageBlob).toBeNull();
+    });
+
+    it('should store blob after processImage fetches from data URL', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['fake image data'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,abc123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedImageBlob).toBe(mockBlob);
+    });
+
+    it('should allow manual setExtractedImageBlob', () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const testBlob = new Blob(['test data'], { type: 'image/jpeg' });
+
+      act(() => {
+        result.current.setExtractedImageBlob(testBlob);
+      });
+
+      expect(result.current.extractedImageBlob).toBe(testBlob);
+    });
+
+    it('should allow clearing extractedImageBlob by setting to null', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['test'], { type: 'image/jpeg' });
+
+      act(() => {
+        result.current.setExtractedImageBlob(mockBlob);
+      });
+
+      expect(result.current.extractedImageBlob).toBe(mockBlob);
+
+      act(() => {
+        result.current.setExtractedImageBlob(null);
+      });
+
+      expect(result.current.extractedImageBlob).toBeNull();
+    });
+  });
+
+  describe('extractedItems with image_processing metadata', () => {
+    it('should store extractedItems with image_processing from AI response', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['image'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,abc123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedItems).toHaveLength(1);
+      expect(result.current.extractedItems[0]).toMatchObject({
+        name: 'Test Item',
+        category: 'Electronics',
+        type: 'Resistor',
+        part_number: 'R-001',
+        image_processing: {
+          crop_bounds: { x: 10, y: 20, width: 100, height: 100 },
+          rotation_degrees: 0,
+          confidence: 0.95
+        }
+      });
+    });
+
+    it('should preserve image_processing when handling wrapped AI responses', async () => {
+      (api.inventoryApi.analyzeLabel as jest.Mock).mockResolvedValue({
+        data: {
+          items: [
+            {
+              name: 'Wrapped Item',
+              Item: 'Wrapped Item',
+              image_processing: {
+                crop_bounds: { x: 5, y: 15, width: 200, height: 150 },
+                rotation_degrees: 90,
+                confidence: 0.87
+              }
+            }
+          ]
+        }
+      });
+
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['image'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,xyz789';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedItems[0].image_processing).toEqual({
+        crop_bounds: { x: 5, y: 15, width: 200, height: 150 },
+        rotation_degrees: 90,
+        confidence: 0.87
+      });
+    });
+
+    it('should handle multiple items each with independent image_processing', async () => {
+      (api.inventoryApi.analyzeLabel as jest.Mock).mockResolvedValue({
+        items: [
+          {
+            name: 'Item 1',
+            Item: 'Item 1',
+            image_processing: {
+              crop_bounds: { x: 0, y: 0, width: 100, height: 100 },
+              rotation_degrees: 0,
+              confidence: 0.9
+            }
+          },
+          {
+            name: 'Item 2',
+            Item: 'Item 2',
+            image_processing: {
+              crop_bounds: { x: 110, y: 0, width: 100, height: 100 },
+              rotation_degrees: 45,
+              confidence: 0.85
+            }
+          }
+        ]
+      });
+
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['image'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,multi123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedItems).toHaveLength(2);
+      expect(result.current.extractedItems[0].image_processing.crop_bounds).toEqual({
+        x: 0,
+        y: 0,
+        width: 100,
+        height: 100
+      });
+      expect(result.current.extractedItems[1].image_processing.crop_bounds).toEqual({
+        x: 110,
+        y: 0,
+        width: 100,
+        height: 100
+      });
+    });
+  });
+
+  describe('blob and metadata together', () => {
+    it('should store both blob and image_processing for use in photo upload', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['image data'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,together123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      // Both blob and metadata should be available
+      expect(result.current.extractedImageBlob).toBe(mockBlob);
+      expect(result.current.extractedItems).toHaveLength(1);
+
+      const item = result.current.extractedItems[0];
+      expect(item.image_processing).toBeDefined();
+      expect(item.image_processing.crop_bounds).toBeDefined();
+    });
+
+    it('should maintain blob when extractedItems are updated', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['image'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,maintain123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      const originalBlob = result.current.extractedImageBlob;
+
+      act(() => {
+        result.current.updateEditingItem({ name: 'Updated Name' });
+      });
+
+      expect(result.current.extractedImageBlob).toBe(originalBlob);
+    });
+  });
+
+  describe('cleanup and reset', () => {
+    it('should clear extractedImageBlob when resetting extracted items', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['image'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,reset123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedImageBlob).toBe(mockBlob);
+
+      act(() => {
+        result.current.setExtractedItems([]);
+        result.current.setExtractedImageBlob(null);
+      });
+
+      expect(result.current.extractedItems).toHaveLength(0);
+      expect(result.current.extractedImageBlob).toBeNull();
+    });
+
+    it('should allow resetting image without affecting blob storage', () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['image'], { type: 'image/jpeg' });
+
+      act(() => {
+        result.current.setExtractedImageBlob(mockBlob);
+        result.current.setImage('data:image/jpeg;base64,somedata');
+      });
+
+      expect(result.current.extractedImageBlob).toBe(mockBlob);
+      expect(result.current.image).toBe('data:image/jpeg;base64,somedata');
+
+      act(() => {
+        result.current.setImage(null);
+      });
+
+      expect(result.current.extractedImageBlob).toBe(mockBlob);
+      expect(result.current.image).toBeNull();
+    });
+  });
+
+  describe('error handling', () => {
+    it('should not set extractedImageBlob if fetch fails', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const dataURL = 'data:image/jpeg;base64,error123';
+      global.fetch = jest.fn().mockRejectedValue(new Error('Fetch failed'));
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedImageBlob).toBeNull();
+    });
+
+    it('should not set extractedImageBlob if blob conversion fails', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const dataURL = 'data:image/jpeg;base64,blobfail123';
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockRejectedValue(new Error('Blob conversion failed'))
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedImageBlob).toBeNull();
+    });
+  });
+
+  describe('accessibility for photo upload', () => {
+    it('should provide extractedImageBlob as FormData-ready Blob for later photo upload', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const mockBlob = new Blob(['fake jpeg data'], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,formdata123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      // Blob should be usable in FormData
+      const formData = new FormData();
+      formData.append('file', result.current.extractedImageBlob!, 'photo.jpg');
+
+      expect(formData.get('file')).toBe(mockBlob);
+    });
+
+    it('should preserve blob size and type for upload validation', async () => {
+      const { result } = renderHook(() =>
+        useAIExtraction(mockInventory, mockOnComplete)
+      );
+
+      const blobData = new Uint8Array(5000); // 5KB blob
+      const mockBlob = new Blob([blobData], { type: 'image/jpeg' });
+      const dataURL = 'data:image/jpeg;base64,size123';
+
+      global.fetch = jest.fn().mockResolvedValue({
+        blob: jest.fn().mockResolvedValue(mockBlob)
+      });
+
+      act(() => {
+        result.current.setImage(dataURL);
+      });
+
+      await act(async () => {
+        await result.current.processImage();
+      });
+
+      expect(result.current.extractedImageBlob?.size).toBe(5000);
+      expect(result.current.extractedImageBlob?.type).toBe('image/jpeg');
+    });
+  });
+});