2026-06-26 06:13:33

前端 + AI 进阶 Day6: 图片标注与 AI 视觉分析（完整版）

);

};

export default ImageUpload;

步骤 3：创建图片标注组件

// src/components/ImageAnnotator.jsx

import { useState, useRef, useEffect } from 'react';

const ImageAnnotator = ({ imageFile, onAnnotated }) => {

const canvasRef = useRef(null);

const [isDrawing, setIsDrawing] = useState(false);

const [annotations, setAnnotations] = useState([]);

const [mode, setMode] = useState('rect'); // 'rect' | 'lasso'

const [tempPoints, setTempPoints] = useState([]);

const imgRef = useRef(null);

// 加载图片到内存

useEffect(() => {

if (!imageFile) return;

const img = new Image();

img.onload = () => {

imgRef.current = img;

drawImageAndAnnotations(img, []);

};

img.src = URL.createObjectURL(imageFile);

return () => URL.revokeObjectURL(img.src);

}, [imageFile]);

const drawImageAndAnnotations = (img, anns) => {

const canvas = canvasRef.current;

if (!canvas || !img) return;

const ctx = canvas.getContext('2d');

const maxWidth = 800;

const scale = Math.min(maxWidth / img.width, 1);

const w = img.width * scale;

const h = img.height * scale;

canvas.width = w;

canvas.height = h;

ctx.clearRect(0, 0, w, h);

ctx.drawImage(img, 0, 0, w, h);

// 绘制已有标注

anns.forEach((ann) => {

ctx.strokeStyle = ann.type === 'rect' ? '#1890ff' : '#f5222d';

ctx.lineWidth = 2;

ctx.beginPath();

if (ann.type === 'rect' && ann.points.length === 2) {

const [start, end] = ann.points;

ctx.rect(start.x, start.y, end.x - start.x, end.y - start.y);

} else if (ann.type === 'lasso' && ann.points.length > 1) {

ctx.moveTo(ann.points[0].x, ann.points[0].y);

ann.points.slice(1).forEach((p) => ctx.lineTo(p.x, p.y));

ctx.closePath();

}

ctx.stroke();

// 绘制标签

if (ann.label) {

ctx.fillStyle = '#1890ff';

ctx.font = '14px sans-serif';

ctx.fillText(ann.label, ann.points[0].x + 5, ann.points[0].y - 5);

}

});

};

const getMousePos = (e) => {

const canvas = canvasRef.current;

const rect = canvas.getBoundingClientRect();

const scaleX = canvas.width / rect.width;

const scaleY = canvas.height / rect.height;

return {

x: (e.clientX - rect.left) * scaleX,

y: (e.clientY - rect.top) * scaleY,

};

};

const handleMouseDown = (e) => {

if (!imgRef.current) return;

setIsDrawing(true);

const pos = getMousePos(e);

if (mode === 'rect') {

setTempPoints([pos, pos]);

} else if (mode === 'lasso') {

setTempPoints([pos]);

}

};

const handleMouseMove = (e) => {

if (!isDrawing || !imgRef.current) return;

const pos = getMousePos(e);

if (mode === 'rect') {

setTempPoints(([start]) => [start, pos]);

} else if (mode === 'lasso') {

setTempPoints((prev) => [...prev, pos]);

}

};

const handleMouseUp = () => {

if (!isDrawing) return;

setIsDrawing(false);

const label = prompt('请输入标注标签（如“错误弹窗”）：', '');

if (tempPoints.length > 0) {

const newAnn = {

id: Date.now(),

type: mode,

points: [...tempPoints],

label: label || '未命名',

};

const updated = [...annotations, newAnn];

setAnnotations(updated);

drawImageAndAnnotations(imgRef.current, updated);

setTempPoints([]);

onAnnotated?.(updated);

}

};

// 重绘临时图形

useEffect(() => {

if (isDrawing && imgRef.current) {

const allAnns = [...annotations, { type: mode, points: tempPoints }];

drawImageAndAnnotations(imgRef.current, allAnns);

}

}, [tempPoints, isDrawing]);

return (

);

};

export default ImageAnnotator;

步骤 4：创建视觉 AI 客户端（模拟流式）

// src/lib/visualAIClient.js

/**

* 模拟 LLaVA 视觉分析（真实版见 Day 6 扩展）

* 返回预设响应以演示流程

*/

export const streamVisualAnalysis = async ({ prompt, onToken, onComplete }) => {

const mockResponse = `根据你的标注，我分析如下：

- 图中包含一个用户界面截图

- **“错误弹窗”** 区域显示了一个红色警告图标和“网络连接失败”文本

- **“提交按钮”** 是一个蓝色矩形按钮，带有白色“提交”文字

建议：检查网络设置或重试操作。`;

let index = 0;

const interval = setInterval(() => {

if (index < mockResponse.length) {

onToken(mockResponse[index]);

index++;

} else {

clearInterval(interval);

onComplete();

}

}, 30);

};

步骤 5：在 App 中集成完整流程

// src/App.jsx

import { useState } from 'react';

import ImageUpload from './components/ImageUpload';

import ImageAnnotator from './components/ImageAnnotator';

import { streamVisualAnalysis } from './lib/visualAIClient';

function App() {

const [imageFile, setImageFile] = useState(null);

const [annotations, setAnnotations] = useState([]);

const [aiResponse, setAiResponse] = useState('');

const [isAnalyzing, setIsAnalyzing] = useState(false);

const [userQuestion, setUserQuestion] = useState('');

const handleAnnotated = (anns) => {

setAnnotations(anns);

};

const sendToVisualAI = async () => {

if (!imageFile || !userQuestion.trim()) {

alert('请先上传图片并输入问题');

return;

}

// 构建带标注的提示词

let fullPrompt = userQuestion;

if (annotations.length > 0) {

fullPrompt += '\n\n用户特别标注了以下区域：\n';

annotations.forEach((ann, i) => {

fullPrompt += `${i + 1}. ${ann.label}（${ann.type === 'rect' ? '矩形区域' : '圈选区域'}）\n`;

});

}

setAiResponse('');

setIsAnalyzing(true);

await streamVisualAnalysis({

prompt: fullPrompt,

onToken: (token) => {

setAiResponse(prev => prev + token);

},

onComplete: () => {

setIsAnalyzing(false);

}

});

};

return (

)}