Spaces:

HGKo
/

vision_llm_agent

Running

App Files Files Community

Sunhey Cho commited on Aug 14

Commit

9aee46b

1 Parent(s): 0e9cdf3

Add frontend source code structure for direct development

Browse files

Files changed (15) hide show

frontend/package-lock.json +0 -0
frontend/package.json +42 -0
frontend/public/index.html +24 -0
frontend/public/manifest.json +15 -0
frontend/src/App.css +47 -0
frontend/src/App.js +207 -0
frontend/src/components/ImageUploader.js +193 -0
frontend/src/components/LlmAnalysis.js +156 -0
frontend/src/components/ModelSelector.js +168 -0
frontend/src/components/OpenAIChat.js +136 -0
frontend/src/components/ResultDisplay.js +198 -0
frontend/src/components/VectorDBActions.js +434 -0
frontend/src/index.css +14 -0
frontend/src/index.js +17 -0
frontend/src/reportWebVitals.js +13 -0

frontend/package-lock.json ADDED Viewed

The diff for this file is too large to render. See raw diff

frontend/package.json ADDED Viewed

	@@ -0,0 +1,42 @@

+{
+  "name": "vision-web-app",
+  "version": "0.1.0",
+  "private": true,
+  "dependencies": {
+    "@material-ui/core": "^4.12.4",
+    "@material-ui/icons": "^4.11.3",
+    "@material-ui/lab": "^4.0.0-alpha.61",
+    "@testing-library/jest-dom": "^4.2.4",
+    "@testing-library/react": "^9.5.0",
+    "@testing-library/user-event": "^7.2.1",
+    "axios": "^0.21.1",
+    "react": "^16.14.0",
+    "react-dom": "^16.14.0",
+    "react-scripts": "3.4.3",
+    "web-vitals": "^0.2.4"
+  },
+  "scripts": {
+    "start": "react-scripts start",
+    "build": "react-scripts build",
+    "test": "react-scripts test",
+    "eject": "react-scripts eject"
+  },
+  "eslintConfig": {
+    "extends": [
+      "react-app"
+    ]
+  },
+  "browserslist": {
+    "production": [
+      ">0.2%",
+      "not dead",
+      "not op_mini all"
+    ],
+    "development": [
+      "last 1 chrome version",
+      "last 1 firefox version",
+      "last 1 safari version"
+    ]
+  },
+  "proxy": "http://localhost:5000"
+}

frontend/public/index.html ADDED Viewed

	@@ -0,0 +1,24 @@

+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="utf-8" />
+    <link rel="icon" href="%PUBLIC_URL%/favicon.ico" />
+    <meta name="viewport" content="width=device-width, initial-scale=1" />
+    <meta name="theme-color" content="#000000" />
+    <meta
+      name="description"
+      content="Multi-Model Object Detection Demo"
+    />
+    <link rel="apple-touch-icon" href="%PUBLIC_URL%/logo192.png" />
+    <link rel="manifest" href="%PUBLIC_URL%/manifest.json" />
+    <title>Vision Web App</title>
+    <link
+      rel="stylesheet"
+      href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700&display=swap"
+    />
+  </head>
+  <body>
+    <noscript>You need to enable JavaScript to run this app.</noscript>
+    <div id="root"></div>
+  </body>
+</html>

frontend/public/manifest.json ADDED Viewed

	@@ -0,0 +1,15 @@

+{
+  "short_name": "Vision Web App",
+  "name": "Multi-Model Object Detection Demo",
+  "icons": [
+    {
+      "src": "favicon.ico",
+      "sizes": "64x64 32x32 24x24 16x16",
+      "type": "image/x-icon"
+    }
+  ],
+  "start_url": ".",
+  "display": "standalone",
+  "theme_color": "#000000",
+  "background_color": "#ffffff"
+}

frontend/src/App.css ADDED Viewed

	@@ -0,0 +1,47 @@

+.App {
+  text-align: center;
+}
+.preview-image {
+  max-width: 100%;
+  max-height: 300px;
+  margin-top: 16px;
+}
+.result-image {
+  max-width: 100%;
+  border: 1px solid #ddd;
+  border-radius: 4px;
+  padding: 4px;
+}
+.detection-list {
+  margin-top: 16px;
+  text-align: left;
+}
+.model-card {
+  cursor: pointer;
+  transition: all 0.3s;
+}
+.model-card:hover {
+  transform: translateY(-5px);
+  box-shadow: 0 10px 20px rgba(0,0,0,0.1);
+}
+.model-card.selected {
+  border: 2px solid #3f51b5;
+  background-color: #e8eaf6;
+}
+.model-card.disabled {
+  opacity: 0.6;
+  cursor: not-allowed;
+}
+.performance-info {
+  margin-top: 16px;
+  font-size: 0.9rem;
+  color: #666;
+}

frontend/src/App.js ADDED Viewed

	@@ -0,0 +1,207 @@

+import React, { useState, useEffect } from 'react';
+import {
+  Container,
+  Typography,
+  Box,
+  Paper,
+  Grid,
+  CircularProgress,
+  AppBar,
+  Toolbar,
+  ThemeProvider,
+  createMuiTheme
+} from '@material-ui/core';
+import ImageUploader from './components/ImageUploader';
+import ModelSelector from './components/ModelSelector';
+import ResultDisplay from './components/ResultDisplay';
+import LlmAnalysis from './components/LlmAnalysis';
+import OpenAIChat from './components/OpenAIChat';
+import './App.css';
+// Create a theme
+const theme = createMuiTheme({
+  palette: {
+    primary: {
+      main: '#3f51b5',
+    },
+    secondary: {
+      main: '#f50057',
+    },
+  },
+  typography: {
+    fontFamily: 'Roboto, Arial, sans-serif',
+  },
+});
+function App() {
+  const [selectedImage, setSelectedImage] = useState(null);
+  const [selectedModel, setSelectedModel] = useState('');
+  const [isProcessing, setIsProcessing] = useState(false);
+  const [results, setResults] = useState(null);
+  const [error, setError] = useState(null);
+  const [modelsStatus, setModelsStatus] = useState({
+    yolo: false,
+    detr: false,
+    vit: false
+  });
+  // Check API status on component mount
+  useEffect(() => {
+    fetch('/api/status')
+      .then(response => response.json())
+      .then(data => {
+        setModelsStatus(data.models);
+      })
+      .catch(err => {
+        console.error('Error checking API status:', err);
+        setError('Error connecting to the backend API. Please make sure the server is running.');
+      });
+  }, []);
+  const handleImageUpload = (image) => {
+    setSelectedImage(image);
+    setResults(null);
+    setError(null);
+  };
+  const handleModelSelect = (model) => {
+    setSelectedModel(model);
+    setResults(null);
+    setError(null);
+  };
+  const processImage = async () => {
+    if (!selectedImage || !selectedModel) {
+      setError('Please select both an image and a model');
+      return;
+    }
+    setIsProcessing(true);
+    setError(null);
+    // Create form data for the image
+    const formData = new FormData();
+    formData.append('image', selectedImage);
+    let endpoint = '';
+    switch (selectedModel) {
+      case 'yolo':
+        endpoint = '/api/detect/yolo';
+        break;
+      case 'detr':
+        endpoint = '/api/detect/detr';
+        break;
+      case 'vit':
+        endpoint = '/api/classify/vit';
+        break;
+      default:
+        setError('Invalid model selection');
+        setIsProcessing(false);
+        return;
+    }
+    try {
+      const response = await fetch(endpoint, {
+        method: 'POST',
+        body: formData,
+      });
+      if (!response.ok) {
+        throw new Error(`HTTP error! Status: ${response.status}`);
+      }
+      const data = await response.json();
+      setResults({ model: selectedModel, data });
+    } catch (err) {
+      console.error('Error processing image:', err);
+      setError(`Error processing image: ${err.message}`);
+    } finally {
+      setIsProcessing(false);
+    }
+  };
+  return (
+    <ThemeProvider theme={theme}>
+      <Box style={{ flexGrow: 1 }}>
+        <AppBar position="static">
+          <Toolbar>
+            <Typography variant="h6" style={{ flexGrow: 1 }}>
+              Multi-Model Object Detection Demo
+            </Typography>
+          </Toolbar>
+        </AppBar>
+        <Container maxWidth="lg" style={{ marginTop: theme.spacing(4), marginBottom: theme.spacing(4) }}>
+          <Grid container spacing={3}>
+            <Grid item xs={12}>
+              <Paper style={{ padding: theme.spacing(2) }}>
+                <Typography variant="h5" gutterBottom>
+                  Upload an image to see how each model performs!
+                </Typography>
+                <Typography variant="body1" paragraph>
+                  This demo showcases three different object detection and image classification models:
+                </Typography>
+                <Typography variant="body1" component="div">
+                  <ul>
+                    <li><strong>YOLOv8</strong>: Fast and accurate object detection</li>
+                    <li><strong>DETR</strong>: DEtection TRansformer for object detection</li>
+                    <li><strong>ViT</strong>: Vision Transformer for image classification</li>
+                  </ul>
+                </Typography>
+              </Paper>
+            </Grid>
+            <Grid item xs={12} md={6}>
+              <ImageUploader onImageUpload={handleImageUpload} />
+            </Grid>
+            <Grid item xs={12} md={6}>
+              <ModelSelector
+                onModelSelect={handleModelSelect}
+                onProcess={processImage}
+                isProcessing={isProcessing}
+                modelsStatus={modelsStatus}
+                selectedModel={selectedModel}
+                imageSelected={!!selectedImage}
+              />
+            </Grid>
+            {error && (
+              <Grid item xs={12}>
+                <Paper style={{ padding: theme.spacing(2), backgroundColor: '#ffebee' }}>
+                  <Typography color="error">{error}</Typography>
+                </Paper>
+              </Grid>
+            )}
+            {isProcessing && (
+              <Grid item xs={12} style={{ textAlign: 'center', margin: `${theme.spacing(4)}px 0` }}>
+                <CircularProgress />
+                <Typography variant="h6" style={{ marginTop: theme.spacing(2) }}>
+                  Processing image...
+                </Typography>
+              </Grid>
+            )}
+            {results && (
+              <>
+                <Grid item xs={12}>
+                  <ResultDisplay results={results} />
+                </Grid>
+                <Grid item xs={12}>
+                  <LlmAnalysis visionResults={results.data} model={results.model} />
+                </Grid>
+              </>
+            )}
+            {/* OpenAI Chat section at the end */}
+            <Grid item xs={12}>
+              <OpenAIChat />
+            </Grid>
+          </Grid>
+        </Container>
+      </Box>
+    </ThemeProvider>
+  );
+}
+export default App;

frontend/src/components/ImageUploader.js ADDED Viewed

	@@ -0,0 +1,193 @@

+import React, { useState, useRef } from 'react';
+import {
+  Paper,
+  Typography,
+  Box,
+  Button,
+  IconButton
+} from '@material-ui/core';
+import CloudUploadIcon from '@material-ui/icons/CloudUpload';
+import DeleteIcon from '@material-ui/icons/Delete';
+import { makeStyles } from '@material-ui/core/styles';
+const useStyles = makeStyles((theme) => ({
+  paper: {
+    padding: theme.spacing(2),
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    height: '100%',
+    minHeight: 300,
+    transition: 'all 0.3s ease'
+  },
+  dragActive: {
+    border: '2px dashed #3f51b5',
+    backgroundColor: 'rgba(63, 81, 181, 0.05)'
+  },
+  dragInactive: {
+    border: '2px dashed #ccc',
+    backgroundColor: 'white'
+  },
+  uploadBox: {
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    justifyContent: 'center',
+    height: '100%',
+    width: '100%',
+    cursor: 'pointer'
+  },
+  uploadIcon: {
+    fontSize: 60,
+    color: '#3f51b5',
+    marginBottom: theme.spacing(2)
+  },
+  supportText: {
+    marginTop: theme.spacing(2)
+  },
+  previewBox: {
+    display: 'flex',
+    flexDirection: 'column',
+    alignItems: 'center',
+    width: '100%',
+    height: '100%',
+    position: 'relative'
+  },
+  imageContainer: {
+    position: 'relative',
+    width: '100%',
+    height: '100%',
+    display: 'flex',
+    justifyContent: 'center',
+    alignItems: 'center',
+    overflow: 'hidden',
+    marginTop: theme.spacing(2)
+  },
+  deleteButton: {
+    position: 'absolute',
+    top: 0,
+    right: 0,
+    backgroundColor: 'rgba(255, 255, 255, 0.7)',
+    '&:hover': {
+      backgroundColor: 'rgba(255, 255, 255, 0.9)',
+    }
+  }
+}));
+const ImageUploader = ({ onImageUpload }) => {
+  const classes = useStyles();
+  const [previewUrl, setPreviewUrl] = useState(null);
+  const [dragActive, setDragActive] = useState(false);
+  const fileInputRef = useRef(null);
+  const handleDrag = (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+    if (e.type === 'dragenter' || e.type === 'dragover') {
+      setDragActive(true);
+    } else if (e.type === 'dragleave') {
+      setDragActive(false);
+    }
+  };
+  const handleDrop = (e) => {
+    e.preventDefault();
+    e.stopPropagation();
+    setDragActive(false);
+    if (e.dataTransfer.files && e.dataTransfer.files[0]) {
+      handleFiles(e.dataTransfer.files[0]);
+    }
+  };
+  const handleChange = (e) => {
+    e.preventDefault();
+    if (e.target.files && e.target.files[0]) {
+      handleFiles(e.target.files[0]);
+    }
+  };
+  const handleFiles = (file) => {
+    if (file.type.startsWith('image/')) {
+      setPreviewUrl(URL.createObjectURL(file));
+      onImageUpload(file);
+    } else {
+      alert('Please upload an image file');
+    }
+  };
+  const onButtonClick = () => {
+    fileInputRef.current.click();
+  };
+  const handleRemoveImage = () => {
+    setPreviewUrl(null);
+    onImageUpload(null);
+    fileInputRef.current.value = "";
+  };
+  return (
+    <Paper
+      className={`${classes.paper} ${dragActive ? classes.dragActive : classes.dragInactive}`}
+      onDragEnter={handleDrag}
+      onDragLeave={handleDrag}
+      onDragOver={handleDrag}
+      onDrop={handleDrop}
+    >
+      <input
+        ref={fileInputRef}
+        type="file"
+        accept="image/*"
+        onChange={handleChange}
+        style={{ display: 'none' }}
+      />
+      {!previewUrl ? (
+        <Box
+          className={classes.uploadBox}
+          onClick={onButtonClick}
+        >
+          <CloudUploadIcon className={classes.uploadIcon} />
+          <Typography variant="h6" gutterBottom>
+            Drag & Drop an image here
+          </Typography>
+          <Typography variant="body2" color="textSecondary" gutterBottom>
+            or
+          </Typography>
+          <Button
+            variant="contained"
+            color="primary"
+            component="span"
+            startIcon={<CloudUploadIcon />}
+          >
+            Browse Files
+          </Button>
+          <Typography variant="body2" color="textSecondary" className={classes.supportText}>
+            Supported formats: JPG, PNG, GIF
+          </Typography>
+        </Box>
+      ) : (
+        <Box className={classes.previewBox}>
+          <Typography variant="h6" gutterBottom>
+            Preview
+          </Typography>
+          <Box className={classes.imageContainer}>
+            <img
+              src={previewUrl}
+              alt="Preview"
+              className="preview-image"
+            />
+            <IconButton
+              aria-label="delete"
+              className={classes.deleteButton}
+              onClick={handleRemoveImage}
+            >
+              <DeleteIcon />
+            </IconButton>
+          </Box>
+        </Box>
+      )}
+    </Paper>
+  );
+};
+export default ImageUploader;

frontend/src/components/LlmAnalysis.js ADDED Viewed

	@@ -0,0 +1,156 @@

+import React, { useState } from 'react';
+import {
+  Paper,
+  Typography,
+  Box,
+  TextField,
+  Button,
+  CircularProgress,
+  Divider
+} from '@material-ui/core';
+import { makeStyles } from '@material-ui/core/styles';
+const useStyles = makeStyles((theme) => ({
+  paper: {
+    padding: theme.spacing(2),
+    marginTop: theme.spacing(2)
+  },
+  marginBottom: {
+    marginBottom: theme.spacing(2)
+  },
+  dividerMargin: {
+    margin: `${theme.spacing(2)}px 0`
+  },
+  responseBox: {
+    padding: theme.spacing(2),
+    backgroundColor: '#f5f5f5',
+    borderRadius: theme.shape.borderRadius,
+    marginTop: theme.spacing(2),
+    whiteSpace: 'pre-wrap'
+  },
+  buttonProgress: {
+    marginLeft: theme.spacing(1)
+  }
+}));
+const LlmAnalysis = ({ visionResults, model }) => {
+  const classes = useStyles();
+  const [userQuery, setUserQuery] = useState('');
+  const [isAnalyzing, setIsAnalyzing] = useState(false);
+  const [analysisResult, setAnalysisResult] = useState(null);
+  const [error, setError] = useState(null);
+  // Format time for display
+  const formatTime = (ms) => {
+    if (ms === undefined || ms === null || isNaN(ms)) return '-';
+    const num = Number(ms);
+    if (num < 1000) return `${num.toFixed(2)} ms`;
+    return `${(num / 1000).toFixed(2)} s`;
+  };
+  const handleAnalyze = async () => {
+    if (!userQuery.trim()) return;
+    setIsAnalyzing(true);
+    setError(null);
+    try {
+      const response = await fetch('/api/analyze', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify({
+          visionResults: visionResults,
+          userQuery: userQuery
+        }),
+      });
+      if (!response.ok) {
+        throw new Error(`HTTP error! Status: ${response.status}`);
+      }
+      const data = await response.json();
+      if (data.error) {
+        setError(data.error);
+      } else {
+        setAnalysisResult(data);
+      }
+    } catch (err) {
+      console.error('Error analyzing with LLM:', err);
+      setError(`Error analyzing with LLM: ${err.message}`);
+    } finally {
+      setIsAnalyzing(false);
+    }
+  };
+  if (!visionResults) return null;
+  return (
+    <Paper className={classes.paper}>
+      <Typography variant="h6" gutterBottom>
+        Ask AI about the {model === 'vit' ? 'Classification' : 'Detection'} Results
+      </Typography>
+      <Typography variant="body2" className={classes.marginBottom}>
+        Ask a question about the detected objects or classifications to get an AI-powered analysis.
+      </Typography>
+      <TextField
+        fullWidth
+        label="Your question about the image"
+        variant="outlined"
+        value={userQuery}
+        onChange={(e) => setUserQuery(e.target.value)}
+        disabled={isAnalyzing}
+        className={classes.marginBottom}
+        placeholder={model === 'vit'
+          ? "E.g., What category does this image belong to?"
+          : "E.g., How many people are in this image?"}
+      />
+      <Button
+        variant="contained"
+        color="primary"
+        onClick={handleAnalyze}
+        disabled={isAnalyzing || !userQuery.trim()}
+      >
+        Analyze with AI
+        {isAnalyzing && <CircularProgress size={24} className={classes.buttonProgress} />}
+      </Button>
+      {error && (
+        <Box mt={2}>
+          <Typography color="error">{error}</Typography>
+        </Box>
+      )}
+      {analysisResult && (
+        <>
+          <Divider className={classes.dividerMargin} />
+          <Typography variant="subtitle1" gutterBottom>
+            AI Analysis:
+          </Typography>
+          <Box className={classes.responseBox}>
+            <Typography variant="body1">
+              {analysisResult.response}
+            </Typography>
+          </Box>
+          {analysisResult.performance && (
+            <Box mt={1}>
+              <Typography variant="body2" color="textSecondary">
+                Analysis time: {formatTime(analysisResult.performance.inference_time)} on {analysisResult.performance.device}
+              </Typography>
+            </Box>
+          )}
+        </>
+      )}
+    </Paper>
+  );
+};
+export default LlmAnalysis;

frontend/src/components/ModelSelector.js ADDED Viewed

	@@ -0,0 +1,168 @@

+import React from 'react';
+import {
+  Grid,
+  Card,
+  CardContent,
+  CardActions,
+  Typography,
+  Button,
+  Chip,
+  Box
+} from '@material-ui/core';
+import VisibilityIcon from '@material-ui/icons/Visibility';
+import CategoryIcon from '@material-ui/icons/Category';
+import PlayArrowIcon from '@material-ui/icons/PlayArrow';
+import { makeStyles } from '@material-ui/core/styles';
+const useStyles = makeStyles((theme) => ({
+  card: {
+    height: '100%',
+    display: 'flex',
+    flexDirection: 'column',
+  },
+  selectedCard: {
+    border: '2px solid #3f51b5',
+  },
+  unavailableCard: {
+    opacity: 0.6,
+  },
+  cardContent: {
+    flexGrow: 1,
+  },
+  chipContainer: {
+    marginBottom: theme.spacing(1.5),
+  },
+  successChip: {
+    backgroundColor: '#34C759',
+    color: '#fff',
+  },
+  errorChip: {
+    backgroundColor: '#FF3B3F',
+    color: '#fff',
+  },
+  modelType: {
+    marginTop: theme.spacing(1),
+  },
+  processButton: {
+    marginTop: theme.spacing(3),
+    textAlign: 'center',
+  }
+}));
+const ModelSelector = ({
+  onModelSelect,
+  onProcess,
+  isProcessing,
+  modelsStatus,
+  selectedModel,
+  imageSelected
+}) => {
+  const classes = useStyles();
+  const models = [
+    {
+      id: 'yolo',
+      name: 'YOLOv8',
+      description: 'Fast and accurate object detection',
+      icon: <VisibilityIcon />,
+      available: modelsStatus.yolo
+    },
+    {
+      id: 'detr',
+      name: 'DETR',
+      description: 'DEtection TRansformer for object detection',
+      icon: <VisibilityIcon />,
+      available: modelsStatus.detr
+    },
+    {
+      id: 'vit',
+      name: 'ViT',
+      description: 'Vision Transformer for image classification',
+      icon: <CategoryIcon />,
+      available: modelsStatus.vit
+    }
+  ];
+  const handleModelClick = (modelId) => {
+    if (models.find(m => m.id === modelId).available) {
+      onModelSelect(modelId);
+    }
+  };
+  return (
+    <Box sx={{ p: 2, height: '100%' }}>
+      <Typography variant="h6" gutterBottom>
+        Select Model
+      </Typography>
+      <Grid container spacing={2}>
+        {models.map((model) => (
+          <Grid item xs={12} sm={4} key={model.id}>
+            <Card
+              className={`
+                ${classes.card}
+                ${selectedModel === model.id ? classes.selectedCard : ''}
+                ${!model.available ? classes.unavailableCard : ''}
+              `}
+              onClick={() => handleModelClick(model.id)}
+            >
+              <CardContent className={classes.cardContent}>
+                <Box sx={{ mb: 2, color: 'primary' }}>
+                  {model.icon}
+                </Box>
+                <Typography variant="h5" component="div" gutterBottom>
+                  {model.name}
+                </Typography>
+                <div className={classes.chipContainer}>
+                  {model.available ? (
+                    <Chip
+                      label="Available"
+                      className={classes.successChip}
+                      size="small"
+                    />
+                  ) : (
+                    <Chip
+                      label="Not Available"
+                      className={classes.errorChip}
+                      size="small"
+                    />
+                  )}
+                </div>
+                <Typography variant="body2" color="textSecondary">
+                  {model.description}
+                </Typography>
+              </CardContent>
+              <CardActions>
+                <Button
+                  size="small"
+                  onClick={() => handleModelClick(model.id)}
+                  disabled={!model.available}
+                  color={selectedModel === model.id ? "primary" : "default"}
+                  variant={selectedModel === model.id ? "contained" : "outlined"}
+                  fullWidth
+                >
+                  {selectedModel === model.id ? 'Selected' : 'Select'}
+                </Button>
+              </CardActions>
+            </Card>
+          </Grid>
+        ))}
+      </Grid>
+      <div className={classes.processButton}>
+        <Button
+          variant="contained"
+          color="primary"
+          size="large"
+          startIcon={<PlayArrowIcon />}
+          onClick={onProcess}
+          disabled={!selectedModel || !imageSelected || isProcessing}
+        >
+          {isProcessing ? 'Processing...' : 'Process Image'}
+        </Button>
+      </div>
+    </Box>
+  );
+};
+export default ModelSelector;

frontend/src/components/OpenAIChat.js ADDED Viewed

	@@ -0,0 +1,136 @@

+import React, { useState } from 'react';
+import {
+  Paper,
+  Typography,
+  Grid,
+  TextField,
+  Button,
+  Divider
+} from '@material-ui/core';
+function OpenAIChat() {
+  const [model, setModel] = useState('gpt-4o-mini');
+  const [apiKey, setApiKey] = useState('');
+  const [system, setSystem] = useState('You are a helpful assistant.');
+  const [prompt, setPrompt] = useState('');
+  const [response, setResponse] = useState('');
+  const [loading, setLoading] = useState(false);
+  const [error, setError] = useState('');
+  const onSend = async () => {
+    setError('');
+    setResponse('');
+    const p = (prompt || '').trim();
+    if (!p) { setError('Please enter a question.'); return; }
+    setLoading(true);
+    try {
+      const res = await fetch('/api/openai/chat', {
+        method: 'POST',
+        headers: { 'Content-Type': 'application/json' },
+        credentials: 'include',
+        body: JSON.stringify({
+          prompt: p,
+          model: (model || '').trim() || 'gpt-4o-mini',
+          api_key: (apiKey || undefined),
+          system: (system || undefined)
+        })
+      });
+      if (!res.ok) {
+        let txt = await res.text();
+        try { txt = JSON.stringify(JSON.parse(txt), null, 2); } catch {}
+        throw new Error(txt);
+      }
+      const data = await res.json();
+      const meta = `Model: ${data.model} | Latency: ${data.latency_sec}s` + (data.usage ? ` | Usage: ${JSON.stringify(data.usage)}` : '');
+      setResponse((data.response || '(Empty response)') + '\n\n---\n' + meta);
+    } catch (e) {
+      setError('Error: ' + e.message);
+    } finally {
+      setLoading(false);
+    }
+  };
+  const onClear = () => {
+    setPrompt('');
+    setResponse('');
+    setError('');
+  };
+  return (
+    <Paper style={{ padding: 16 }}>
+      <Typography variant="h5" gutterBottom>
+        OpenAI Chat (OpenAI API)
+      </Typography>
+      <Typography variant="body2" color="textSecondary" gutterBottom>
+        If the server env var OPENAI_API_KEY is set, the API Key field is optional.
+      </Typography>
+      <Grid container spacing={2}>
+        <Grid item xs={12} md={6}>
+          <TextField
+            label="Model"
+            value={model}
+            onChange={(e) => setModel(e.target.value)}
+            fullWidth
+            variant="outlined"
+            size="small"
+          />
+        </Grid>
+        <Grid item xs={12} md={6}>
+          <TextField
+            label="OpenAI API Key (optional)"
+            value={apiKey}
+            onChange={(e) => setApiKey(e.target.value)}
+            fullWidth
+            variant="outlined"
+            size="small"
+            type="password"
+            placeholder="sk-..."
+          />
+        </Grid>
+        <Grid item xs={12}>
+          <TextField
+            label="System Prompt (optional)"
+            value={system}
+            onChange={(e) => setSystem(e.target.value)}
+            fullWidth
+            variant="outlined"
+            size="small"
+          />
+        </Grid>
+        <Grid item xs={12}>
+          <TextField
+            label="User Question"
+            value={prompt}
+            onChange={(e) => setPrompt(e.target.value)}
+            fullWidth
+            multiline
+            rows={4}
+            variant="outlined"
+          />
+        </Grid>
+        {error && (
+          <Grid item xs={12}>
+            <Typography color="error">{error}</Typography>
+          </Grid>
+        )}
+        <Grid item xs={12}>
+          <div style={{ display: 'flex', gap: 8 }}>
+            <Button color="primary" variant="contained" onClick={onSend} disabled={loading}>
+              {loading ? 'Sending...' : 'Send Question'}
+            </Button>
+            <Button variant="outlined" onClick={onClear}>Clear</Button>
+          </div>
+        </Grid>
+        <Grid item xs={12}>
+          <Divider style={{ margin: '12px 0' }} />
+          <Typography variant="subtitle2" color="textSecondary">Response</Typography>
+          <pre style={{ whiteSpace: 'pre-wrap', fontFamily: 'ui-monospace, monospace' }}>{response}</pre>
+        </Grid>
+      </Grid>
+    </Paper>
+  );
+}
+export default OpenAIChat;

frontend/src/components/ResultDisplay.js ADDED Viewed

	@@ -0,0 +1,198 @@

+import React from 'react';
+import {
+  Paper,
+  Typography,
+  Box,
+  List,
+  ListItem,
+  ListItemText,
+  Divider,
+  Grid,
+  Chip
+} from '@material-ui/core';
+import VectorDBActions from './VectorDBActions';
+import { makeStyles } from '@material-ui/core/styles';
+const useStyles = makeStyles((theme) => ({
+  paper: {
+    padding: theme.spacing(2)
+  },
+  marginBottom: {
+    marginBottom: theme.spacing(2)
+  },
+  resultImage: {
+    maxWidth: '100%',
+    maxHeight: '400px',
+    objectFit: 'contain'
+  },
+  dividerMargin: {
+    margin: `${theme.spacing(2)}px 0`
+  },
+  chipContainer: {
+    display: 'flex',
+    gap: theme.spacing(1),
+    flexWrap: 'wrap'
+  }
+}));
+const ResultDisplay = ({ results }) => {
+  const classes = useStyles();
+  if (!results) return null;
+  const { model, data } = results;
+  // Helper to format times nicely
+  const formatTime = (ms) => {
+    if (ms === undefined || ms === null || isNaN(ms)) return '-';
+    const num = Number(ms);
+    if (num < 1000) return `${num.toFixed(2)} ms`;
+    return `${(num / 1000).toFixed(2)} s`;
+  };
+  // Check if there's an error
+  if (data.error) {
+    return (
+      <Paper sx={{ p: 2, bgcolor: '#ffebee' }}>
+        <Typography color="error">{data.error}</Typography>
+      </Paper>
+    );
+  }
+  // Display performance info
+  const renderPerformanceInfo = () => {
+    if (!data.performance) return null;
+    return (
+      <Box className="performance-info">
+        <Divider className={classes.dividerMargin} />
+        <Typography variant="body2">
+          Inference time: {formatTime(data.performance.inference_time)} on {data.performance.device}
+        </Typography>
+      </Box>
+    );
+  };
+  // Render for YOLO and DETR (object detection)
+  if (model === 'yolo' || model === 'detr') {
+    return (
+      <Paper className={classes.paper}>
+        <Typography variant="h6" gutterBottom>
+          {model === 'yolo' ? 'YOLOv8' : 'DETR'} Detection Results
+        </Typography>
+        <Grid container spacing={3}>
+          <Grid item xs={12} md={6}>
+            {data.image && (
+              <Box className={classes.marginBottom}>
+                <Typography variant="subtitle1" gutterBottom>
+                  Detection Result
+                </Typography>
+                <img
+                  src={`data:image/png;base64,${data.image}`}
+                  alt="Detection Result"
+                  className={classes.resultImage}
+                />
+              </Box>
+            )}
+          </Grid>
+          <Grid item xs={12} md={6}>
+            <Box className={classes.marginBottom}>
+              <Typography variant="subtitle1" gutterBottom>
+                Detected Objects:
+              </Typography>
+              {data.detections && data.detections.length > 0 ? (
+                <List>
+                  {data.detections.map((detection, index) => (
+                    <React.Fragment key={index}>
+                      <ListItem>
+                        <ListItemText
+                          primary={
+                            <Box style={{ display: 'flex', alignItems: 'center' }}>
+                              <Typography variant="body1" component="span">
+                                {detection.class}
+                              </Typography>
+                              <Chip
+                                label={`${(detection.confidence * 100).toFixed(0)}%`}
+                                size="small"
+                                color="primary"
+                                style={{ marginLeft: 8 }}
+                              />
+                            </Box>
+                          }
+                          secondary={`Bounding Box: [${detection.bbox.join(', ')}]`}
+                        />
+                      </ListItem>
+                      {index < data.detections.length - 1 && <Divider />}
+                    </React.Fragment>
+                  ))}
+                </List>
+              ) : (
+                <Typography variant="body1">No objects detected</Typography>
+              )}
+            </Box>
+          </Grid>
+        </Grid>
+        {renderPerformanceInfo()}
+        {/* Vector DB Actions for Object Detection */}
+        <VectorDBActions results={results} />
+      </Paper>
+    );
+  }
+  // Render for ViT (classification)
+  if (model === 'vit') {
+    return (
+      <Paper className={classes.paper}>
+        <Typography variant="h6" gutterBottom>
+          ViT Classification Results
+        </Typography>
+        <Typography variant="subtitle1" gutterBottom>
+          Top Predictions:
+        </Typography>
+        {data.top_predictions && data.top_predictions.length > 0 ? (
+          <List>
+            {data.top_predictions.map((prediction, index) => (
+              <React.Fragment key={index}>
+                <ListItem>
+                  <ListItemText
+                    primary={
+                      <Box style={{ display: 'flex', alignItems: 'center' }}>
+                        <Typography variant="body1" component="span">
+                          {prediction.rank}. {prediction.class}
+                        </Typography>
+                        <Chip
+                          label={`${(prediction.probability * 100).toFixed(1)}%`}
+                          size="small"
+                          color={index === 0 ? "primary" : "default"}
+                          style={{ marginLeft: 8 }}
+                        />
+                      </Box>
+                    }
+                  />
+                </ListItem>
+                {index < data.top_predictions.length - 1 && <Divider />}
+              </React.Fragment>
+            ))}
+          </List>
+        ) : (
+          <Typography variant="body1">No classifications available</Typography>
+        )}
+        {renderPerformanceInfo()}
+        {/* Vector DB Actions for ViT Classification */}
+        <VectorDBActions results={results} />
+      </Paper>
+    );
+  }
+  return null;
+};
+export default ResultDisplay;

frontend/src/components/VectorDBActions.js ADDED Viewed

	@@ -0,0 +1,434 @@

+import React, { useState } from 'react';
+import {
+  Button,
+  Box,
+  Typography,
+  CircularProgress,
+  Snackbar,
+  Dialog,
+  DialogTitle,
+  DialogContent,
+  DialogActions,
+  TextField,
+  FormControl,
+  InputLabel,
+  Select,
+  MenuItem,
+  Grid,
+  Card,
+  CardMedia,
+  CardContent,
+  Chip
+} from '@material-ui/core';
+import { Alert } from '@material-ui/lab';
+import { makeStyles } from '@material-ui/core/styles';
+const useStyles = makeStyles((theme) => ({
+  root: {
+    marginTop: theme.spacing(2),
+    marginBottom: theme.spacing(2),
+    padding: theme.spacing(2),
+    backgroundColor: '#f5f5f5',
+    borderRadius: theme.shape.borderRadius,
+  },
+  button: {
+    marginRight: theme.spacing(2),
+  },
+  searchDialog: {
+    minWidth: '500px',
+  },
+  formControl: {
+    marginBottom: theme.spacing(2),
+    minWidth: '100%',
+  },
+  searchResults: {
+    marginTop: theme.spacing(2),
+  },
+  resultCard: {
+    marginBottom: theme.spacing(2),
+  },
+  resultImage: {
+    height: 140,
+    objectFit: 'contain',
+  },
+  chip: {
+    margin: theme.spacing(0.5),
+  },
+  similarityChip: {
+    backgroundColor: theme.palette.primary.main,
+    color: 'white',
+  }
+}));
+const VectorDBActions = ({ results }) => {
+  const classes = useStyles();
+  const [isSaving, setIsSaving] = useState(false);
+  const [saveSuccess, setSaveSuccess] = useState(false);
+  const [saveError, setSaveError] = useState(null);
+  const [openSearchDialog, setOpenSearchDialog] = useState(false);
+  const [searchType, setSearchType] = useState('image');
+  const [searchClass, setSearchClass] = useState('');
+  const [searchResults, setSearchResults] = useState([]);
+  const [isSearching, setIsSearching] = useState(false);
+  const [searchError, setSearchError] = useState(null);
+  // Extract model and data from results
+  const { model, data } = results;
+  // Handle saving to vector DB
+  const handleSaveToVectorDB = async () => {
+    setIsSaving(true);
+    setSaveError(null);
+    try {
+      let response;
+      if (model === 'vit') {
+        // For ViT, save the whole image with classifications
+        response = await fetch('/api/add-to-collection', {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({
+            image: data.image,
+            metadata: {
+              model: 'vit',
+              classifications: data.classifications
+            }
+          })
+        });
+      } else {
+        // For YOLO and DETR, save detected objects
+        response = await fetch('/api/add-detected-objects', {
+          method: 'POST',
+          headers: {
+            'Content-Type': 'application/json',
+          },
+          body: JSON.stringify({
+            image: data.image,
+            objects: data.detections,
+            imageId: generateUUID()
+          })
+        });
+      }
+      if (!response.ok) {
+        throw new Error(`HTTP error! Status: ${response.status}`);
+      }
+      const result = await response.json();
+      if (result.error) {
+        throw new Error(result.error);
+      }
+      setSaveSuccess(true);
+      setTimeout(() => setSaveSuccess(false), 5000);
+    } catch (err) {
+      console.error('Error saving to vector DB:', err);
+      setSaveError(`Error saving to vector DB: ${err.message}`);
+    } finally {
+      setIsSaving(false);
+    }
+  };
+  // Handle opening search dialog
+  const handleOpenSearchDialog = () => {
+    setOpenSearchDialog(true);
+    setSearchResults([]);
+    setSearchError(null);
+  };
+  // Handle closing search dialog
+  const handleCloseSearchDialog = () => {
+    setOpenSearchDialog(false);
+  };
+  // Handle search type change
+  const handleSearchTypeChange = (event) => {
+    setSearchType(event.target.value);
+    setSearchResults([]);
+    setSearchError(null);
+  };
+  // Handle search class change
+  const handleSearchClassChange = (event) => {
+    setSearchClass(event.target.value);
+  };
+  // Handle search
+  const handleSearch = async () => {
+    setIsSearching(true);
+    setSearchError(null);
+    try {
+      let requestBody = {};
+      if (searchType === 'image') {
+        // Search by current image
+        requestBody = {
+          searchType: 'image',
+          image: data.image,
+          n_results: 5
+        };
+      } else {
+        // Search by class name
+        if (!searchClass.trim()) {
+          throw new Error('Please enter a class name');
+        }
+        requestBody = {
+          searchType: 'class',
+          class_name: searchClass.trim(),
+          n_results: 5
+        };
+      }
+      const response = await fetch('/api/search-similar-objects', {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+        },
+        body: JSON.stringify(requestBody)
+      });
+      if (!response.ok) {
+        throw new Error(`HTTP error! Status: ${response.status}`);
+      }
+      const result = await response.json();
+      if (result.error) {
+        throw new Error(result.error);
+      }
+      console.log('Search API response:', result);
+      // The backend responds with {success, searchType, results} structure, so extract only the results array
+      if (result.success && Array.isArray(result.results)) {
+        console.log('Setting search results array:', result.results);
+        console.log('Results array length:', result.results.length);
+        console.log('First result item:', result.results[0]);
+        setSearchResults(result.results);
+      } else {
+        console.error('Unexpected API response format:', result);
+        throw new Error('Unexpected API response format');
+      }
+    } catch (err) {
+      console.error('Error searching vector DB:', err);
+      setSearchError(`Error searching vector DB: ${err.message}`);
+    } finally {
+      setIsSearching(false);
+    }
+  };
+  // Generate UUID for image ID
+  const generateUUID = () => {
+    return 'xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx'.replace(/[xy]/g, function(c) {
+      const r = Math.random() * 16 | 0;
+      const v = c === 'x' ? r : (r & 0x3 | 0x8);
+      return v.toString(16);
+    });
+  };
+  // Render search results
+  const renderSearchResults = () => {
+    console.log('Rendering search results:', searchResults);
+    console.log('Search results length:', searchResults.length);
+    if (searchResults.length === 0) {
+      console.log('No results to render');
+      return (
+        <Typography variant="body1">No results found.</Typography>
+      );
+    }
+    return (
+      <Grid container spacing={2}>
+        {searchResults.map((result, index) => {
+          const similarity = (1 - result.distance) * 100;
+          return (
+            <Grid item xs={12} sm={6} key={index}>
+              <Card className={classes.resultCard}>
+                {result.metadata && result.metadata.image_data ? (
+                  <CardMedia
+                    className={classes.resultImage}
+                    component="img"
+                    height="200"
+                    image={`data:image/jpeg;base64,${result.metadata.image_data}`}
+                    alt={result.metadata && result.metadata.class ? result.metadata.class : 'Object'}
+                  />
+                ) : (
+                  <Box
+                    className={classes.resultImage}
+                    style={{
+                      backgroundColor: '#f0f0f0',
+                      display: 'flex',
+                      alignItems: 'center',
+                      justifyContent: 'center',
+                      height: 200
+                    }}
+                  >
+                    <Typography variant="body2" color="textSecondary">
+                      {result.metadata && result.metadata.class ? result.metadata.class : 'Object'} Image
+                    </Typography>
+                  </Box>
+                )}
+                <CardContent>
+                  <Box display="flex" justifyContent="space-between" alignItems="center" mb={1}>
+                    <Typography variant="subtitle1">Result #{index + 1}</Typography>
+                    <Chip
+                      label={`Similarity: ${similarity.toFixed(2)}%`}
+                      className={classes.similarityChip}
+                      size="small"
+                    />
+                  </Box>
+                  <Typography variant="body2" color="textSecondary">
+                    <strong>Class:</strong> {result.metadata.class || 'N/A'}
+                  </Typography>
+                  {result.metadata.confidence && (
+                    <Typography variant="body2" color="textSecondary">
+                      <strong>Confidence:</strong> {(result.metadata.confidence * 100).toFixed(2)}%
+                    </Typography>
+                  )}
+                  <Typography variant="body2" color="textSecondary">
+                    <strong>Object ID:</strong> {result.id}
+                  </Typography>
+                </CardContent>
+              </Card>
+            </Grid>
+          );
+        })}
+      </Grid>
+    );
+  };
+  return (
+    <Box className={classes.root}>
+      <Typography variant="h6" gutterBottom>
+        Vector Database Actions
+      </Typography>
+      <Box display="flex" alignItems="center" mb={2}>
+        <Button
+          variant="contained"
+          color="primary"
+          onClick={handleSaveToVectorDB}
+          disabled={isSaving}
+          className={classes.button}
+        >
+          {isSaving ? (
+            <>
+              <CircularProgress size={20} color="inherit" style={{ marginRight: 8 }} />
+              Saving...
+            </>
+          ) : (
+            'Save to Vector DB'
+          )}
+        </Button>
+        <Button
+          variant="outlined"
+          color="primary"
+          onClick={handleOpenSearchDialog}
+          className={classes.button}
+        >
+          Search Similar
+        </Button>
+      </Box>
+      {saveError && (
+        <Alert severity="error" style={{ marginTop: 8 }}>
+          {saveError}
+        </Alert>
+      )}
+      <Snackbar open={saveSuccess} autoHideDuration={5000} onClose={() => setSaveSuccess(false)}>
+        <Alert severity="success">
+          {model === 'vit' ? (
+            'Image and classifications successfully saved to vector DB!'
+          ) : (
+            'Detected objects successfully saved to vector DB!'
+          )}
+        </Alert>
+      </Snackbar>
+      {/* Search Dialog */}
+      <Dialog
+        open={openSearchDialog}
+        onClose={handleCloseSearchDialog}
+        maxWidth="md"
+        fullWidth
+      >
+        <DialogTitle>Search Vector Database</DialogTitle>
+        <DialogContent>
+          <FormControl className={classes.formControl}>
+            <InputLabel id="search-type-label">Search Type</InputLabel>
+            <Select
+              labelId="search-type-label"
+              id="search-type"
+              value={searchType}
+              onChange={handleSearchTypeChange}
+            >
+              <MenuItem value="image">Search by Current Image</MenuItem>
+              <MenuItem value="class">Search by Class Name</MenuItem>
+            </Select>
+          </FormControl>
+          {searchType === 'class' && (
+            <FormControl className={classes.formControl}>
+              <TextField
+                label="Class Name"
+                value={searchClass}
+                onChange={handleSearchClassChange}
+                placeholder="e.g. person, car, dog..."
+                fullWidth
+              />
+            </FormControl>
+          )}
+          {searchError && (
+            <Alert severity="error" style={{ marginBottom: 16 }}>
+              {searchError}
+            </Alert>
+          )}
+          <Box className={classes.searchResults}>
+            {isSearching ? (
+              <Box display="flex" justifyContent="center" alignItems="center" p={4}>
+                <CircularProgress />
+                <Typography variant="body1" style={{ marginLeft: 16 }}>
+                  Searching...
+                </Typography>
+              </Box>
+            ) : (
+              <>
+                {console.log('Search dialog render - searchResults:', searchResults)}
+                {searchResults.length > 0 ? renderSearchResults() :
+                  <Typography variant="body1">No results found. Please try another search.</Typography>
+                }
+              </>
+            )}
+          </Box>
+        </DialogContent>
+        <DialogActions>
+          <Button onClick={handleCloseSearchDialog} color="default">
+            Close
+          </Button>
+          <Button
+            onClick={handleSearch}
+            color="primary"
+            variant="contained"
+            disabled={isSearching || (searchType === 'class' && !searchClass.trim())}
+          >
+            Search
+          </Button>
+        </DialogActions>
+      </Dialog>
+    </Box>
+  );
+};
+export default VectorDBActions;

frontend/src/index.css ADDED Viewed

	@@ -0,0 +1,14 @@

+body {
+  margin: 0;
+  font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', 'Roboto', 'Oxygen',
+    'Ubuntu', 'Cantarell', 'Fira Sans', 'Droid Sans', 'Helvetica Neue',
+    sans-serif;
+  -webkit-font-smoothing: antialiased;
+  -moz-osx-font-smoothing: grayscale;
+  background-color: #f5f5f5;
+}
+code {
+  font-family: source-code-pro, Menlo, Monaco, Consolas, 'Courier New',
+    monospace;
+}

frontend/src/index.js ADDED Viewed

	@@ -0,0 +1,17 @@

+import React from 'react';
+import ReactDOM from 'react-dom';
+import './index.css';
+import App from './App';
+import reportWebVitals from './reportWebVitals';
+ReactDOM.render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>,
+  document.getElementById('root')
+);
+// If you want to start measuring performance in your app, pass a function
+// to log results (for example: reportWebVitals(console.log))
+// or send to an analytics endpoint. Learn more: https://bit.ly/CRA-vitals
+reportWebVitals();

frontend/src/reportWebVitals.js ADDED Viewed

	@@ -0,0 +1,13 @@

+const reportWebVitals = (onPerfEntry) => {
+  if (onPerfEntry && onPerfEntry instanceof Function) {
+    import('web-vitals').then(({ getCLS, getFID, getFCP, getLCP, getTTFB }) => {
+      getCLS(onPerfEntry);
+      getFID(onPerfEntry);
+      getFCP(onPerfEntry);
+      getLCP(onPerfEntry);
+      getTTFB(onPerfEntry);
+    });
+  }
+};
+export default reportWebVitals;