Repository Initialisation.

.devcontainer/devcontainer.json

@@ -0,0 +1,28 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+{
+	"name": "video_analizer",
+	"build": {
+		"dockerfile": "../Dockerfile"
+	},
+	"customizations": {
+		"vscode": {
+			"extensions": [
+				"ms-python.python",
+				"ms-python.vscode-pylance",
+				"ms-toolsai.jupyter",
+				"ms-toolsai.vscode-jupyter-cell-tags",
+				"ms-toolsai.tensorboard",
+				"ms-toolsai.jupyter-keymap",
+				"ms-toolsai.jupyter-renderers",
+				"ms-toolsai.vscode-jupyter-slideshow"
+			]
+		}
+	},								// You can read about these additional settings by following the links:
+	"forwardPorts": ["7860:7860"], 	// https://code.visualstudio.com/docs/devcontainers/containers#_forwarding-or-publishing-a-port
+	"overrideCommand": false,		// https://code.visualstudio.com/remote/advancedcontainers/start-processes
+	"runArgs": [
+		"--shm-size=1gb",
+		"--gpus",
+		"all"
+	]
+}

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+images/thumbnail.png filter=lfs diff=lfs merge=lfs -text
+app_srv/images/thumbnail.png filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,72 @@
+FROM ubuntu:24.04
+
+# ENV PIP_ROOT_USER_ACTION=ignore
+ARG USERNAME=mcp_user
+ARG USER_UID=1000
+ARG APP_WORKDIR=app_srv
+
+# Update OS and install packages
+WORKDIR /tmp/
+COPY requirements.txt packages.txt ./
+RUN apt-get -y update && apt-get -y upgrade 
+# RUN xargs apt -y install < packages.txt
+RUN apt-get -y install ffmpeg git git-lfs htop iotop libxml2 libopenblas-dev libssl-dev python3-pip python3-wheel python3-setuptools python-is-python3 wget zlib1g net-tools curl
+
+# Install CUDA 12.8 from Internet
+RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \
+    && dpkg -i cuda-keyring_1.1-1_all.deb \
+    && apt-get update \
+    && apt-get -y install cuda-toolkit-12-8
+# https://docs.nvidia.com/cuda/cuda-installation-guide-linux/#post-installation-actions
+ENV PATH=/usr/local/cuda-12.8/bin:${PATH}									
+ENV LD_LIBRARY_PATH=/usr/local/cuda-12.8/lib64:/usr/lib/x86_64-linux-gnu/
+
+# Install cuDNN for CUDA 12
+RUN wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2404/x86_64/cuda-keyring_1.1-1_all.deb \
+	&& dpkg -i cuda-keyring_1.1-1_all.deb \
+	&& apt-get update \
+	&& apt-get -y install cudnn-cuda-12
+
+# Remove EXTERNALLY-MANAGED
+RUN rm -f /usr/lib/python3.12/EXTERNALLY-MANAGED
+
+# Install Pytorch for CUDA 12.8
+RUN pip3 install --no-cache-dir torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cu128
+
+# Install other python packages
+RUN pip3 install --no-cache-dir -r requirements.txt
+
+# Final image cleanup
+RUN apt-get clean
+RUN rm -rf /tmp/*
+
+# Preparing an image for HuggingFace Space
+## Delete exist user with UID 1000
+RUN userdel -r ubuntu || true
+
+## Set up a new user with user ID 1000
+## https://huggingface.co/docs/hub/spaces-sdks-docker#permissions
+RUN useradd -s /bin/bash -m -u $USER_UID $USERNAME
+
+## Switch to the "user" user
+USER $USERNAME
+
+## Set home to the user's home directory
+ENV HOME=/home/$USERNAME \
+	PATH=/home/$USERNAME/.local/bin:$PATH
+
+## Copy the contents of the application directory to the /home/mcp_user/app_srv container directory, setting mcp_user as the user owner
+RUN mkdir -p $HOME/$APP_WORKDIR
+COPY --chown=$USERNAME ./$APP_WORKDIR/* $HOME/$APP_WORKDIR
+
+# Set the working directory to the user's home directory
+WORKDIR $HOME/$APP_WORKDIR
+
+# Gradio network configuration 
+EXPOSE 7860
+ENV GRADIO_SERVER_NAME="0.0.0.0"
+
+# Temporarily set an environment variable for unbuffered output. The goal is to test the application.
+ENV PYTHONUNBUFFERED=1
+
+CMD ["python", "app_srv.py"]

README.md

@@ -1,11 +1,16 @@
 ---
 title: Aura AI Scan
-emoji: 🚀
-colorFrom: indigo
-colorTo: blue
+short_description: Flexible tool for in-depth analysis of YouTube videos.
+emoji: 📊
 sdk: docker
+colorFrom: blue
+colorTo: red
 pinned: false
-short_description: Flexible tool for in-depth analysis of YouTube videos.
+license: gpl-3.0
+tags:
+  - mcp-server-track
+thumbnail: >-
+  https://cdn-uploads.huggingface.co/production/uploads/630f0cfbe52a259b855d290e/MeV4xfCNESPfFCmtZM_rg.jpeg
 ---
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

app_client/app_client.py

@@ -0,0 +1,45 @@
+import gradio as gr
+import torch
+
+from mcp.client.stdio import StdioServerParameters
+from smolagents import ToolCollection, CodeAgent
+from smolagents import CodeAgent, ToolCallingAgent, InferenceClientModel, TransformersModel
+from smolagents.mcp_client import MCPClient
+
+# model = InferenceClientModel()
+
+hf_model = "Qwen/Qwen2.5-7B-Instruct"
+model = TransformersModel(model_id=hf_model, device_map="cuda", torch_dtype=torch.bfloat16)
+
+test_question = """What's this video about? https://www.youtube.com/watch?v=R3GfuzLMPkA"""
+
+tool_call_prompt = """Be careful when calling for tools. This is how a parameter of string type is set - 'parameter': '720'.
+A parameter of int type is set like this - 'parameter': 720. A parameter of the bool type is set like this - 'parameter': True.
+"""
+
+try:
+    mcp_client = MCPClient(
+        {"url": "https://artyomboyko-test-mcp-2.hf.space/gradio_api/mcp/sse"}
+    )
+
+    tools = mcp_client.get_tools()
+    # agent = ToolCallingAgent(tools=[*tools], model=model)
+    agent = ToolCallingAgent(tools=[*tools], model=model, max_steps=20, planning_interval=1)
+    agent.prompt_templates["system_prompt"] = agent.prompt_templates["system_prompt"] + tool_call_prompt
+
+    def call_agent(message, history):
+        return str(agent.run(message))
+
+    demo = gr.ChatInterface(
+        fn=call_agent,
+        type="messages",
+        examples=[test_question],
+        title="Agent with MCP Tools",
+        description="This is a simple agent that uses MCP tools to answer questions.",
+    )
+
+    demo.launch()
+except Exception as e:
+    raise e
+finally:
+    mcp_client.stop()

app_srv/app_srv.py

@@ -0,0 +1,501 @@
+import gradio as gr
+import torch
+import os
+import shutil
+import json
+import base64
+import tempfile
+import time # To simulate delays and show progress more smoothly
+from pathlib import Path
+
+from downloader import download_youtube_video
+from video_processing import extract_frames_with_timestamps, generate_frame_descriptions
+from audio_processing import transcribe_audio
+from model_api import get_device_and_dtype
+
+device, dtype = get_device_and_dtype()
+
+gui_header_element = """# 🚀 Aura AI Scan: Deep Analysis of YouTube Videos
+
+([alpha version](https://en.wikipedia.org/wiki/Software_release_life_cycle))
+
+Aura AI Scan is an MCP tool designed to deeply analyse YouTube videos, providing a frame-by-frame description with timestamps and full transcription of the audio track.
+
+The following technologies were used in the implementation:
+- [VSCode devcontainer](https://code.visualstudio.com/docs/devcontainers/containers), to simplify the development process
+- [Docker](https://huggingface.co/docs/hub/spaces-sdks-docker), to simplify local deployment. This space is also implemented as [HuggingFace Docker Space](https://huggingface.co/docs/hub/spaces-sdks-docker).
+- You can view the final results on the server's complete GUI in three formats: visual, audio, and JSON.
+- Flexible settings that can be changed (prompt used during analysis, quality of the analysed video, time interval between frames).
+- ability to include audio and frames used in the analysis process in the resulting JSON.
+
+[Quick video tutorial]https://drive.google.com/file/d/1LiZ9v5KsT3C_pJ8xLYmYkZDiM54vftjm/view?usp=drive_link)
+
+"""
+
+DEFAULT_PROMPT = """You are an expert at analyzing video, so pay close attention. Your main goal is to analyze the frame and find information in it to answer the MAIN QUESTION. Pay attention to details.  
+Provide the analysis for each frame in the following format, focusing on the frame at timestamp {timestamp}:
+FRAME: {timestamp}
+OBJECTS: List of objects with their count, for example: Bengal tiger - 1, Volvo car - 1, Person - 2 (male, female). Mentioning an object in the text on the frame does not count as a separate object. 
+If there are no objects, the field is equal to NONE.
+BACKGROUND: Description of background and surroundings, e.g.: Muddy brown water. A road in the distance. An abandoned building on the horizon.
+ACTION: A detailed description of what is happening in the frame, for example: A Bengal tiger swimming in murky water, its head and part of its back visible above the surface. 
+The shot is taken from above, above the tiger. A blue Volvo car is driving along the road in the distance. A part of a tree is visible in the right part of the frame.  
+If there are no actions, the field is equal to NONE.
+RECOGNIZED TEXT: Any text recognized in the frame, e.g.: "STOP", "VOLVO", "EXIT 25". Only the text that is present in the frame, if it is not present, this field is NONE.
+
+OBJECTS:
+BACKGROUND:
+ACTION:
+RECOGNIZED TEXT:
+"""
+
+def analyze_video_data(
+    youtube_url: str,
+    quality: str = '720',
+    time_step: str = '5.0',
+    include_audio_data: str = 'False',
+    include_frame_data: str = 'False'
+) -> str:
+    """
+    This tool returns a text description of the frames from a YouTube clip and a full transcription of the audio track of that clip.
+    Analyzing clips can be time-consuming (depending on the specified quality). You should always wait for the process to complete.
+    
+    Args:
+        youtube_url: (str) String containing the URL of the YouTube clip.
+        quality: (str, optional) Desired video quality for uploading (e.g. '144', '240', '360', '480', '720', '1080', '1440', '2160').
+                                 This parameter is optional, if it is not set, the default value of '720' will be used.
+        time_step: (str, optional) The interval in seconds at which frames will be extracted
+                                     from the video. This parameter is optional, if it is not set, the default value '5.0' will be used.
+
+        include_audio_data: (str, optional) If 'True', the base64 encoded audio data (MP3) will be included in the JSON results. Defaults to 'False'.
+        include_frame_data: (str, optional) If 'True', base64 encoded image data (JPG) for each extracted frame will be included in the JSON results. Defaults to 'False'.
+
+    Returns:
+        str: A JSON string containing the analysis results.
+             On success, it includes 'status': 'success', 'frame_analysis' (list of dictionaries
+             with 'timestamp', 'description', and optional 'image_base64'),
+             'audio_transcription', and optional 'audio_base64'.
+             On error, it includes 'status': 'error' and a 'message' detailing the error.
+    """
+    results = {
+        "status": "success",
+        "message": "",
+        "frame_analysis": [],
+        "audio_transcription": "",
+        "audio_base64": ""
+    }
+
+    try:
+
+        # DEBUG
+        print("Function analyze_video_data passed arguments: {youtube_url}, {quality}, {time_step}, {include_audio_data}, {include_frame_data}.")
+        print("Arguments types: ", type(youtube_url), type(quality), type(time_step), type(include_audio_data), type(include_frame_data))
+
+        # The tool call agent uses strings, so all function arguments must be of string type.
+        quality_int = int(quality)
+        time_step_float = float(time_step)
+        include_frame_data_bool = include_frame_data.lower() == 'true'
+        include_audio_data_bool = include_audio_data.lower() == 'true'
+
+        # For debugging purpose
+        print(f'Starting pprocessing tast with {youtube_url}, quality: {quality}, time step: {time_step}, include audio: {include_audio_data}, include frames {include_frame_data}.')
+
+        video_data = download_youtube_video(
+            url=youtube_url, video_quality=quality_int, # youtube_cookies=cookies
+        )
+
+        frames_dict = extract_frames_with_timestamps(
+            video_path=video_data["video_path"],
+            output_dir=video_data["data_path"],
+            time_step=time_step_float,
+            hw_device="cuda",
+        )
+
+        descriptions = generate_frame_descriptions(
+            frames_dict=frames_dict,
+            custom_prompt=DEFAULT_PROMPT,
+            device=device,
+            torch_dtype=dtype,
+        )
+
+        transcription_text = transcribe_audio(video_data["audio_path"])
+
+        for timestamp, frame_path in frames_dict.items():
+            description = descriptions.get(timestamp, "No description available")
+            frame_entry = {"timestamp": timestamp, "description": description, "image_base64": ""}
+
+            if include_frame_data_bool and os.path.exists(frame_path):
+                with open(frame_path, "rb") as f:
+                    frame_entry["image_base64"] = base64.b64encode(f.read()).decode("utf-8")
+
+            results["frame_analysis"].append(frame_entry)
+
+        results["audio_transcription"] = transcription_text
+
+        if include_audio_data_bool and os.path.exists(video_data["audio_path"]):
+            with open(video_data["audio_path"], "rb") as f:
+                results["audio_base64"] = base64.b64encode(f.read()).decode("utf-8")
+
+        return json.dumps(results, indent=2, ensure_ascii=False)
+
+    except Exception as e:
+
+        # DEBUG
+        print("Function analyze_video_data passed arguments: {youtube_url}, {quality}, {time_step}, {include_audio_data}, {include_frame_data}.")
+        print("Arguments types: ", type(youtube_url), type(quality), type(time_step), type(include_audio_data), type(include_frame_data))
+
+        error_message = f"Processing error: {str(e)}"
+        results["status"] = "error"
+        results["message"] = error_message
+        results["frame_analysis"] = []
+        results["audio_transcription"] = ""
+        results["audio_base64"] = ""
+
+        for frame_entry in results["frame_analysis"]:
+            frame_entry["image_base64"] = ""
+
+        return json.dumps(results, indent=2, ensure_ascii=False)
+
+
+def get_video_html_from_json(json_string: str) -> str:
+    try:
+        data = json.loads(json_string)
+        if data["status"] == "error":
+            return f"<p style='color:red;'>Error: {data['message']}</p>"
+
+        html_content = ""
+        if not data["frame_analysis"]:
+            html_content += "<p>No frames analyzed or included.</p>"
+        else:
+            for frame in data["frame_analysis"]:
+                timestamp = frame.get("timestamp", "N/A")
+                description = frame.get("description", "No description available")
+                image_base64 = frame.get("image_base64", "")
+
+                html_content += f"<div style='margin-bottom: 20px; border: 1px solid #eee; padding: 10px; border-radius: 8px;'>"
+                html_content += f"<h3>FRAME: {timestamp}</h3>"
+                if image_base64:
+                    html_content += f"<img src='data:image/jpeg;base64,{image_base64}' style='max-width: 100%; height: auto; border-radius: 4px; margin-bottom: 10px;'><br>"
+                else:
+                    html_content += f"<p>Image data not included for this frame (checkbox 'Include Frame Data' was not selected).</p>"
+                html_content += f"<p><strong>Description:</strong> {description}</p>"
+                html_content += "</div>"
+        return html_content
+    except json.JSONDecodeError:
+        return "<p style='color:red;'>Invalid JSON response.</p>"
+    except Exception as e:
+        return f"<p style='color:red;'>Error processing video data for display: {str(e)}</p>"
+
+
+def get_audio_data_from_json(json_string: str) -> tuple[str, str | None]:
+    try:
+        data = json.loads(json_string)
+        if data["status"] == "error":
+            return f"Error: {data['message']}", None
+
+        transcription = data.get("audio_transcription", "No transcription available.")
+        audio_base64 = data.get("audio_base64", "")
+
+        if audio_base64:
+            with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as temp_audio_file:
+                temp_audio_file.write(base64.b64decode(audio_base64))
+                temp_audio_path = temp_audio_file.name
+            
+            return transcription, temp_audio_path
+        else:
+            transcription += "\n\nAudio data not included (checkbox 'Include Audio Data' was not selected)."
+            return transcription, None
+
+    except json.JSONDecodeError:
+        return "Invalid JSON response for audio.", None
+    except Exception as e:
+        return f"Error processing audio data for display: {str(e)}", None
+
+# Wrapper function for analysis with progress bar
+def analyze_video_data_with_progress_wrapper(
+    youtube_url: str,
+    prompt: str,
+    quality: int,
+    time_step: float,
+    include_audio_data: bool,
+    include_frame_data: bool,
+    progress=gr.Progress()
+):
+    results = {
+        "status": "pending",
+        "message": "Starting analysis...",
+        "frame_analysis": [],
+        "audio_transcription": "",
+        "audio_base64": ""
+    }
+
+    try:
+        progress(0, desc="Downloading video...")
+
+
+        # Step 1: Downloading a YouTube video
+        video_data = download_youtube_video(
+            url=youtube_url, video_quality=quality, # youtube_cookies=cookies
+        )
+        progress(0.25, desc="Extracting frames...")
+        
+        # Step 2: Extract frames from video
+        frames_dict = extract_frames_with_timestamps(
+            video_path=video_data["video_path"],
+            output_dir=video_data["data_path"],
+            time_step=time_step,
+            hw_device="cuda",
+        )
+        progress(0.5, desc="Generating frame descriptions...")
+        
+        # Step 3: Generate frames descriptions
+        descriptions = generate_frame_descriptions(
+            frames_dict=frames_dict,
+            custom_prompt=prompt,
+            device=device,
+            torch_dtype=dtype,
+        )
+        progress(0.75, desc="Transcribing audio...")
+        
+        # Step 4: Transcribe the audio
+        transcription_text = transcribe_audio(video_data["audio_path"])
+        progress(0.9, desc="Consolidating results...")
+        
+        # Build the final results dictionary
+        for timestamp, frame_path in frames_dict.items():
+            description = descriptions.get(timestamp, "No description available")
+            frame_entry = {"timestamp": timestamp, "description": description, "image_base64": ""}
+
+            if include_frame_data and os.path.exists(frame_path):
+                with open(frame_path, "rb") as f:
+                    frame_entry["image_base64"] = base64.b64encode(f.read()).decode("utf-8")
+
+            results["frame_analysis"].append(frame_entry)
+
+        results["audio_transcription"] = transcription_text
+
+        if include_audio_data and os.path.exists(video_data["audio_path"]):
+            with open(video_data["audio_path"], "rb") as f:
+                results["audio_base64"] = base64.b64encode(f.read()).decode("utf-8")
+        
+        results["status"] = "success"
+        results["message"] = "Analysis complete!"
+        progress(1.0, desc="Analysis complete!")
+        yield json.dumps(results, indent=2, ensure_ascii=False)
+
+    except Exception as e:
+        error_message = f"Processing error: {str(e)}"
+        results["status"] = "error"
+        results["message"] = error_message
+        results["frame_analysis"] = []
+        results["audio_transcription"] = ""
+        results["audio_base64"] = ""
+        
+        progress(1.0, desc="Analysis failed!")
+        yield json.dumps(results, indent=2, ensure_ascii=False)
+
+
+# The path where the cookie.txt file is saved
+working_cookies_file_path = "/home/mcp_user/app_srv/cookies.txt"
+
+# Global variable to store the path to the last temporary Gradio file
+gradio_temp_cookies_file_path = None 
+
+
+def upload_cookies_file(file):
+    """
+    Copies the uploaded file from Gradio temporary storage to working_cookies_file_path.
+    Saves the path of the temporary file to a global variable for later deletion.
+    """
+    global gradio_temp_cookies_file_path # Declare global variable to be changed
+
+    if file is None:
+        return "Please first select a cookie file to upload."
+
+    source_path = file.name
+    gradio_temp_cookies_file_path = source_path
+
+    try:
+        shutil.copy(source_path, working_cookies_file_path)
+    
+        return (f"File successfully copied and saved as: {working_cookies_file_path}.\n"
+                f"Path to the Gradio temporary file: {source_path}.")
+    
+    except Exception as e:
+        return f"Error occurred while file copying: {e}"
+
+
+def clear_cookies_files():
+    """
+    Deletes working_cookies_file_path and also attempts to delete the last temporary Gradio file,
+    path to which is stored globally.
+    """
+    global gradio_temp_cookies_file_path # Declare global variable to be changed
+
+    status_messages = [] # List to collect status messages
+
+    if os.path.exists(working_cookies_file_path): 
+        try:
+            os.remove(working_cookies_file_path)
+            status_messages.append(f"File {working_cookies_file_path} successfully deleted.")
+        except Exception as e:
+            status_messages.append(f"Error while deleting a file {working_cookies_file_path}: {e}.")
+    else:
+        status_messages.append(f"File {working_cookies_file_path} not found.")
+    
+
+    if gradio_temp_cookies_file_path and os.path.exists(gradio_temp_cookies_file_path):
+        try:
+            os.remove(gradio_temp_cookies_file_path)
+            status_messages.append(f"Temporary Gradio file ({os.path.basename(gradio_temp_cookies_file_path)}) was successfully deleted.")
+        except Exception as e:
+            status_messages.append(f"Error deleting a temporary Gradio file ({os.path.basename(gradio_temp_cookies_file_path)}): {e}.")
+        finally:
+            gradio_temp_cookies_file_path = None 
+    elif gradio_temp_cookies_file_path is None:
+        status_messages.append("Path to the Gradio temporary file was unknown.")
+    else:
+        status_messages.append(f"Temporary Gradio file ({os.path.basename(gradio_temp_cookies_file_path)}) no longer exists.")
+
+    # Merge all status messages
+    return "\n".join(status_messages)
+
+
+with gr.Blocks(title="Video Analysis Tool",) as demo:
+
+    gr.Markdown(gui_header_element)
+
+    with gr.Row():
+        youtube_url = gr.Textbox(
+            label="YouTube Video URL",
+            value="https://www.youtube.com/watch?v=FK3dav4bA4s&t=36s",
+            lines=1,
+            scale=5
+        )
+
+    with gr.Row():
+        prompt = gr.Textbox(
+            label="Analysis Prompt",
+            value=DEFAULT_PROMPT,
+            lines=3,
+            scale=4
+        )
+
+        with gr.Column(scale=2, min_width=200):
+
+            file_input = gr.File(label="Select a cookie file to upload", file_count="single", height=263)
+
+            output_message = gr.Textbox(label="Status of uploading file with cookies")
+
+            upload_cookies_file_button = gr.Button("Save file with cookies")
+        
+
+        with gr.Column(scale=2, min_width=200):
+            quality = gr.Dropdown(
+                label="Video Quality",
+                choices=[144, 240, 360, 480, 720, 1080, 1440, 2160],
+                value=720
+            )
+            time_step = gr.Slider(
+                label="Frame Interval (seconds)",
+                minimum=0.5,
+                maximum=30,
+                step=0.5,
+                value=30
+            )
+            include_audio_data = gr.Checkbox(
+                label="Include Audio Data (MP3) in Results", value=False
+            )
+            include_frame_data = gr.Checkbox(
+                label="Include Frame Data (JPG) in Results", value=False
+            )
+            
+            t1 = gr.Textbox(value="Waiting for task...",label="Task Progress", show_label=True, lines=3, interactive=False)
+
+            # Button to create an MCP server point
+            submit_btn = gr.Button("Start Video Analysis (No Progress Bar, for MCP Server use)", variant="primary", visible=False)
+
+            # Analyze button with progress bar
+            submit_btn_with_progress = gr.Button("Analyze Video", variant="secondary")
+        
+    with gr.Tabs() as results_tabs:
+        with gr.TabItem("Video"):
+            video_output_html = gr.HTML(label="Video Frames Analysis", elem_id="video-output-html")
+        with gr.TabItem("Audio"):
+               audio_player_output = gr.Audio(label="Play Audio", type="filepath", render=True)
+               audio_transcription_output = gr.Textbox(label="Audio Transcription", lines=10)
+        with gr.TabItem("JSON"):
+            results_json_viewer = gr.JSON(
+                label="Raw Analysis Results (JSON)",
+                elem_classes=["output-box", "results-output"],
+            )
+
+    raw_json_output = gr.State()
+
+    # Logic for normal button (without progress bar), this button becomes the MCP server point.
+    submit_btn.click(
+        fn=analyze_video_data,
+        inputs=[youtube_url, quality, time_step, include_audio_data, include_frame_data],
+        outputs=[raw_json_output],
+        api_name="analyze_video_data",
+        show_api=True
+    ).then(
+        fn=get_video_html_from_json,
+        inputs=[raw_json_output],
+        outputs=[video_output_html],
+        show_api=False
+    ).then(
+        fn=get_audio_data_from_json,
+        inputs=[raw_json_output],
+        outputs=[audio_transcription_output, audio_player_output],
+        show_api=False
+    ).then(
+        fn=lambda x: json.loads(x),
+        inputs=[raw_json_output],
+        outputs=[results_json_viewer],
+        show_api=False
+    )
+
+    # Logic for button with progress bar
+    submit_btn_with_progress.click(
+        fn=analyze_video_data_with_progress_wrapper,
+        inputs=[youtube_url, prompt, quality, time_step, include_audio_data, include_frame_data],
+        outputs=[raw_json_output],
+        api_name="analyze_video_data_with_progress_button",
+        show_progress_on=t1,
+        show_api=False
+    ).then(
+        fn=get_video_html_from_json,
+        inputs=[raw_json_output],
+        outputs=[video_output_html],
+        show_api=False
+    ).then(
+        fn=get_audio_data_from_json,
+        inputs=[raw_json_output],
+        outputs=[audio_transcription_output, audio_player_output],
+        show_api=False
+    ).then(
+        fn=lambda x: json.loads(x),
+        inputs=[raw_json_output],
+        outputs=[results_json_viewer],
+        show_api=False
+    )
+
+    # Logic of processing cookies
+    upload_cookies_file_button.click(
+        fn=upload_cookies_file,
+        inputs=file_input,
+        outputs=output_message,
+        show_api=False
+    )
+
+    file_input.clear(
+        fn=clear_cookies_files,
+        inputs=None,
+        outputs=output_message,
+        show_api=False
+    )
+
+
+if __name__ == "__main__":
+    demo.launch(mcp_server=True)

app_srv/audio_processing.py

@@ -0,0 +1,54 @@
+from transformers import pipeline
+import torch
+import gc
+import librosa
+
+from model_api import clear_gpu_cache, get_device_and_dtype
+
+
+def transcribe_audio(audio_path: str, device: str = "cuda", torch_dtype: torch.dtype = torch.float16) -> str:
+    """
+    Transcribes an MP3 audio file.
+
+    Args:
+        audio_path: Path to the audio file.
+        device: The device to use for transcription (e.g., "cuda" for GPU, "cpu" for CPU).
+        torch_dtype: The torch data type to use for model computations.
+
+    Returns:
+        The transcribed text.
+    """
+    try:
+        # Create a pipeline with explicit device specification
+        pipe = pipeline(
+            "automatic-speech-recognition",
+            model="openai/whisper-small",
+            device=device,
+            chunk_length_s=25,
+            stride_length_s=2,
+            torch_dtype=torch_dtype,
+        )
+
+        # Perform transcription
+        result = pipe(audio_path, return_timestamps=True)
+
+        # Extract text
+        text = result['text']
+
+        return text
+
+    except Exception as e:
+        print(f"Error during transcription: {str(e)}")
+        return ""
+
+    finally:
+        del pipe
+        clear_gpu_cache()
+
+
+if __name__ == "__main__":
+
+    selected_device, selected_dtype = get_device_and_dtype()
+
+    result = transcribe_audio("/workspaces/Video_Analyser/app_srv/downloads/45677153-510d-4f47-95ee-c1b4b0843433/audio.mp3.mp3", selected_device, selected_dtype)
+    print(result)

app_srv/downloader.py

@@ -0,0 +1,145 @@
+import yt_dlp
+import os
+import uuid
+import json
+from pathlib import Path
+from typing import Dict, Any
+from datetime import datetime
+
+
+def download_youtube_video(url: str,
+                           base_dir: str = None,
+                           video_quality: int = 720,
+                           youtube_cookies: str = "./cookies.txt") -> Dict[str, str]:
+    """
+    Downloads video and audio from YouTube, saving them to a unique GUID folder.
+    Metadata is saved in JSON format including download datetime and timezone.
+    
+    Args:
+        url (str): YouTube video URL
+        base_dir (str): Base download directory (default './downloads')
+        video_quality (int): preferred quality of the downloaded video, acceptable values 144, 240, 360, 480, 720, 1080, 1440, 2160. Default value: 720.
+    
+    Returns:
+        dict: Dictionary with file paths and information:
+            {
+                'data_path': str,     # Path to download directory
+                'video_path': str,     # Full path to video.mp4
+                'audio_path': str,     # Full path to audio.mp3
+                'metadata_path': str  # Full path to metadata.json
+            }
+    
+    Raises:
+        RuntimeError: If download fails
+    """
+    
+    youtube_quality = [144, 240, 360, 480, 720, 1080, 1440, 2160]
+
+    if video_quality not in youtube_quality:
+        raise ValueError(
+            f"Invalid video quality: '{video_quality}'. "
+            f"Allowed qualities are: {', '.join(map(str, youtube_quality))}"
+        )
+
+    try:
+
+        # Determine the base directory
+        if base_dir is None:
+            # Get the directory of the current script file
+            script_dir = Path(__file__).parent
+            base_dir = script_dir / "downloads" # Создаем папку 'downloads' рядом со скриптом
+        else:
+            base_dir = Path(base_dir)
+
+        # Generate GUID and create folder
+        guid = str(uuid.uuid4())
+        download_dir = Path(base_dir) / guid
+        os.makedirs(download_dir, exist_ok=True)
+        
+        # File paths
+        video_path = download_dir / "video.mp4"
+        audio_path = download_dir / "audio.mp3"
+        metadata_path = download_dir / "metadata.json"
+
+        # Record exact download start time
+        download_datetime = datetime.now()
+        current_timezone = download_datetime.astimezone().tzinfo
+        
+        # 1. Download video (MP4)
+        video_opts = {
+            'format': (
+                f"bestvideo[height={video_quality}][ext=mp4]"
+                f"/worstvideo[height>{video_quality}][ext=mp4]"
+                f"/bestvideo[height<={video_quality}][ext=mp4]"
+                ),
+            'outtmpl': str(video_path),
+            'quiet': True,
+            'no_warnings': True,
+            'restrict_filenames': True,
+            'cookiefile': youtube_cookies,
+        }
+        
+        with yt_dlp.YoutubeDL(video_opts) as ydl:
+            video_info = ydl.extract_info(url, download=True)
+        
+        # 2. Download audio (MP3)
+        audio_opts = {
+            'format': 'bestaudio/best',
+            'outtmpl': str(audio_path),
+            'quiet': True,
+            'postprocessors': [{
+                'key': 'FFmpegExtractAudio',
+                'preferredcodec': 'mp3',
+                'preferredquality': '128',
+            }],
+            'cookiefile': youtube_cookies,
+        }
+        
+        with yt_dlp.YoutubeDL(audio_opts) as ydl:
+            audio_info = ydl.extract_info(url, download=True)
+        
+        # Format date and time for storage
+        formatted_date = download_datetime.strftime('%Y-%m-%d')
+        formatted_time = download_datetime.strftime('%H:%M:%S')
+        
+        # 3. Save metadata to JSON
+        metadata = {
+            'original_url': url,
+            'guid': guid,
+            'download_info': {
+                'date': formatted_date,
+                'time': formatted_time,
+                'timezone': str(current_timezone),
+                'datetime_iso': download_datetime.isoformat(),
+            },
+            'video': {
+                'path': str(video_path),
+                'title': video_info.get('title'),
+                'duration': video_info.get('duration'),
+                'resolution': video_info.get('resolution'),
+                'upload_date': video_info.get('upload_date'),
+            },
+            'audio': {
+                'path': str(audio_path),
+                'bitrate': audio_info.get('abr'),
+                'codec': 'mp3',
+            },
+        }
+        
+        with open(metadata_path, 'w', encoding='utf-8') as f:
+            json.dump(metadata, f, indent=2, ensure_ascii=False)
+        
+        return {
+            'data_path': str(download_dir.absolute()),
+            'video_path': str(video_path.absolute()),
+            'audio_path': str(audio_path.absolute()) + ".mp3",
+            'metadata_path': str(metadata_path),
+        }
+        
+    except Exception as e:
+        raise RuntimeError(f"Media download error: {str(e)}")
+
+if __name__ == "__main__":
+    video_url = "https://www.youtube.com/watch?v=FK3dav4bA4s"
+    downloaded_video = download_youtube_video(video_url, "./temp")
+    print(downloaded_video)

app_srv/model_api.py

@@ -0,0 +1,122 @@
+from transformers import Qwen2_5_VLForConditionalGeneration, AutoProcessor, logging
+from qwen_vl_utils import process_vision_info
+import torch
+from PIL import Image
+import gc
+import warnings
+
+
+### Utility functions ###
+def clear_gpu_cache():
+    """Clears GPU cache and performs garbage collection"""
+    torch.cuda.empty_cache()
+    gc.collect()
+
+
+def get_device_and_dtype():
+    if torch.cuda.is_available():
+        device = 'cuda'
+        torch_dtype = torch.float16
+    else:
+        device = 'cpu'
+        torch_dtype = torch.float32
+    return device, torch_dtype
+
+
+### Model-related functions ###
+def initialize_model(device: str, torch_dtype: torch.dtype):
+    """Initialize and return model with GPU optimization"""
+    with warnings.catch_warnings():
+        warnings.simplefilter("ignore")
+        return Qwen2_5_VLForConditionalGeneration.from_pretrained(
+            "Qwen/Qwen2.5-VL-7B-Instruct-AWQ",
+            device_map=device,
+            torch_dtype=torch_dtype, 
+        )
+
+
+def initialize_processor():
+    """Initialize and return processor"""
+    return AutoProcessor.from_pretrained("Qwen/Qwen2.5-VL-7B-Instruct-AWQ")
+
+
+def prepare_model_inputs(processor, messages, device: str):
+    """Prepare all inputs for model inference"""
+    text = processor.apply_chat_template(
+        messages, tokenize=False, add_generation_prompt=True
+    )
+    image_inputs, video_inputs = process_vision_info(messages)
+
+    return processor(
+        text=[text],
+        images=image_inputs,
+        videos=video_inputs,
+        padding=True,
+        return_tensors="pt"
+    ).to(device)
+
+
+def generate_description(model, inputs):
+    """Generate description using model"""
+    return model.generate(
+        **inputs,
+        max_new_tokens=512,
+        do_sample=True,
+        temperature=0.7,
+        top_p=0.9
+    )
+
+
+def process_model_output(processor, inputs, generated_ids):
+    """Process model output to get clean text"""
+    generated_ids_trimmed = [
+        out_ids[len(in_ids):] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
+    ]
+    return processor.batch_decode(
+        generated_ids_trimmed,
+        skip_special_tokens=True,
+        clean_up_tokenization_spaces=False
+    )[0]
+
+
+def create_prompt_message(image_path, timestamp, custom_prompt=None):
+    """
+    Create standardized message prompt for model with optional custom prompt
+    
+    Args:
+        image_path (str): Path to the image file
+        timestamp (str): Video timestamp in HH:MM:SS.SSS format
+        custom_prompt (str, optional): Custom prompt text. If None, uses default prompt.
+    
+    Returns:
+        list: Formatted message for model input
+    """
+    default_prompt = f"Video timestamp: {timestamp}. Describe in detail what is happening in this frame."
+    prompt_text = custom_prompt.format(timestamp=timestamp) if custom_prompt else default_prompt
+    
+    return [{
+        "role": "user",
+        "content": [
+            {"type": "image", "image": f"file://{image_path}"},
+            {"type": "text", "text": prompt_text},
+        ],
+    }]
+
+
+if __name__ == "__main__":
+
+    selected_device, selected_dtype = get_device_and_dtype()
+
+    image_url = "/workspaces/Video_Analyser/app_srv/temp/a28af289-377d-468d-b0eb-ed0f7dcd2ab3/frames/frame_00_01_50_110.jpg"
+    timestamp = "00:01:16.076"
+    custom_prompt = "Timestamp {timestamp}. Analyze this frame focusing on objects in the foreground."
+
+    model = initialize_model(selected_device, selected_dtype)
+    processor = initialize_processor()
+    messages = create_prompt_message(image_url, timestamp, custom_prompt)
+    inputs = prepare_model_inputs(processor, messages, selected_device)
+
+    generated_ids = generate_description(model, inputs)
+    image_description = process_model_output(processor, inputs, generated_ids)
+
+    print(image_description)

app_srv/test.ipynb

@@ -0,0 +1,555 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "7f265e58",
+   "metadata": {},
+   "source": [
+    "# Download video and audio from YouTube"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "69ee0ec3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import yt_dlp\n",
+    "import os\n",
+    "import uuid\n",
+    "import json\n",
+    "from pathlib import Path\n",
+    "from typing import Dict, Any\n",
+    "from datetime import datetime\n",
+    "\n",
+    "\n",
+    "def download_youtube_media(url: str,\n",
+    "                           base_dir: str = \"./downloads\",\n",
+    "                           video_quality: int = 720) -> Dict[str, str]:\n",
+    "    \"\"\"\n",
+    "    Downloads video and audio from YouTube, saving them to a unique GUID folder.\n",
+    "    Metadata is saved in JSON format including download datetime and timezone.\n",
+    "    \n",
+    "    Args:\n",
+    "        url (str): YouTube video URL\n",
+    "        base_dir (str): Base download directory (default './downloads')\n",
+    "        video_quality (int): preferred quality of the downloaded video, acceptable values 144, 240, 360, 480, 720, 1080, 1440, 2160.\n",
+    "    \n",
+    "    Returns:\n",
+    "        dict: Dictionary with file paths and information:\n",
+    "            {\n",
+    "                'data_path': str,     # Path to download directory\n",
+    "                'video_path': str,     # Full path to video.mp4\n",
+    "                'audio_path': str,     # Full path to audio.mp3\n",
+    "                'metadata_path': str  # Full path to metadata.json\n",
+    "            }\n",
+    "    \n",
+    "    Raises:\n",
+    "        RuntimeError: If download fails\n",
+    "    \"\"\"\n",
+    "    \n",
+    "    youtube_quality = [144, 240, 360, 480, 720, 1080, 1440, 2160]\n",
+    "\n",
+    "\n",
+    "    if video_quality not in youtube_quality:\n",
+    "        raise ValueError(\n",
+    "            f\"Invalid video quality: '{video_quality}'. \"\n",
+    "            f\"Allowed qualities are: {', '.join(map(str, youtube_quality))}\"\n",
+    "        )\n",
+    "\n",
+    "    try:\n",
+    "        # Generate GUID and create folder\n",
+    "        guid = str(uuid.uuid4())\n",
+    "        download_dir = Path(base_dir) / guid\n",
+    "        os.makedirs(download_dir, exist_ok=True)\n",
+    "        \n",
+    "        # File paths\n",
+    "        video_path = download_dir / \"video.mp4\"\n",
+    "        audio_path = download_dir / \"audio.mp3\"\n",
+    "        metadata_path = download_dir / \"metadata.json\"\n",
+    "\n",
+    "        # Record exact download start time\n",
+    "        download_datetime = datetime.now()\n",
+    "        current_timezone = download_datetime.astimezone().tzinfo\n",
+    "        \n",
+    "        # 1. Download video (MP4)\n",
+    "        video_opts = {\n",
+    "            'format': (\n",
+    "                f\"bestvideo[height={video_quality}][ext=mp4]\"\n",
+    "                f\"/worstvideo[height>{video_quality}][ext=mp4]\"\n",
+    "                f\"/bestvideo[height<={video_quality}][ext=mp4]\"\n",
+    "                ),\n",
+    "            'outtmpl': str(video_path),\n",
+    "            'quiet': True,\n",
+    "            'no_warnings': True,\n",
+    "            'restrict_filenames': True,\n",
+    "        }\n",
+    "        \n",
+    "        with yt_dlp.YoutubeDL(video_opts) as ydl:\n",
+    "            video_info = ydl.extract_info(url, download=True)\n",
+    "        \n",
+    "        # 2. Download audio (MP3)\n",
+    "        audio_opts = {\n",
+    "            'format': 'bestaudio/best',\n",
+    "            'outtmpl': str(audio_path),\n",
+    "            'quiet': True,\n",
+    "            'postprocessors': [{\n",
+    "                'key': 'FFmpegExtractAudio',\n",
+    "                'preferredcodec': 'mp3',\n",
+    "                'preferredquality': '128',\n",
+    "            }],\n",
+    "        }\n",
+    "        \n",
+    "        with yt_dlp.YoutubeDL(audio_opts) as ydl:\n",
+    "            audio_info = ydl.extract_info(url, download=True)\n",
+    "        \n",
+    "        # Format date and time for storage\n",
+    "        formatted_date = download_datetime.strftime('%Y-%m-%d')\n",
+    "        formatted_time = download_datetime.strftime('%H:%M:%S')\n",
+    "        \n",
+    "        # 3. Save metadata to JSON\n",
+    "        metadata = {\n",
+    "            'original_url': url,\n",
+    "            'guid': guid,\n",
+    "            'download_info': {\n",
+    "                'date': formatted_date,\n",
+    "                'time': formatted_time,\n",
+    "                'timezone': str(current_timezone),\n",
+    "                'datetime_iso': download_datetime.isoformat(),\n",
+    "            },\n",
+    "            'video': {\n",
+    "                'path': str(video_path),\n",
+    "                'title': video_info.get('title'),\n",
+    "                'duration': video_info.get('duration'),\n",
+    "                'resolution': video_info.get('resolution'),\n",
+    "                'upload_date': video_info.get('upload_date'),\n",
+    "            },\n",
+    "            'audio': {\n",
+    "                'path': str(audio_path),\n",
+    "                'bitrate': audio_info.get('abr'),\n",
+    "                'codec': 'mp3',\n",
+    "            },\n",
+    "        }\n",
+    "        \n",
+    "        with open(metadata_path, 'w', encoding='utf-8') as f:\n",
+    "            json.dump(metadata, f, indent=2, ensure_ascii=False)\n",
+    "        \n",
+    "        return {\n",
+    "            'data_path': str(download_dir.absolute()),\n",
+    "            'video': str(video_path.absolute()),\n",
+    "            'audio': str(audio_path.absolute()) + \".mp3\",\n",
+    "            'metadata': str(metadata_path),\n",
+    "        }\n",
+    "        \n",
+    "    except Exception as e:\n",
+    "        raise RuntimeError(f\"Media download error: {str(e)}\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    video_url = \"https://www.youtube.com/watch?v=FK3dav4bA4s\"\n",
+    "    downloaded_video = download_youtube_media(video_url, \"./temp\")\n",
+    "    print(downloaded_video)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e79c5071",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "downloaded_video"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "745320a1",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import copy\n",
+    "test = copy.deepcopy(downloaded_video)\n",
+    "\n",
+    "print(test)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "f62e8b83",
+   "metadata": {},
+   "source": [
+    "# Split video to frames in jpg"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5461045d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import os\n",
+    "from pathlib import Path\n",
+    "from typing import Dict\n",
+    "import av\n",
+    "\n",
+    "def extract_frames_with_timestamps(\n",
+    "    video_path: str,\n",
+    "    output_dir: str,\n",
+    "    time_step: float = 1.0,\n",
+    "    quality: int = 95,\n",
+    "    frame_prefix: str = \"frame\",\n",
+    "    use_hw_accel: bool = True,\n",
+    "    hw_device: str = \"cuda\"\n",
+    ") -> Dict[str, str]:\n",
+    "    \"\"\"\n",
+    "    Extracts frames from video with NVIDIA hardware acceleration (NVDEC/CUDA).\n",
+    "    \n",
+    "    Args:\n",
+    "        video_path: Path to the video file\n",
+    "        output_dir: Directory to save frames\n",
+    "        time_step: Interval between frames (seconds)\n",
+    "        quality: JPEG quality (1-100)\n",
+    "        frame_prefix: Prefix for saved frames\n",
+    "        use_hw_accel: Enable NVIDIA hardware decoding\n",
+    "        hw_device: GPU device (e.g., 'cuda:0')\n",
+    "\n",
+    "    Returns:\n",
+    "        Dict of {timestamp: frame_path}\n",
+    "    \"\"\"\n",
+    "    result = {}\n",
+    "    try:\n",
+    "        video_path = Path(video_path).absolute()\n",
+    "        output_dir = Path(output_dir).absolute()\n",
+    "        \n",
+    "        if not video_path.exists():\n",
+    "            raise ValueError(f\"Video file not found: {video_path}\")\n",
+    "\n",
+    "        frames_dir = output_dir / \"frames\"\n",
+    "        frames_dir.mkdir(parents=True, exist_ok=True)\n",
+    "\n",
+    "        # Configure hardware acceleration\n",
+    "        options = {}\n",
+    "        if use_hw_accel:\n",
+    "            options.update({\n",
+    "                'hwaccel': 'cuda',\n",
+    "                'hwaccel_device': hw_device,\n",
+    "                'hwaccel_output_format': 'cuda'  # Keep frames in GPU memory\n",
+    "            })\n",
+    "\n",
+    "        # Open video with hardware acceleration\n",
+    "        container = av.open(str(video_path), options=options)\n",
+    "        video_stream = next(s for s in container.streams if s.type == 'video')\n",
+    "\n",
+    "        fps = float(video_stream.average_rate)\n",
+    "        if fps <= 0:\n",
+    "            raise RuntimeError(\"Invalid frame rate\")\n",
+    "\n",
+    "        frame_interval = max(1, int(round(fps * time_step)))\n",
+    "        frame_count = 0\n",
+    "\n",
+    "        for frame in container.decode(video_stream):\n",
+    "            if frame_count % frame_interval == 0:\n",
+    "                current_time = float(frame.pts * video_stream.time_base)\n",
+    "                hh = int(current_time // 3600)\n",
+    "                mm = int((current_time % 3600) // 60)\n",
+    "                ss = current_time % 60\n",
+    "                \n",
+    "                timestamp = f\"{hh:02d}:{mm:02d}:{ss:06.3f}\"\n",
+    "                safe_timestamp = timestamp.replace(':', '_').replace('.', '_')\n",
+    "                frame_path = frames_dir / f\"{frame_prefix}_{safe_timestamp}.jpg\"\n",
+    "\n",
+    "                # Convert GPU frame to CPU if needed\n",
+    "                if hasattr(frame, 'to_ndarray'):  # CUDA frame\n",
+    "                    img = frame.to_ndarray(format='rgb24')\n",
+    "                    img = av.VideoFrame.from_ndarray(img, format='rgb24')\n",
+    "                else:\n",
+    "                    img = frame\n",
+    "\n",
+    "                img.to_image().save(str(frame_path), quality=quality)\n",
+    "                result[timestamp] = str(frame_path)\n",
+    "\n",
+    "            frame_count += 1\n",
+    "\n",
+    "        return result\n",
+    "\n",
+    "    except Exception as e:\n",
+    "        for path in result.values():\n",
+    "            try: os.remove(path)\n",
+    "            except: pass\n",
+    "        raise RuntimeError(f\"Frame extraction failed: {str(e)}\")\n",
+    "\n",
+    "if __name__ == \"__main__\":\n",
+    "    frames = extract_frames_with_timestamps(downloaded_video['video'], downloaded_video['data_path'], time_step=2)\n",
+    "    print(frames)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "ba7b44d6",
+   "metadata": {},
+   "source": [
+    "# Video Analyzer"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "418ae84e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# pip install autoawq --upgrade"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "fe840ffa",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import Qwen2_5_VLForConditionalGeneration, AutoTokenizer, AutoProcessor\n",
+    "from qwen_vl_utils import process_vision_info\n",
+    "import torch\n",
+    "\n",
+    "# default: Load the model on the available device(s)\n",
+    "model = Qwen2_5_VLForConditionalGeneration.from_pretrained(\n",
+    "    \"Qwen/Qwen2.5-VL-7B-Instruct-AWQ\",\n",
+    "    torch_dtype=torch.float16,\n",
+    "    device_map=\"auto\",\n",
+    ")\n",
+    "\n",
+    "# default processer\n",
+    "processor = AutoProcessor.from_pretrained(\"Qwen/Qwen2.5-VL-7B-Instruct-AWQ\")\n",
+    "\n",
+    "messages = [\n",
+    "    {\n",
+    "        \"role\": \"user\",\n",
+    "        \"content\": [\n",
+    "            {\"type\": \"image\", \"image\": \"file:///workspaces/Video_Analyser/temp/fcaaa3e8-d99d-47c5-b464-617e4c9a1b1a/frames/frame_00_02_51_171.jpg\"},\n",
+    "            {\"type\": \"text\", \"text\": \"Describe this image in\"},\n",
+    "        ],\n",
+    "    }\n",
+    "]\n",
+    "\n",
+    "# Preparation for inference\n",
+    "text = processor.apply_chat_template(\n",
+    "    messages, tokenize=False, add_generation_prompt=True\n",
+    ")\n",
+    "\n",
+    "image_inputs, video_inputs = process_vision_info(messages)\n",
+    "inputs = processor(\n",
+    "    text=[text],\n",
+    "    images=image_inputs,\n",
+    "    videos=video_inputs,\n",
+    "    padding=True,\n",
+    "    return_tensors=\"pt\",\n",
+    ")\n",
+    "inputs = inputs.to(\"cuda\")\n",
+    "\n",
+    "# Inference: Generation of the output\n",
+    "generated_ids = model.generate(**inputs, max_new_tokens=128)\n",
+    "generated_ids_trimmed = [\n",
+    "    out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)\n",
+    "]\n",
+    "output_text = processor.batch_decode(\n",
+    "    generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False\n",
+    ")\n",
+    "print(output_text)\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "3ba7ef97",
+   "metadata": {},
+   "source": [
+    "# Audio content"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "bf798dcf",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline\n",
+    "from datasets import load_dataset\n",
+    "\n",
+    "\n",
+    "device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
+    "torch_dtype = torch.float16 if torch.cuda.is_available() else torch.float32\n",
+    "\n",
+    "model_id = \"openai/whisper-large-v3-turbo\"\n",
+    "\n",
+    "model = AutoModelForSpeechSeq2Seq.from_pretrained(\n",
+    "    model_id, torch_dtype=torch_dtype, low_cpu_mem_usage=True, use_safetensors=True\n",
+    ").to(device)\n",
+    "\n",
+    "pipe = pipeline(\n",
+    "    \"automatic-speech-recognition\",\n",
+    "    model=model,\n",
+    "    torch_dtype=torch_dtype,\n",
+    "    device=device,\n",
+    "    return_timestamps=True\n",
+    ")\n",
+    "\n",
+    "\n",
+    "result = pipe(\"/workspaces/Video_Analyser/app_srv/temp/a6fba6eb-038e-4f4e-bcb7-f41d87ee1422/audio.mp3.mp3\")\n",
+    "\n",
+    "result"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "980e9742",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install librosa"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a66eabd3",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "! pip install -U openai-whisper"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e6d0e5fd",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import torch\n",
+    "from transformers import pipeline\n",
+    "from typing import Dict, Union\n",
+    "\n",
+    "def transcribe_with_timestamps_optimized(\n",
+    "    audio_path: str,\n",
+    "    model_name: str = \"openai/whisper-small\",\n",
+    "    language: str = \"en\",\n",
+    "    chunk_length_s: int = 5,\n",
+    "    stride_length_s: Union[int, tuple] = (2, 2)\n",
+    ") -> Dict[float, str]:\n",
+    "    device = \"cuda:0\" if torch.cuda.is_available() else \"cpu\"\n",
+    "    print(f\"Используемое устройство: {device}\")\n",
+    "\n",
+    "    try:\n",
+    "        transcriber = pipeline(\n",
+    "            \"automatic-speech-recognition\",\n",
+    "            model=model_name,\n",
+    "            chunk_length_s=chunk_length_s,\n",
+    "            stride_length_s=stride_length_s,\n",
+    "            device=device,\n",
+    "        )\n",
+    "    except Exception as e:\n",
+    "        print(f\"Ошибка при инициализации pipeline: {e}\")\n",
+    "        print(\"Убедитесь, что модель установлена или доступна на Hugging Face Hub.\")\n",
+    "        raise\n",
+    "\n",
+    "    print(f\"Начало транскрипции файла: {audio_path}\")\n",
+    "    try:\n",
+    "        result = transcriber(\n",
+    "            audio_path,\n",
+    "            return_timestamps=\"True\",\n",
+    "            generate_kwargs={\"language\": language} if language else {}\n",
+    "        )\n",
+    "    except Exception as e:\n",
+    "        print(f\"Ошибка при транскрипции аудиофайла: {e}\")\n",
+    "        return {}\n",
+    "\n",
+    "    transcribed_segments = {}\n",
+    "    if \"chunks\" in result and result[\"chunks\"]:\n",
+    "        for chunk in result[\"chunks\"]:\n",
+    "            start_time = chunk[\"timestamp\"][0] if chunk[\"timestamp\"][0] is not None else 0.0\n",
+    "            text = chunk[\"text\"].strip()\n",
+    "            transcribed_segments[float(start_time)] = text\n",
+    "    else:\n",
+    "        if \"text\" in result:\n",
+    "            transcribed_segments[0.0] = result[\"text\"].strip()\n",
+    "            print(\"Предупреждение: получена только общая транскрипция, без посегментных временных меток.\")\n",
+    "            print(\"Убедитесь, что 'return_timestamps=\\\"True\\\"' или 'return_timestamps=\\\"word\\\"' используется.\")\n",
+    "        else:\n",
+    "            print(\"Не удалось получить транскрипцию или временные метки.\")\n",
+    "\n",
+    "    print(\"Транскрипция завершена.\")\n",
+    "    return transcribed_segments\n",
+    "\n",
+    "\n",
+    "# Пример использования\n",
+    "if __name__ == \"__main__\":\n",
+    "    \n",
+    "    result = transcribe_with_timestamps_optimized(\n",
+    "            audio_path=\"/workspaces/Video_Analyser/app_srv/temp/a6fba6eb-038e-4f4e-bcb7-f41d87ee1422/audio.mp3.mp3\",\n",
+    "        )\n",
+    "\n",
+    "    print(result)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "ca9a4832",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from transformers import pipeline\n",
+    "import librosa\n",
+    "\n",
+    "def transcribe_with_pipeline(audio_path):\n",
+    "    pipe = pipeline(\n",
+    "        \"automatic-speech-recognition\",\n",
+    "        model=\"openai/whisper-small\",\n",
+    "        chunk_length_s=30,  # разбивает на чанки по 30 секунд\n",
+    "        stride_length_s=2,   # перекрытие между чанками\n",
+    "    )\n",
+    "    \n",
+    "    result = pipe(audio_path, return_timestamps=True)\n",
+    "    return result['text']\n",
+    "\n",
+    "result = transcribe_with_pipeline(\"/workspaces/Video_Analyser/app_srv/temp/a6fba6eb-038e-4f4e-bcb7-f41d87ee1422/audio.mp3.mp3\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "7cd4e28e",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "result"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

app_srv/video_processing.py

@@ -0,0 +1,157 @@
+import os
+import av
+import torch
+from pathlib import Path
+from typing import Dict
+from tqdm import tqdm
+
+from downloader import download_youtube_video
+from model_api import initialize_model, initialize_processor, prepare_model_inputs, generate_description, process_model_output, clear_gpu_cache, create_prompt_message, get_device_and_dtype
+
+def extract_frames_with_timestamps(
+    video_path: str,
+    output_dir: str,
+    time_step: float = 1.0,
+    quality: int = 95,
+    frame_prefix: str = "frame",
+    use_hw_accel: bool = True,
+    hw_device: str = "cuda"
+) -> Dict[str, str]:
+    """
+    Extracts frames from video with NVIDIA hardware acceleration (NVDEC/CUDA).
+    
+    Args:
+        video_path: Path to the video file
+        output_dir: Directory to save frames
+        time_step: Interval between frames (seconds)
+        quality: JPEG quality (1-100)
+        frame_prefix: Prefix for saved frames
+        use_hw_accel: Enable NVIDIA hardware decoding
+        hw_device: GPU device (e.g., 'cuda:0')
+
+    Returns:
+        Dict of {timestamp: frame_path}
+    """
+    result = {}
+    try:
+        video_path = Path(video_path).absolute()
+        output_dir = Path(output_dir).absolute()
+        
+        if not video_path.exists():
+            raise ValueError(f"Video file not found: {video_path}")
+
+        frames_dir = output_dir / "frames"
+        frames_dir.mkdir(parents=True, exist_ok=True)
+
+        # Configure hardware acceleration
+        options = {}
+        if use_hw_accel:
+            options.update({
+                'hwaccel': 'cuda',
+                'hwaccel_device': hw_device,
+                'hwaccel_output_format': 'cuda'  # Keep frames in GPU memory
+            })
+
+        # Open video with hardware acceleration
+        container = av.open(str(video_path), options=options)
+        video_stream = next(s for s in container.streams if s.type == 'video')
+
+        fps = float(video_stream.average_rate)
+        if fps <= 0:
+            raise RuntimeError("Invalid frame rate")
+
+        frame_interval = max(1, int(round(fps * time_step)))
+        frame_count = 0
+
+        for frame in container.decode(video_stream):
+            if frame_count % frame_interval == 0:
+                current_time = float(frame.pts * video_stream.time_base)
+                hh = int(current_time // 3600)
+                mm = int((current_time % 3600) // 60)
+                ss = current_time % 60
+                
+                timestamp = f"{hh:02d}:{mm:02d}:{ss:06.3f}"
+                safe_timestamp = timestamp.replace(':', '_').replace('.', '_')
+                frame_path = frames_dir / f"{frame_prefix}_{safe_timestamp}.jpg"
+
+                # Convert GPU frame to CPU if needed
+                if hasattr(frame, 'to_ndarray'):  # CUDA frame
+                    img = frame.to_ndarray(format='rgb24')
+                    img = av.VideoFrame.from_ndarray(img, format='rgb24')
+                else:
+                    img = frame
+
+                img.to_image().save(str(frame_path), quality=quality)
+                result[timestamp] = str(frame_path)
+
+            frame_count += 1
+
+        return result
+
+    except Exception as e:
+        for path in result.values():
+            try: os.remove(path)
+            except: pass
+        raise RuntimeError(f"Frame extraction failed: {str(e)}")
+
+
+def generate_frame_descriptions(frames_dict: Dict, custom_prompt: str = None, device: str = "cuda", torch_dtype: torch.dtype = torch.float16):
+    """
+    Generate descriptions for video frames with progress tracking
+    
+    Args:
+        frames_dict (dict): Dictionary of {timestamp: image_path} pairs
+        custom_prompt (str, optional): Custom prompt to use for all frames.
+                                      Can include {timestamp} placeholder.
+    
+    Returns:
+        dict: Dictionary of {timestamp: description} pairs
+    """
+
+    # Instantiating model components
+    model = initialize_model(device, torch_dtype)
+    processor = initialize_processor()
+    descriptions = {}
+
+    with tqdm(
+        frames_dict.items(),
+        total=len(frames_dict),
+        desc="Processing frames",
+        unit="frame"
+    ) as progress_bar:
+        
+        for timestamp, image_path in progress_bar:
+            try:
+                progress_bar.set_postfix({"current": timestamp})
+                
+                # Prepare model input with custom prompt
+                messages = create_prompt_message(image_path, timestamp, custom_prompt)
+                inputs = prepare_model_inputs(processor, messages, device)
+                
+                # Generate and process output
+                generated_ids = generate_description(model, inputs)
+                output_text = process_model_output(processor, inputs, generated_ids)
+                descriptions[timestamp] = output_text
+                
+                # Memory cleanup
+                del inputs, generated_ids
+                clear_gpu_cache()
+                
+            except Exception as e:
+                print(f"\nError processing frame {timestamp}: {str(e)}")
+                descriptions[timestamp] = f"Description generation error: {str(e)}"
+                clear_gpu_cache()
+
+    # Final cleanup
+    del model, processor
+    clear_gpu_cache()
+    
+    return descriptions
+
+
+
+if __name__ == "__main__":
+    video_url = "https://www.youtube.com/watch?v=L1vXCYZAYYM"
+    video_data = download_youtube_video(video_url)
+    frames = extract_frames_with_timestamps(video_path=video_data['video_path'], output_dir=video_data['data_path'], time_step=10)
+    print(frames)

packages.txt

@@ -0,0 +1,16 @@
+ffmpeg
+git
+git-lfs
+htop
+iotop
+libxml2
+libopenblas-dev
+libssl-dev
+python3-pip
+python3-wheel
+python3-setuptools
+python-is-python3
+wget
+zlib1g
+net-tools
+curl

requirements.txt

@@ -0,0 +1,22 @@
+accelerate==1.7.0
+autoawq==0.2.9
+av==14.4.0
+datasets==3.6.0
+evaluate==0.4.3
+ffmpeg-python==0.2.0
+gradio==5.33.0
+gradio[mcp]
+gradio_client==1.10.2
+intel_extension_for_pytorch==2.7.0
+ipykernel==6.29.5
+ipython==9.3.0
+ipywidgets==8.1.7
+librosa==0.11.0
+qwen-vl-utils==0.0.11
+tqdm==4.67.1
+transformers==4.52.4
+yt-dlp==2025.5.22
+smolagents
+smolagents[mcp]
+mcp
+fastmcp