feat: Add comprehensive HF Jobs monitoring system

Implemented full job monitoring capabilities for HuggingFace Jobs:

**New Features:**
- Job Monitoring screen with 3 tabs (Inspect Job, Recent Jobs, Guide)
- Real-time job status inspection with color-coded indicators
- Job logs viewing directly in the UI
- Recent jobs list with pagination
- HF token validation and helpful error messages

**Technical Implementation:**
- Added screens/job_monitoring.py - Complete job monitoring UI
- Enhanced utils/hf_jobs_submission.py with 3 new API functions:
- check_job_status() - Inspect job details via HF API
- get_job_logs() - Fetch job logs
- list_user_jobs() - List recent user jobs
- Updated app.py - Integrated job monitoring into navigation system

**API Integration:**
- Uses official HuggingFace Hub API (inspect_job, fetch_job_logs, list_jobs)
- Proper handling of JobInfo and JobStatus objects
- Supports both username/job_hash and job_hash formats

**UI Improvements:**
- Fixed job ID display (now shows actual HF Job ID: username/hash)
- Status emojis for all job states (QUEUED, RUNNING, SUCCEEDED, CANCELED, etc.)
- Clickable job URLs to HF dashboard
- Hardware flavor display (cpu-basic, a10g-small, etc.)
- Comprehensive troubleshooting guides

**Bug Fixes:**
- Fixed JobStatus enum handling (status.stage vs status.upper())
- Removed non-existent timing fields (started_at, finished_at, runtime)
- Added all status variants (CANCELED/CANCELLED, COMPLETED/SUCCEEDED, ERROR/FAILED)
- Proper token checking before API calls

app.py

@@ -61,6 +61,7 @@ from screens.chat import (
 )
 from screens.documentation import create_documentation_screen
 from screens.settings import create_settings_screen
+from screens.job_monitoring import create_job_monitoring_screen
 from screens.mcp_helpers import (
     call_analyze_leaderboard_sync,
     call_debug_trace_sync,
@@ -1593,6 +1594,7 @@ with gr.Blocks(title="TraceMind-AI", theme=theme) as app:
             new_eval_nav_btn = gr.Button("▶️ New Evaluation", variant="secondary", size="lg")
             compare_nav_btn = gr.Button("⚖️ Compare", variant="secondary", size="lg")
             chat_nav_btn = gr.Button("🤖 Agent Chat", variant="secondary", size="lg")
+            job_monitoring_nav_btn = gr.Button("🔍 Job Monitoring", variant="secondary", size="lg")
             synthetic_data_nav_btn = gr.Button("🔬 Synthetic Data", variant="secondary", size="lg")
             docs_nav_btn = gr.Button("📚 Documentation", variant="secondary", size="lg")
             settings_nav_btn = gr.Button("⚙️ Settings", variant="secondary", size="lg")
@@ -2451,6 +2453,11 @@ with gr.Blocks(title="TraceMind-AI", theme=theme) as app:
         # ============================================================================
         settings_screen = create_settings_screen()
 
+        # ============================================================================
+        # Screen 11: Job Monitoring
+        # ============================================================================
+        job_monitoring_screen = create_job_monitoring_screen()
+
         # ============================================================================
         # Evaluation Helper Functions
         # ============================================================================
@@ -2689,6 +2696,7 @@ No historical data available for **{model}**.
 
             # Success - build success message
             job_id = result.get('job_id', 'unknown')
+            hf_job_id = result.get('hf_job_id', job_id)  # Get actual HF job ID
             job_platform = result.get('platform', infra_provider)
             job_hardware = result.get('hardware', hardware)
             job_status = result.get('status', 'submitted')
@@ -2760,8 +2768,11 @@ No historical data available for **{model}**.
                 <h2 style="margin-top: 0;">✅ Evaluation Job Configured!</h2>
 
                 <div style="background: rgba(255,255,255,0.15); padding: 15px; border-radius: 5px; margin: 15px 0;">
-                    <div style="font-size: 0.9em; opacity: 0.9; margin-bottom: 5px;">Job ID</div>
-                    <div style="font-family: monospace; font-size: 1.1em; font-weight: bold;">{job_id}</div>
+                    <div style="font-size: 0.9em; opacity: 0.9; margin-bottom: 5px;">Run ID (SMOLTRACE)</div>
+                    <div style="font-family: monospace; font-size: 0.95em; font-weight: bold;">{job_id}</div>
+                    <div style="font-size: 0.9em; opacity: 0.9; margin-top: 10px; margin-bottom: 5px;">HF Job ID</div>
+                    <div style="font-family: monospace; font-size: 0.95em; font-weight: bold;">{hf_job_id}</div>
+                    <div style="font-size: 0.8em; opacity: 0.8; margin-top: 8px;">Use this ID to monitor: <code style="background: rgba(0,0,0,0.2); padding: 2px 6px; border-radius: 3px;">hf jobs inspect {hf_job_id}</code></div>
                 </div>
 
                 <div style="display: grid; grid-template-columns: 1fr 1fr 1fr; gap: 10px; margin-top: 15px;">
@@ -2884,11 +2895,13 @@ No historical data available for **{model}**.
                 new_evaluation_screen: gr.update(visible=False),
                 documentation_screen: gr.update(visible=False),
                 settings_screen: gr.update(visible=False),
+                job_monitoring_screen: gr.update(visible=False),
                 dashboard_nav_btn: gr.update(variant="primary"),
                 leaderboard_nav_btn: gr.update(variant="secondary"),
                 new_eval_nav_btn: gr.update(variant="secondary"),
                 compare_nav_btn: gr.update(variant="secondary"),
                 chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                 docs_nav_btn: gr.update(variant="secondary"),
                 settings_nav_btn: gr.update(variant="secondary"),
@@ -2909,11 +2922,13 @@ No historical data available for **{model}**.
                 new_evaluation_screen: gr.update(visible=False),
                 documentation_screen: gr.update(visible=False),
                 settings_screen: gr.update(visible=False),
+                job_monitoring_screen: gr.update(visible=False),
                 dashboard_nav_btn: gr.update(variant="secondary"),
                 leaderboard_nav_btn: gr.update(variant="primary"),
                 new_eval_nav_btn: gr.update(variant="secondary"),
                 compare_nav_btn: gr.update(variant="secondary"),
                 chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                 docs_nav_btn: gr.update(variant="secondary"),
                 settings_nav_btn: gr.update(variant="secondary"),
@@ -2932,11 +2947,13 @@ No historical data available for **{model}**.
                 new_evaluation_screen: gr.update(visible=True),
                 documentation_screen: gr.update(visible=False),
                 settings_screen: gr.update(visible=False),
+                job_monitoring_screen: gr.update(visible=False),
                 dashboard_nav_btn: gr.update(variant="secondary"),
                 leaderboard_nav_btn: gr.update(variant="secondary"),
                 new_eval_nav_btn: gr.update(variant="primary"),
                 compare_nav_btn: gr.update(variant="secondary"),
                 chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                 docs_nav_btn: gr.update(variant="secondary"),
                 settings_nav_btn: gr.update(variant="secondary"),
@@ -2967,11 +2984,13 @@ No historical data available for **{model}**.
                     new_evaluation_screen: gr.update(visible=False),
                     documentation_screen: gr.update(visible=False),
                 settings_screen: gr.update(visible=False),
+                    job_monitoring_screen: gr.update(visible=False),
                     dashboard_nav_btn: gr.update(variant="secondary"),
                     leaderboard_nav_btn: gr.update(variant="secondary"),
                     new_eval_nav_btn: gr.update(variant="secondary"),
                     compare_nav_btn: gr.update(variant="primary"),
                     chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                     docs_nav_btn: gr.update(variant="secondary"),
                     settings_nav_btn: gr.update(variant="secondary"),
@@ -2991,11 +3010,13 @@ No historical data available for **{model}**.
                     new_evaluation_screen: gr.update(visible=False),
                     documentation_screen: gr.update(visible=False),
                     settings_screen: gr.update(visible=False),
+                    job_monitoring_screen: gr.update(visible=False),
                     dashboard_nav_btn: gr.update(variant="secondary"),
                     leaderboard_nav_btn: gr.update(variant="secondary"),
                     new_eval_nav_btn: gr.update(variant="secondary"),
                     compare_nav_btn: gr.update(variant="primary"),
                     chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                     docs_nav_btn: gr.update(variant="secondary"),
                 settings_nav_btn: gr.update(variant="secondary"),
@@ -3014,11 +3035,13 @@ No historical data available for **{model}**.
                 new_evaluation_screen: gr.update(visible=False),
                 documentation_screen: gr.update(visible=False),
                 settings_screen: gr.update(visible=False),
+                job_monitoring_screen: gr.update(visible=False),
                 dashboard_nav_btn: gr.update(variant="secondary"),
                 leaderboard_nav_btn: gr.update(variant="secondary"),
                 new_eval_nav_btn: gr.update(variant="secondary"),
                 compare_nav_btn: gr.update(variant="secondary"),
                 chat_nav_btn: gr.update(variant="primary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                 docs_nav_btn: gr.update(variant="secondary"),
                 settings_nav_btn: gr.update(variant="secondary"),
@@ -3037,11 +3060,13 @@ No historical data available for **{model}**.
                 new_evaluation_screen: gr.update(visible=False),
                 documentation_screen: gr.update(visible=False),
                 settings_screen: gr.update(visible=False),
+                job_monitoring_screen: gr.update(visible=False),
                 dashboard_nav_btn: gr.update(variant="secondary"),
                 leaderboard_nav_btn: gr.update(variant="secondary"),
                 new_eval_nav_btn: gr.update(variant="secondary"),
                 compare_nav_btn: gr.update(variant="secondary"),
                 chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="primary"),
                 docs_nav_btn: gr.update(variant="secondary"),
                 settings_nav_btn: gr.update(variant="secondary"),
@@ -3060,11 +3085,13 @@ No historical data available for **{model}**.
                 new_evaluation_screen: gr.update(visible=False),
                 documentation_screen: gr.update(visible=True),
                 settings_screen: gr.update(visible=False),
+                job_monitoring_screen: gr.update(visible=False),
                 dashboard_nav_btn: gr.update(variant="secondary"),
                 leaderboard_nav_btn: gr.update(variant="secondary"),
                 new_eval_nav_btn: gr.update(variant="secondary"),
                 compare_nav_btn: gr.update(variant="secondary"),
                 chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                 docs_nav_btn: gr.update(variant="primary"),
                 settings_nav_btn: gr.update(variant="secondary"),
@@ -3083,16 +3110,43 @@ No historical data available for **{model}**.
                 new_evaluation_screen: gr.update(visible=False),
                 documentation_screen: gr.update(visible=False),
                 settings_screen: gr.update(visible=True),
+                job_monitoring_screen: gr.update(visible=False),
                 dashboard_nav_btn: gr.update(variant="secondary"),
                 leaderboard_nav_btn: gr.update(variant="secondary"),
                 new_eval_nav_btn: gr.update(variant="secondary"),
                 compare_nav_btn: gr.update(variant="secondary"),
                 chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="secondary"),
                 synthetic_data_nav_btn: gr.update(variant="secondary"),
                 docs_nav_btn: gr.update(variant="secondary"),
                 settings_nav_btn: gr.update(variant="primary"),
             }
 
+        def navigate_to_job_monitoring():
+            """Navigate to job monitoring screen"""
+            return {
+                dashboard_screen: gr.update(visible=False),
+                leaderboard_screen: gr.update(visible=False),
+                run_detail_screen: gr.update(visible=False),
+                trace_detail_screen: gr.update(visible=False),
+                compare_screen: gr.update(visible=False),
+                chat_screen: gr.update(visible=False),
+                synthetic_data_screen: gr.update(visible=False),
+                new_evaluation_screen: gr.update(visible=False),
+                documentation_screen: gr.update(visible=False),
+                settings_screen: gr.update(visible=False),
+                job_monitoring_screen: gr.update(visible=True),
+                dashboard_nav_btn: gr.update(variant="secondary"),
+                leaderboard_nav_btn: gr.update(variant="secondary"),
+                new_eval_nav_btn: gr.update(variant="secondary"),
+                compare_nav_btn: gr.update(variant="secondary"),
+                chat_nav_btn: gr.update(variant="secondary"),
+                job_monitoring_nav_btn: gr.update(variant="primary"),
+                synthetic_data_nav_btn: gr.update(variant="secondary"),
+                docs_nav_btn: gr.update(variant="secondary"),
+                settings_nav_btn: gr.update(variant="secondary"),
+            }
+
         # Synthetic Data Generator Callbacks
         def on_generate_synthetic_data(domain, tools, num_tasks, difficulty, agent_type):
             """Generate synthetic dataset AND prompt template using MCP server"""
@@ -3381,8 +3435,8 @@ Result: {result}
             fn=navigate_to_dashboard,
             outputs=[
                 dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
-                new_evaluation_screen, documentation_screen, settings_screen,
-                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ] + list(dashboard_components.values())
         )
 
@@ -3493,24 +3547,24 @@ Result: {result}
             fn=navigate_to_dashboard,
             outputs=[
                 dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
-                new_evaluation_screen, documentation_screen, settings_screen,
-                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ] + list(dashboard_components.values())
         )
 
         leaderboard_nav_btn.click(
             fn=navigate_to_leaderboard,
             outputs=[
-                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen,
-                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
 
         new_eval_nav_btn.click(
             fn=navigate_to_new_evaluation,
             outputs=[
-                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen,
-                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
 
@@ -3518,8 +3572,8 @@ Result: {result}
             fn=navigate_to_compare,
             outputs=[
                 dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
-                new_evaluation_screen, documentation_screen, settings_screen,
-                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn,
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn,
                 compare_components['compare_run_a_dropdown'], compare_components['compare_run_b_dropdown']
             ]
         )
@@ -3528,16 +3582,25 @@ Result: {result}
             fn=navigate_to_chat,
             outputs=[
                 dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
-                new_evaluation_screen, documentation_screen, settings_screen,
-                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
         synthetic_data_nav_btn.click(
             fn=navigate_to_synthetic_data,
             outputs=[
                 dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
-                new_evaluation_screen, documentation_screen, settings_screen,
-                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
+            ]
+        )
+
+        job_monitoring_nav_btn.click(
+            fn=navigate_to_job_monitoring,
+            outputs=[
+                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
 
@@ -3545,8 +3608,8 @@ Result: {result}
             fn=navigate_to_documentation,
             outputs=[
                 dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
-                new_evaluation_screen, documentation_screen, settings_screen,
-                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
 
@@ -3554,8 +3617,8 @@ Result: {result}
             fn=navigate_to_settings,
             outputs=[
                 dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen,
-                new_evaluation_screen, documentation_screen, settings_screen,
-                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
 
@@ -3576,8 +3639,8 @@ Result: {result}
         back_to_leaderboard_from_eval_btn.click(
             fn=navigate_to_leaderboard,
             outputs=[
-                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen,
-                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
 
@@ -3691,8 +3754,8 @@ Result: {result}
         compare_components['back_to_leaderboard_btn'].click(
             fn=navigate_to_leaderboard,
             outputs=[
-                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen,
-                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, docs_nav_btn, settings_nav_btn
+                dashboard_screen, leaderboard_screen, run_detail_screen, trace_detail_screen, compare_screen, chat_screen, synthetic_data_screen, new_evaluation_screen, documentation_screen, settings_screen, job_monitoring_screen,
+                dashboard_nav_btn, leaderboard_nav_btn, new_eval_nav_btn, compare_nav_btn, chat_nav_btn, synthetic_data_nav_btn, job_monitoring_nav_btn, docs_nav_btn, settings_nav_btn
             ]
         )
 

screens/job_monitoring.py

@@ -0,0 +1,442 @@
+"""
+Job Monitoring Screen for TraceMind-AI
+Allows users to monitor HuggingFace Jobs status and view logs
+"""
+
+import gradio as gr
+import os
+from typing import Optional
+
+
+def create_job_monitoring_screen():
+    """
+    Create the job monitoring screen for HF Jobs
+
+    Returns:
+        gr.Column: Gradio Column component for job monitoring
+    """
+    with gr.Column(visible=False) as job_monitoring_interface:
+        gr.Markdown("""
+        # 🔍 Job Monitoring
+
+        Monitor your HuggingFace Jobs in real-time. Check job status, view logs, and track evaluation progress.
+        """)
+
+        with gr.Tabs():
+            # Tab 1: Single Job Inspection
+            with gr.Tab("📋 Inspect Job"):
+                gr.Markdown("""
+                ### Inspect a Specific Job
+
+                Enter a HuggingFace Job ID to view its status and logs.
+                """)
+
+                with gr.Row():
+                    job_id_input = gr.Textbox(
+                        label="HF Job ID",
+                        placeholder="e.g., kshitijthakkar/691eb073748f86bfa7144fcc",
+                        info="Format: username/job_hash"
+                    )
+
+                with gr.Row():
+                    inspect_btn = gr.Button("🔍 Inspect Job", variant="primary")
+                    refresh_btn = gr.Button("🔄 Refresh", variant="secondary")
+
+                # Job Status Section
+                with gr.Accordion("📊 Job Status", open=True):
+                    job_status_display = gr.Markdown("Enter a Job ID and click 'Inspect Job' to view status")
+
+                # Job Logs Section
+                with gr.Accordion("📜 Job Logs", open=True):
+                    with gr.Row():
+                        show_logs_btn = gr.Button("📥 Load Logs", variant="secondary")
+                        auto_refresh_logs = gr.Checkbox(
+                            label="Auto-refresh logs (every 5s)",
+                            value=False
+                        )
+
+                    job_logs_display = gr.Code(
+                        label="Job Logs",
+                        language="shell",
+                        value="Click 'Load Logs' to view job output",
+                        lines=20
+                    )
+
+            # Tab 2: Recent Jobs List
+            with gr.Tab("📑 Recent Jobs"):
+                gr.Markdown("""
+                ### Your Recent Jobs
+
+                View a list of your recent HuggingFace Jobs.
+                """)
+
+                with gr.Row():
+                    list_jobs_btn = gr.Button("📋 Load Recent Jobs", variant="primary")
+                    jobs_limit = gr.Slider(
+                        minimum=5,
+                        maximum=50,
+                        value=10,
+                        step=5,
+                        label="Number of jobs to fetch"
+                    )
+
+                recent_jobs_display = gr.Markdown("Click 'Load Recent Jobs' to view your jobs")
+
+            # Tab 3: Job Monitoring Guide
+            with gr.Tab("📖 Guide"):
+                gr.Markdown("""
+                ### Using Job Monitoring
+
+                #### How to Get Your Job ID
+
+                After submitting an evaluation from the "New Evaluation" tab, you'll receive:
+                - **Run ID (SMOLTRACE)**: Used for tracking results in datasets (e.g., `job_3a22ceca`)
+                - **HF Job ID**: Used for monitoring the actual job (e.g., `kshitijthakkar/691eb073748f86bfa7144fcc`)
+
+                Use the **HF Job ID** here to monitor your job.
+
+                #### Job Status Values
+
+                - **QUEUED**: Job is waiting to start
+                - **STARTING**: Job is being initialized
+                - **RUNNING**: Job is currently executing
+                - **SUCCEEDED**: Job completed successfully
+                - **FAILED**: Job encountered an error
+                - **CANCELLED**: Job was manually cancelled
+                - **STOPPED**: Job was stopped by the system
+
+                #### CLI Commands Reference
+
+                You can also use the HuggingFace CLI to monitor jobs:
+
+                ```bash
+                # List your running jobs
+                hf jobs ps
+
+                # Inspect a specific job
+                hf jobs inspect <job_id>
+
+                # View logs from a job
+                hf jobs logs <job_id>
+
+                # Follow logs in real-time
+                hf jobs logs <job_id> --follow
+
+                # Cancel a job
+                hf jobs cancel <job_id>
+                ```
+
+                #### Tips
+
+                - 💡 **Bookmark your Job ID** after submission for easy access
+                - 🔄 **Use auto-refresh** for logs when job is running
+                - 📊 **Check status regularly** to catch any issues early
+                - 📝 **Review logs** if your job fails to understand what went wrong
+                - 🎯 **Results appear in leaderboard** once job succeeds and uploads datasets
+                """)
+
+        # Functions for job monitoring
+        def inspect_job(job_id: str):
+            """Inspect a specific job's status"""
+            import os
+
+            if not job_id or not job_id.strip():
+                return gr.update(value="❌ Please enter a Job ID")
+
+            # Check if token is configured before making API call
+            token = os.environ.get("HF_TOKEN")
+            if not token or not token.strip():
+                return gr.update(
+                    value="""
+### ⚠️ HuggingFace Token Not Configured
+
+**Action Required**:
+1. Go to "⚙️ Settings" in the sidebar
+2. Enter your HuggingFace token (must have "Run Jobs" permission)
+3. Click "💾 Save API Keys"
+4. Return to this tab and try again
+                    """
+                )
+
+            from utils.hf_jobs_submission import check_job_status
+
+            result = check_job_status(job_id.strip())
+
+            if not result.get("success"):
+                error_msg = result.get('error', 'Unknown error')
+
+                return gr.update(
+                    value=f"""
+### ❌ Failed to Fetch Job Status
+
+**Error**: {error_msg}
+
+**Job ID**: `{job_id}`
+
+**Troubleshooting**:
+- Verify the Job ID format is correct (format: `username/job_hash`)
+- Check that the job exists in your account
+- Ensure your HF token has the correct permissions
+- Token must have **Run Jobs** permission enabled
+                    """
+                )
+
+            # Format status with emoji
+            status = result.get("status", "unknown")
+            # Convert status to string if it's an enum
+            status_str = str(status).upper() if status else "UNKNOWN"
+
+            status_emoji = {
+                "QUEUED": "⏳",
+                "STARTING": "🔄",
+                "RUNNING": "▶️",
+                "SUCCEEDED": "✅",
+                "COMPLETED": "✅",  # Alternative success status
+                "FAILED": "❌",
+                "ERROR": "❌",  # Alternative failure status
+                "CANCELLED": "🚫",
+                "CANCELED": "🚫",  # US spelling variant
+                "STOPPED": "⏹️",
+                "TIMEOUT": "⏱️"
+            }.get(status_str, "❓")
+
+            status_color = {
+                "QUEUED": "#FFA500",
+                "STARTING": "#1E90FF",
+                "RUNNING": "#00CED1",
+                "SUCCEEDED": "#32CD32",
+                "COMPLETED": "#32CD32",  # Alternative success status
+                "FAILED": "#DC143C",
+                "ERROR": "#DC143C",  # Alternative failure status
+                "CANCELLED": "#696969",
+                "CANCELED": "#696969",  # US spelling variant
+                "STOPPED": "#A9A9A9",
+                "TIMEOUT": "#FF8C00"
+            }.get(status_str, "#888888")
+
+            created_at = result.get("created_at", "N/A")
+            flavor = result.get("flavor", "N/A")
+            job_url = result.get("url", None)
+
+            # Format job URL as clickable link
+            job_url_display = f"[Open in HuggingFace]({job_url})" if job_url else "N/A"
+
+            return gr.update(
+                value=f"""
+### {status_emoji} Job Status: <span style="color: {status_color};">{status_str}</span>
+
+**Job ID**: `{job_id}`
+
+#### Details
+
+- **Created**: {created_at}
+- **Hardware**: {flavor}
+- **Job URL**: {job_url_display}
+
+#### Next Steps
+
+{_get_next_steps(status_str)}
+
+---
+
+💡 **Tip**: Use "📥 Load Logs" button below to view detailed execution logs and check progress.
+                """
+            )
+
+        def _get_next_steps(status: str) -> str:
+            """Get next steps based on job status"""
+            status_upper = str(status).upper() if status else "UNKNOWN"
+
+            if status_upper == "QUEUED":
+                return "⏳ Your job is waiting in the queue. It will start soon."
+            elif status_upper == "STARTING":
+                return "🔄 Your job is being initialized. This usually takes 1-2 minutes."
+            elif status_upper == "RUNNING":
+                return "▶️ Your job is running! Click 'Load Logs' below to view progress."
+            elif status_upper in ["SUCCEEDED", "COMPLETED"]:
+                return "✅ Your job completed successfully! Check the Leaderboard tab for results."
+            elif status_upper in ["FAILED", "ERROR"]:
+                return "❌ Your job failed. Click 'Load Logs' below to see what went wrong."
+            elif status_upper in ["CANCELLED", "CANCELED", "STOPPED"]:
+                return "🚫 Your job was stopped. You can submit a new job from the 'New Evaluation' tab."
+            elif status_upper == "TIMEOUT":
+                return "⏱️ Your job exceeded the time limit. Consider optimizing your model or increasing the timeout."
+            else:
+                return "❓ Unknown status. Try refreshing or check the HF Jobs dashboard."
+
+        def load_job_logs(job_id: str):
+            """Load logs for a specific job"""
+            import os
+
+            if not job_id or not job_id.strip():
+                return gr.update(value="❌ Please enter a Job ID first")
+
+            # Check if token is configured before making API call
+            token = os.environ.get("HF_TOKEN")
+            if not token or not token.strip():
+                return gr.update(
+                    value="⚠️ HuggingFace Token Not Configured\n\nPlease configure your HF token in Settings first."
+                )
+
+            from utils.hf_jobs_submission import get_job_logs
+
+            result = get_job_logs(job_id.strip())
+
+            if not result.get("success"):
+                return gr.update(
+                    value=f"❌ Failed to fetch logs: {result.get('error', 'Unknown error')}\n\nEnsure your HF token has 'Run Jobs' permission."
+                )
+
+            logs = result.get("logs", "")
+            if not logs or not logs.strip():
+                return gr.update(value="ℹ️ No logs available yet. Job may not have started.\n\nTry refreshing after a minute.")
+
+            return gr.update(value=logs)
+
+        def list_recent_jobs(limit: int):
+            """List user's recent jobs"""
+            import os
+            from utils.hf_jobs_submission import list_user_jobs
+
+            # Check if token is configured before making API call
+            token = os.environ.get("HF_TOKEN")
+            if not token or not token.strip():
+                return gr.update(
+                    value="""
+### ⚠️ HuggingFace Token Not Configured
+
+**Action Required**:
+1. Go to "⚙️ Settings" in the sidebar
+2. Enter your HuggingFace token (must have "Run Jobs" permission)
+3. Click "💾 Save API Keys"
+4. Return to this tab and try again
+
+**Note**: Your HF token must:
+- Start with `hf_`
+- Have **Read**, **Write**, AND **Run Jobs** permissions
+- Be from a HuggingFace Pro account ($9/month)
+
+Get your token at: https://huggingface.co/settings/tokens
+                    """
+                )
+
+            result = list_user_jobs(limit=int(limit))
+
+            if not result.get("success"):
+                error_msg = result.get('error', 'Unknown error')
+
+                # Check for common error patterns
+                if "invalid" in error_msg.lower() or "token" in error_msg.lower():
+                    troubleshooting = """
+**Troubleshooting**:
+- ⚠️ **Token may be invalid** - Regenerate your token at HuggingFace settings
+- ✅ Ensure token has **Run Jobs** permission (not just Read/Write)
+- ✅ Verify you have an active **HuggingFace Pro account**
+- ✅ Token should start with `hf_`
+                    """
+                else:
+                    troubleshooting = """
+**Troubleshooting**:
+- Refresh this page and try again
+- Check your internet connection
+- Verify HuggingFace services are operational
+                    """
+
+                return gr.update(
+                    value=f"""
+### ❌ Failed to Fetch Jobs
+
+**Error**: {error_msg}
+
+{troubleshooting}
+                    """
+                )
+
+            jobs = result.get("jobs", [])
+            if not jobs:
+                return gr.update(
+                    value="""
+### ℹ️ No Jobs Found
+
+You haven't submitted any jobs yet.
+
+**Get Started**:
+1. Go to the "New Evaluation" tab
+2. Configure your model and settings
+3. Submit an evaluation job
+4. Come back here to monitor progress!
+                    """
+                )
+
+            # Build jobs table
+            jobs_table = "### 📋 Your Recent Jobs\n\n"
+            jobs_table += "| Job ID | Status | Created At |\n"
+            jobs_table += "|--------|--------|------------|\n"
+
+            for job in jobs:
+                job_id = job.get("job_id", "N/A")
+                status = job.get("status", "unknown")
+                created = job.get("created_at", "N/A")
+
+                # Convert status to string if it's an enum
+                status_str = str(status).upper() if status else "UNKNOWN"
+
+                status_emoji = {
+                    "QUEUED": "⏳",
+                    "STARTING": "🔄",
+                    "RUNNING": "▶️",
+                    "SUCCEEDED": "✅",
+                    "COMPLETED": "✅",  # Alternative success status
+                    "FAILED": "❌",
+                    "ERROR": "❌",  # Alternative failure status
+                    "CANCELLED": "🚫",
+                    "CANCELED": "🚫",  # US spelling variant
+                    "STOPPED": "⏹️",
+                    "TIMEOUT": "⏱️"
+                }.get(status_str, "❓")
+
+                jobs_table += f"| `{job_id}` | {status_emoji} {status} | {created} |\n"
+
+            jobs_table += f"\n**Total Jobs**: {len(jobs)}\n\n"
+            jobs_table += "💡 **Tip**: Copy a Job ID and paste it in the 'Inspect Job' tab to view details and logs."
+
+            return gr.update(value=jobs_table)
+
+        # Wire up button events
+        inspect_btn.click(
+            fn=inspect_job,
+            inputs=[job_id_input],
+            outputs=[job_status_display]
+        )
+
+        refresh_btn.click(
+            fn=inspect_job,
+            inputs=[job_id_input],
+            outputs=[job_status_display]
+        )
+
+        show_logs_btn.click(
+            fn=load_job_logs,
+            inputs=[job_id_input],
+            outputs=[job_logs_display]
+        )
+
+        list_jobs_btn.click(
+            fn=list_recent_jobs,
+            inputs=[jobs_limit],
+            outputs=[recent_jobs_display]
+        )
+
+        # Auto-refresh functionality (handled by Gradio's auto-update)
+        # Note: For production, consider using gr.Timer or similar for automatic refreshes
+
+        return job_monitoring_interface
+
+
+if __name__ == "__main__":
+    # For standalone testing
+    with gr.Blocks() as demo:
+        job_monitoring = create_job_monitoring_screen()
+        # Make it visible for standalone testing
+        job_monitoring.visible = True
+    demo.launch()

utils/hf_jobs_submission.py

@@ -245,20 +245,191 @@ def _auto_select_hf_hardware(provider: str, model: str) -> str:
         return "t4-small"
 
 
-def check_job_status(job_id: str, hf_token: Optional[str] = None) -> Dict:
+def check_job_status(hf_job_id: str, hf_token: Optional[str] = None) -> Dict:
     """
-    Check the status of a HuggingFace Job
+    Check the status of a HuggingFace Job using the Jobs API
 
     Args:
-        job_id: Job ID to check
+        hf_job_id: HF Job ID (format: username/job_hash or just job_hash)
         hf_token: HuggingFace token (optional, uses env if not provided)
 
     Returns:
         dict: Job status information
     """
-    # Placeholder for when HF Jobs API becomes available
-    return {
-        "job_id": job_id,
-        "status": "unknown",
-        "message": "HuggingFace Jobs status API not yet available programmatically"
-    }
+    try:
+        from huggingface_hub import HfApi
+    except ImportError:
+        return {
+            "success": False,
+            "error": "huggingface_hub package not installed",
+            "job_id": hf_job_id
+        }
+
+    token = hf_token or os.environ.get("HF_TOKEN")
+    if not token:
+        return {
+            "success": False,
+            "error": "HuggingFace token not configured",
+            "job_id": hf_job_id
+        }
+
+    try:
+        api = HfApi(token=token)
+
+        # Parse job_id and namespace (username)
+        # Format can be "username/job_hash" or just "job_hash"
+        if "/" in hf_job_id:
+            namespace, job_id_only = hf_job_id.split("/", 1)
+            job_info = api.inspect_job(job_id=job_id_only, namespace=namespace)
+        else:
+            job_info = api.inspect_job(job_id=hf_job_id)
+
+        # Extract status stage from JobStatus object
+        if hasattr(job_info, 'status') and hasattr(job_info.status, 'stage'):
+            status = job_info.status.stage
+        else:
+            status = str(job_info.status) if hasattr(job_info, 'status') else "unknown"
+
+        return {
+            "success": True,
+            "job_id": hf_job_id,
+            "status": status,
+            "created_at": str(job_info.created_at) if hasattr(job_info, 'created_at') else None,
+            "flavor": job_info.flavor if hasattr(job_info, 'flavor') else None,
+            "url": job_info.url if hasattr(job_info, 'url') else None,
+            "info": str(job_info)
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to fetch job status: {str(e)}",
+            "job_id": hf_job_id
+        }
+
+
+def get_job_logs(hf_job_id: str, hf_token: Optional[str] = None) -> Dict:
+    """
+    Retrieve logs from a HuggingFace Job
+
+    Args:
+        hf_job_id: HF Job ID (format: username/job_hash or just job_hash)
+        hf_token: HuggingFace token (optional, uses env if not provided)
+
+    Returns:
+        dict: Job logs information
+    """
+    try:
+        from huggingface_hub import HfApi
+    except ImportError:
+        return {
+            "success": False,
+            "error": "huggingface_hub package not installed",
+            "job_id": hf_job_id
+        }
+
+    token = hf_token or os.environ.get("HF_TOKEN")
+    if not token:
+        return {
+            "success": False,
+            "error": "HuggingFace token not configured",
+            "job_id": hf_job_id
+        }
+
+    try:
+        api = HfApi(token=token)
+
+        # Parse job_id and namespace (username)
+        # Format can be "username/job_hash" or just "job_hash"
+        if "/" in hf_job_id:
+            namespace, job_id_only = hf_job_id.split("/", 1)
+            logs_iterable = api.fetch_job_logs(job_id=job_id_only, namespace=namespace)
+        else:
+            logs_iterable = api.fetch_job_logs(job_id=hf_job_id)
+
+        # Convert iterable to string
+        logs = "\n".join(logs_iterable)
+
+        return {
+            "success": True,
+            "job_id": hf_job_id,
+            "logs": logs
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to fetch job logs: {str(e)}",
+            "job_id": hf_job_id,
+            "logs": ""
+        }
+
+
+def list_user_jobs(hf_token: Optional[str] = None, limit: int = 10) -> Dict:
+    """
+    List recent jobs for the authenticated user
+
+    Args:
+        hf_token: HuggingFace token (optional, uses env if not provided)
+        limit: Maximum number of jobs to return (applied after fetching)
+
+    Returns:
+        dict: List of user's jobs
+    """
+    try:
+        from huggingface_hub import HfApi
+    except ImportError:
+        return {
+            "success": False,
+            "error": "huggingface_hub package not installed"
+        }
+
+    token = hf_token or os.environ.get("HF_TOKEN")
+    if not token:
+        return {
+            "success": False,
+            "error": "HuggingFace token not configured"
+        }
+
+    try:
+        api = HfApi(token=token)
+        # List user's jobs (no limit parameter in API, so we fetch all and slice)
+        all_jobs = api.list_jobs()
+
+        # Limit the results
+        jobs_to_display = all_jobs[:limit] if limit > 0 else all_jobs
+
+        job_list = []
+        for job in jobs_to_display:
+            # Extract owner name from JobOwner object
+            owner_name = job.owner.name if hasattr(job, 'owner') and hasattr(job.owner, 'name') else None
+
+            # Build job_id in the format: owner/id
+            if owner_name and hasattr(job, 'id'):
+                job_id = f"{owner_name}/{job.id}"
+            elif hasattr(job, 'id'):
+                job_id = job.id
+            else:
+                job_id = "unknown"
+
+            # Extract status stage from JobStatus object
+            if hasattr(job, 'status') and hasattr(job.status, 'stage'):
+                status = job.status.stage
+            else:
+                status = str(job.status) if hasattr(job, 'status') else "unknown"
+
+            job_list.append({
+                "job_id": job_id,
+                "status": status,
+                "created_at": str(job.created_at) if hasattr(job, 'created_at') else None
+            })
+
+        return {
+            "success": True,
+            "jobs": job_list,
+            "count": len(job_list)
+        }
+    except Exception as e:
+        return {
+            "success": False,
+            "error": f"Failed to list jobs: {str(e)}",
+            "jobs": []
+        }