Upload SmolVLM final merged model

added_tokens.json

@@ -0,0 +1,5 @@
+{
+  "<end_of_utterance>": 49154,
+  "<fake_token_around_image>": 49152,
+  "<image>": 49153
+}

chat_template.jinja

@@ -0,0 +1,2 @@
+<|im_start|>{% for message in messages %}{{message['role'] | capitalize}}{% if message['content'][0]['type'] == 'image' %}{{':'}}{% else %}{{': '}}{% endif %}{% for line in message['content'] %}{% if line['type'] == 'text' %}{{line['text']}}{% elif line['type'] == 'image' %}{{ '<image>' }}{% endif %}{% endfor %}<end_of_utterance>
+{% endfor %}{% if add_generation_prompt %}{{ 'Assistant:' }}{% endif %}

config.json

@@ -0,0 +1,157 @@
+{
+  "architectures": [
+    "Idefics3ForConditionalGeneration"
+  ],
+  "image_seq_len": 81,
+  "image_token_id": 49153,
+  "model_type": "idefics3",
+  "pad_token_id": 128002,
+  "scale_factor": 3,
+  "text_config": {
+    "_flash_attn_2_enabled": true,
+    "_name_or_path": "/fsx/m4/experiments/local_experiment_dir/s3_async_temporary_checkpoint_folder/tr_324_opt_400/unwrapped_model",
+    "architectures": [
+      "VLlama3ForCausalLM"
+    ],
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "bos_token_id": 0,
+    "eos_token_id": 0,
+    "head_dim": 64,
+    "hidden_act": "silu",
+    "hidden_size": 2048,
+    "initializer_range": 0.02,
+    "intermediate_size": 8192,
+    "max_position_embeddings": 16384,
+    "mlp_bias": false,
+    "model_type": "llama",
+    "neftune_noise_alpha": 0.0,
+    "num_attention_heads": 32,
+    "num_hidden_layers": 24,
+    "num_key_value_heads": 32,
+    "pad_token_id": 2,
+    "perceiver_config": {
+      "_name_or_path": "",
+      "add_cross_attention": false,
+      "architectures": null,
+      "attention_dropout": 0.0,
+      "bad_words_ids": null,
+      "begin_suppress_tokens": null,
+      "bos_token_id": null,
+      "chunk_size_feed_forward": 0,
+      "cross_attention_hidden_size": null,
+      "decoder_start_token_id": null,
+      "diversity_penalty": 0.0,
+      "do_sample": false,
+      "early_stopping": false,
+      "encoder_no_repeat_ngram_size": 0,
+      "eos_token_id": null,
+      "exponential_decay_length_penalty": null,
+      "finetuning_task": null,
+      "forced_bos_token_id": null,
+      "forced_eos_token_id": null,
+      "hidden_act": "silu",
+      "id2label": {
+        "0": "LABEL_0",
+        "1": "LABEL_1"
+      },
+      "is_decoder": false,
+      "is_encoder_decoder": false,
+      "label2id": {
+        "LABEL_0": 0,
+        "LABEL_1": 1
+      },
+      "length_penalty": 1.0,
+      "max_length": 20,
+      "min_length": 0,
+      "model_type": "vllama3",
+      "no_repeat_ngram_size": 0,
+      "num_beam_groups": 1,
+      "num_beams": 1,
+      "num_key_value_heads": 1,
+      "num_return_sequences": 1,
+      "output_attentions": false,
+      "output_hidden_states": false,
+      "output_scores": false,
+      "pad_token_id": null,
+      "prefix": null,
+      "problem_type": null,
+      "pruned_heads": {},
+      "qk_layer_norms_perceiver": false,
+      "remove_invalid_values": false,
+      "repetition_penalty": 1.0,
+      "resampler_depth": 6,
+      "resampler_head_dim": 96,
+      "resampler_n_heads": 16,
+      "resampler_n_latents": 64,
+      "return_dict": true,
+      "return_dict_in_generate": false,
+      "sep_token_id": null,
+      "suppress_tokens": null,
+      "task_specific_params": null,
+      "temperature": 1.0,
+      "tf_legacy_loss": false,
+      "tie_encoder_decoder": false,
+      "tie_word_embeddings": true,
+      "tokenizer_class": null,
+      "top_k": 50,
+      "top_p": 1.0,
+      "torch_dtype": null,
+      "torchscript": false,
+      "transformers_version": "4.46.0",
+      "typical_p": 1.0,
+      "use_bfloat16": false
+    },
+    "pretraining_tp": 1,
+    "qk_layer_norms": false,
+    "rms_norm_eps": 1e-05,
+    "rope_scaling": null,
+    "rope_theta": 273768.0,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "use_resampler": false,
+    "vocab_size": 49155
+  },
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers.js_config": {
+    "dtype": {
+      "decoder_model_merged": "q4",
+      "embed_tokens": "auto",
+      "vision_encoder": "auto"
+    },
+    "kv_cache_dtype": {
+      "fp16": "float16",
+      "q4f16": "float16"
+    },
+    "use_external_data_format": {
+      "decoder_model_merged.onnx": true,
+      "decoder_model_merged_fp16.onnx": true
+    }
+  },
+  "transformers_version": "4.52.4",
+  "use_cache": true,
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 384,
+    "initializer_range": 0.02,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "max_image_size": {
+      "longest_edge": 384
+    },
+    "model_type": "idefics3_vision",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "size": {
+      "longest_edge": 1920
+    },
+    "tie_word_embeddings": false,
+    "torch_dtype": "bfloat16"
+  },
+  "vocab_size": 49155
+}

generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 0,
+  "eos_token_id": 49154,
+  "pad_token_id": 2,
+  "transformers_version": "4.52.4"
+}

gradio_trac_automation.py

@@ -0,0 +1,188 @@
+import gradio as gr
+import torch
+from transformers import Idefics3ForConditionalGeneration, AutoProcessor
+from PIL import Image
+
+# Global variables
+model = None
+processor = None
+device = None
+
+def get_device():
+    """Determine best device to use"""
+    if torch.cuda.is_available():
+        return 'cuda:0'
+    else:
+        return 'cpu'
+
+def load_model():
+    """Load SmolVLM model with proper device handling"""
+    global model, processor, device
+    
+    try:
+        print("Loading SmolVLM TRAC Automation Agent...")
+        
+        device = get_device()
+        print(f"Using device: {device}")
+        
+        model_path = r"C:\Users\keith\OneDrive\Desktop\admin.trac.jobs-DATA\LLaMA-Factory_local\smolvlm_final_merged"
+        
+        # Load processor first
+        processor = AutoProcessor.from_pretrained(model_path, trust_remote_code=True)
+        print("✅ Processor loaded")
+        
+        # Load model with explicit device placement
+        if device == 'cuda:0':
+            # GPU loading
+            model = Idefics3ForConditionalGeneration.from_pretrained(
+                model_path,
+                torch_dtype=torch.bfloat16,
+                device_map={'': 0},  # Force all components to GPU 0
+                trust_remote_code=True
+            )
+        else:
+            # CPU loading  
+            model = Idefics3ForConditionalGeneration.from_pretrained(
+                model_path,
+                torch_dtype=torch.float32,  # Use float32 for CPU
+                device_map='cpu',
+                trust_remote_code=True
+            )
+        
+        print(f"✅ Model loaded on {device}")
+        return f"✅ Model loaded successfully on {device}! Ready for TRAC automation."
+        
+    except Exception as e:
+        error_msg = f"❌ Error loading model: {str(e)}"
+        print(error_msg)
+        return error_msg
+
+def analyze_interface(image, task_type, custom_prompt):
+    """Analyze TRAC interface with proper device handling"""
+    global model, processor, device
+    
+    if model is None:
+        return "❌ Please load the model first."
+    
+    if image is None:
+        return "❌ Please upload a TRAC screenshot."
+    
+    try:
+        # Convert image to RGB
+        if not isinstance(image, Image.Image):
+            image = Image.fromarray(image)
+        image = image.convert("RGB")
+        
+        # Create task-specific prompts
+        if task_type == "Longlisting":
+            prompt = """<image>
+Analyze this TRAC interface for LONGLISTING candidates. Identify clickable elements, candidate tables, selection controls, and filtering options. Provide automation steps."""
+            
+        elif task_type == "Shortlisting":
+            prompt = """<image>
+Analyze this TRAC interface for SHORTLISTING candidates. Identify evaluation controls, shortlist buttons, and approval workflows. Provide automation steps."""
+            
+        elif task_type == "Interview Setup":
+            prompt = """<image>
+Analyze this TRAC interface for INTERVIEW SETUP. Identify scheduling elements, calendar controls, and interviewer assignment. Provide automation steps."""
+            
+        else:  # Custom
+            if not custom_prompt.strip():
+                return "❌ Please enter a custom prompt for analysis."
+            prompt = f"<image>\n{custom_prompt}"
+        
+        # Process inputs
+        inputs = processor(text=prompt, images=[image], return_tensors="pt")
+        
+        # Move ALL tensors to the same device as model
+        if device == 'cuda:0':
+            inputs = {k: v.to(device) if torch.is_tensor(v) else v for k, v in inputs.items()}
+        
+        # Generate response
+        with torch.no_grad():
+            outputs = model.generate(
+                **inputs,
+                max_new_tokens=250,
+                do_sample=True,
+                temperature=0.7,
+                pad_token_id=processor.tokenizer.eos_token_id if hasattr(processor, 'tokenizer') else None
+            )
+        
+        # Decode response
+        response = processor.decode(outputs[0], skip_special_tokens=True)
+        
+        # Clean up response
+        if prompt in response:
+            response = response.replace(prompt, "").strip()
+        
+        response = response.replace("<image>", "").strip()
+        
+        if not response:
+            response = "Model generated empty response. Try a different screenshot or prompt."
+            
+        return response
+        
+    except Exception as e:
+        error_msg = f"❌ Analysis Error: {str(e)}"
+        print(error_msg)
+        return error_msg
+
+def create_app():
+    """Create Gradio interface"""
+    with gr.Blocks(title="SmolVLM TRAC Automation") as demo:
+        
+        gr.Markdown("""
+        # 🎯 SmolVLM TRAC Automation Agent
+        
+        **AI Assistant for HR Administrative Tasks**
+        - 📋 Longlisting candidates
+        - ⭐ Shortlisting applications  
+        - 📅 Interview setup & scheduling
+        """)
+        
+        with gr.Row():
+            with gr.Column():
+                # Model loading
+                load_btn = gr.Button("🚀 Load Model", variant="primary")
+                status = gr.Textbox(label="Status", value="Model not loaded")
+                
+                # Image upload
+                image_input = gr.Image(label="TRAC Screenshot", type="pil")
+                
+                # Task selection
+                task_type = gr.Radio(
+                    choices=["Longlisting", "Shortlisting", "Interview Setup", "Custom"],
+                    value="Longlisting",
+                    label="Task Type"
+                )
+                
+                # Custom prompt
+                custom_prompt = gr.Textbox(
+                    label="Custom Prompt",
+                    placeholder="Describe what to analyze...",
+                    lines=3
+                )
+                
+                analyze_btn = gr.Button("🔍 Analyze", variant="primary")
+            
+            with gr.Column():
+                result = gr.Textbox(
+                    label="Automation Instructions",
+                    lines=15,
+                    show_copy_button=True
+                )
+        
+        # Event handlers
+        load_btn.click(load_model, outputs=status)
+        analyze_btn.click(
+            analyze_interface,
+            inputs=[image_input, task_type, custom_prompt],
+            outputs=result
+        )
+    
+    return demo
+
+if __name__ == "__main__":
+    print("🌐 Starting SmolVLM TRAC Automation Interface...")
+    app = create_app()
+    app.launch(inbrowser=True)

merge_info.json

@@ -0,0 +1,22 @@
+{
+  "total_batch_models": 14,
+  "successfully_merged": 14,
+  "base_model": "HuggingFaceTB/SmolVLM-Instruct",
+  "final_model_path": "smolvlm_final_merged",
+  "merged_model_paths": [
+    "smolvlm_batched_training_final\\batch_000\\final_model",
+    "smolvlm_batched_training_final\\batch_001\\final_model",
+    "smolvlm_batched_training_final\\batch_002\\final_model",
+    "smolvlm_batched_training_final\\batch_003\\final_model",
+    "smolvlm_batched_training_final\\batch_004\\final_model",
+    "smolvlm_batched_training_final\\batch_005\\final_model",
+    "smolvlm_batched_training_final\\batch_006\\final_model",
+    "smolvlm_batched_training_final\\batch_007\\final_model",
+    "smolvlm_batched_training_final\\batch_008\\final_model",
+    "smolvlm_batched_training_final\\batch_009\\final_model",
+    "smolvlm_batched_training_final\\batch_010\\final_model",
+    "smolvlm_batched_training_final\\batch_011\\final_model",
+    "smolvlm_batched_training_final\\batch_012\\final_model",
+    "smolvlm_batched_training_final\\batch_013\\final_model"
+  ]
+}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a4f76cb64f6f2e4e74716d8fc1cfc6a70bbb3eeea69d424c3ec9902655065eb
+size 4492630912

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "do_convert_rgb": true,
+  "do_image_splitting": true,
+  "do_normalize": true,
+  "do_pad": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "image_processor_type": "Idefics3ImageProcessor",
+  "image_std": [
+    0.5,
+    0.5,
+    0.5
+  ],
+  "max_image_size": {
+    "longest_edge": 384
+  },
+  "processor_class": "Idefics3Processor",
+  "resample": 1,
+  "rescale_factor": 0.00392156862745098,
+  "size": {
+    "longest_edge": 1536
+  }
+}

processor_config.json

@@ -0,0 +1,4 @@
+{
+  "image_seq_len": 81,
+  "processor_class": "Idefics3Processor"
+}

run_model.bat

@@ -0,0 +1 @@
+cd "C:\Users\keith\OneDrive\Desktop\admin.trac.jobs-DATA\LLaMA-Factory_local\smolvlm_final_merged" && "C:\Users\keith\AppData\Local\Programs\Python\Python313\python.exe" test_ui_agent.py

special_tokens_map.json

@@ -0,0 +1,53 @@
+{
+  "additional_special_tokens": [
+    {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    },
+    {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false
+    }
+  ],
+  "bos_token": {
+    "content": "<|im_start|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<end_of_utterance>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|im_end|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

test_ui_agent.py

@@ -0,0 +1,100 @@
+"""
+SmolVLM UI Automation Agent - Test Script
+Your trained model is ready!
+"""
+
+import torch
+from transformers import Idefics3ForConditionalGeneration, AutoProcessor
+from PIL import Image
+import os
+
+def load_model():
+    """Load your trained SmolVLM model"""
+    model_path = r"C:\Users\keith\OneDrive\Desktop\admin.trac.jobs-DATA\LLaMA-Factory_local\smolvlm_final_merged"
+    
+    print("Loading your trained SmolVLM UI automation agent...")
+    model = Idefics3ForConditionalGeneration.from_pretrained(
+        model_path,
+        torch_dtype=torch.bfloat16,
+        device_map="auto",
+        trust_remote_code=True
+    )
+    
+    processor = AutoProcessor.from_pretrained(model_path)
+    print("Model loaded successfully!")
+    return model, processor
+
+def analyze_screenshot(image_path: str, model, processor):
+    """Analyze a screenshot for UI automation"""
+    
+    # Load and process image
+    image = Image.open(image_path).convert("RGB")
+    prompt = "<image>\nAnalyze this interface for UI automation opportunities. Identify clickable elements and automation targets."
+    
+    # Process inputs
+    inputs = processor(text=prompt, images=[image], return_tensors="pt")
+    
+    # Generate response
+    with torch.no_grad():
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=150,
+            do_sample=True,
+            temperature=0.7,
+            top_p=0.9
+        )
+    
+    # Decode response
+    response = processor.decode(outputs[0], skip_special_tokens=True)
+    
+    # Extract just the assistant's response
+    if "Assistant:" in response:
+        response = response.split("Assistant:")[-1].strip()
+    
+    return response
+
+def main():
+    print("🤖 SmolVLM UI Automation Agent")
+    print("=" * 50)
+    print("Your custom-trained model for TRAC administration!")
+    print()
+    
+    try:
+        # Load your trained model
+        model, processor = load_model()
+        
+        while True:
+            print("\nOptions:")
+            print("1. Analyze a screenshot")
+            print("2. Quit")
+            
+            choice = input("\nEnter choice (1-2): ").strip()
+            
+            if choice == "1":
+                image_path = input("Enter path to screenshot: ").strip().strip('"')
+                
+                if os.path.exists(image_path):
+                    print("\n🔍 Analyzing screenshot...")
+                    try:
+                        result = analyze_screenshot(image_path, model, processor)
+                        print("\n🎯 Analysis Result:")
+                        print("-" * 30)
+                        print(result)
+                        print("-" * 30)
+                    except Exception as e:
+                        print(f"❌ Analysis error: {e}")
+                else:
+                    print("❌ Image file not found!")
+                    
+            elif choice == "2":
+                print("👋 Goodbye!")
+                break
+            else:
+                print("❌ Invalid choice!")
+                
+    except Exception as e:
+        print(f"❌ Error loading model: {e}")
+        print("Make sure the model was merged successfully.")
+
+if __name__ == "__main__":
+    main()

tokenizer_config.json

@@ -0,0 +1,182 @@
+{
+  "add_prefix_space": false,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<|im_start|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "<|im_end|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "3": {
+      "content": "<repo_name>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "4": {
+      "content": "<reponame>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "5": {
+      "content": "<file_sep>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "6": {
+      "content": "<filename>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "7": {
+      "content": "<gh_stars>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "8": {
+      "content": "<issue_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "9": {
+      "content": "<issue_comment>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "10": {
+      "content": "<issue_closed>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "11": {
+      "content": "<jupyter_start>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "12": {
+      "content": "<jupyter_text>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "13": {
+      "content": "<jupyter_code>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "14": {
+      "content": "<jupyter_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "15": {
+      "content": "<jupyter_script>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "16": {
+      "content": "<empty_output>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49152": {
+      "content": "<fake_token_around_image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49153": {
+      "content": "<image>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "49154": {
+      "content": "<end_of_utterance>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<fake_token_around_image>",
+    "<image>",
+    "<end_of_utterance>"
+  ],
+  "bos_token": "<|im_start|>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "<end_of_utterance>",
+  "extra_special_tokens": {},
+  "legacy": false,
+  "model_max_length": 16384,
+  "pad_token": "<|im_end|>",
+  "processor_class": "Idefics3Processor",
+  "tokenizer_class": "GPT2Tokenizer",
+  "truncation_side": "left",
+  "unk_token": "<|endoftext|>",
+  "vocab_size": 49152
+}