Update initialize_system.py

initialize_system.py

@@ -1,19 +1,10 @@
 import os
 import sys
-import json
-import joblib
 import shutil
 import pandas as pd
+import json
 from pathlib import Path
 from datetime import datetime
-from sklearn.pipeline import Pipeline
-from model.train import EnhancedModelTrainer
-from sklearn.model_selection import cross_validate
-from sklearn.linear_model import LogisticRegression
-from sklearn.model_selection import train_test_split
-from sklearn.metrics import accuracy_score, f1_score
-from sklearn.feature_extraction.text import TfidfVectorizer
-        
 
 # Import the new path manager
 try:
@@ -188,9 +179,6 @@ def create_minimal_dataset():
         return False
 
 
-
-
-
 def run_initial_training():
     """Run basic model training"""
     log_step("Starting initial model training...")
@@ -236,7 +224,12 @@ def run_initial_training():
             return True
 
         # Import required libraries
-        
+        from sklearn.feature_extraction.text import TfidfVectorizer
+        from sklearn.linear_model import LogisticRegression
+        from sklearn.model_selection import train_test_split
+        from sklearn.metrics import accuracy_score, f1_score
+        from sklearn.pipeline import Pipeline
+        import joblib
 
         # Load dataset
         dataset_path = path_manager.get_combined_dataset_path()
@@ -281,67 +274,22 @@ def run_initial_training():
             ))
         ])
 
-        # Train model with cross-validation
-        log_step("Training model with cross-validation...")
-        
-        # Perform cross-validation before final training
-        cv_results = cross_validate(
-            pipeline, X_train, y_train, 
-            cv=3, 
-            scoring=['accuracy', 'f1_weighted', 'precision_weighted', 'recall_weighted'],
-            return_train_score=True
-        )
-        
-        # Train final model on all training data
+        # Train model
+        log_step("Training model...")
         pipeline.fit(X_train, y_train)
-        
-        # Evaluate on test set
+
+        # Evaluate
         y_pred = pipeline.predict(X_test)
         accuracy = accuracy_score(y_test, y_pred)
         f1 = f1_score(y_test, y_pred, average='weighted')
-        
-        # Save CV results for API access
-        cv_data = {
-            "n_splits": 3,
-            "test_scores": {
-                "accuracy": {
-                    "mean": float(cv_results['test_accuracy'].mean()),
-                    "std": float(cv_results['test_accuracy'].std()),
-                    "scores": cv_results['test_accuracy'].tolist()
-                },
-                "f1": {
-                    "mean": float(cv_results['test_f1_weighted'].mean()),
-                    "std": float(cv_results['test_f1_weighted'].std()),
-                    "scores": cv_results['test_f1_weighted'].tolist()
-                }
-            },
-            "train_scores": {
-                "accuracy": {
-                    "mean": float(cv_results['train_accuracy'].mean()),
-                    "std": float(cv_results['train_accuracy'].std()),
-                    "scores": cv_results['train_accuracy'].tolist()
-                },
-                "f1": {
-                    "mean": float(cv_results['train_f1_weighted'].mean()),
-                    "std": float(cv_results['train_f1_weighted'].std()),
-                    "scores": cv_results['train_f1_weighted'].tolist()
-                }
-            }
-        }
-        
-        # Save CV results to file
-        cv_results_path = path_manager.get_logs_path("cv_results.json")
-        with open(cv_results_path, 'w') as f:
-            json.dump(cv_data, f, indent=2)
-        log_step(f"Saved CV results to: {cv_results_path}")
-        
+
         # Ensure model directory exists
         model_path.parent.mkdir(parents=True, exist_ok=True)
-        
+
         # Save complete pipeline FIRST (this is the priority)
         log_step(f"Saving pipeline to: {pipeline_path}")
         joblib.dump(pipeline, pipeline_path)
-                
+        
         # Verify pipeline was saved
         if pipeline_path.exists():
             log_step(f"✅ Pipeline saved successfully to {pipeline_path}")
@@ -378,12 +326,7 @@ def run_initial_training():
             "data_path": str(dataset_path),
             "class_distribution": class_counts.to_dict(),
             "pipeline_created": pipeline_path.exists(),
-            "individual_components_created": model_path.exists() and vectorizer_path.exists(),
-            # Add CV results to metadata
-            "cv_f1_mean": float(cv_results['test_f1_weighted'].mean()),
-            "cv_f1_std": float(cv_results['test_f1_weighted'].std()),
-            "cv_accuracy_mean": float(cv_results['test_accuracy'].mean()),
-            "cv_accuracy_std": float(cv_results['test_accuracy'].std())
+            "individual_components_created": model_path.exists() and vectorizer_path.exists()
         }
 
         metadata_path = path_manager.get_metadata_path()
@@ -445,20 +388,6 @@ def create_initial_logs():
                     json.dump([], f)
                 log_step(f"✅ Created {log_file}")
 
-        # Create monitoring directory structure
-        monitor_dir = path_manager.get_logs_path("monitor")
-        monitor_dir.mkdir(parents=True, exist_ok=True)
-        log_step(f"✅ Created monitoring directory: {monitor_dir}")
-
-        # Create empty monitoring log files
-        monitor_files = ["predictions.json", "metrics.json", "alerts.json"]
-        for monitor_file in monitor_files:
-            monitor_path = monitor_dir / monitor_file
-            if not monitor_path.exists():
-                with open(monitor_path, 'w') as f:
-                    json.dump([], f)
-                log_step(f"✅ Created {monitor_file}")
-
         return True
 
     except Exception as e:
@@ -591,7 +520,7 @@ def main():
     return len(failed_steps) == 0
 
 
-
 if __name__ == "__main__":
     success = main()
-    sys.exit(0)
+    if not success:
+        sys.exit(1)