Update data/scrape_real_news.py

data/scrape_real_news.py

@@ -1,17 +1,20 @@
-import requests
+import re
 import time
+import json
+import random
+import hashlib
 import logging
+import requests
 import pandas as pd
 from pathlib import Path
-from datetime import datetime, timedelta
-from typing import List, Dict, Optional, Tuple
 from newspaper import Article, build
-import hashlib
-import json
+from datetime import datetime, timedelta
 from urllib.parse import urljoin, urlparse
-import random
+from typing import List, Dict, Optional, Tuple
+from data.validation_schemas import ValidationLevel
+from data.data_validator import DataValidationPipeline
 from concurrent.futures import ThreadPoolExecutor, as_completed
-import re
+
 
 # Configure logging
 logging.basicConfig(
@@ -324,40 +327,42 @@ class RobustNewsScraper:
         return all_articles
     
     def save_scraped_articles(self, articles: List[Dict]) -> bool:
-        """Save scraped articles to CSV with error handling"""
+        """Save scraped articles with validation"""
         try:
             if not articles:
-                logger.info("No articles to save")
                 return True
             
-            # Create DataFrame
-            df_new = pd.DataFrame(articles)
+            # Validate articles first
+            valid_articles, validation_summary = self.validate_scraped_articles(articles)
             
-            # Load existing data if present
+            logger.info(f"Validation: {len(valid_articles)}/{len(articles)} articles passed validation")
+            
+            if not valid_articles:
+                logger.warning("No valid articles to save after validation")
+                return True
+            
+            # Create DataFrame and save
+            df_new = pd.DataFrame(valid_articles)
+            
+            # Existing file handling logic...
             if self.output_path.exists():
-                try:
-                    df_existing = pd.read_csv(self.output_path)
-                    df_combined = pd.concat([df_existing, df_new], ignore_index=True)
-                    
-                    # Remove duplicates based on URL
-                    df_combined = df_combined.drop_duplicates(subset=['url'], keep='last')
-                    
-                    logger.info(f"Combined with existing data. Total: {len(df_combined)} articles")
-                    
-                except Exception as e:
-                    logger.warning(f"Failed to load existing data: {e}")
-                    df_combined = df_new
+                df_existing = pd.read_csv(self.output_path)
+                df_combined = pd.concat([df_existing, df_new], ignore_index=True)
+                df_combined = df_combined.drop_duplicates(subset=['text'], keep='first')
             else:
                 df_combined = df_new
             
-            # Save to CSV
             df_combined.to_csv(self.output_path, index=False)
             
-            logger.info(f"Successfully saved {len(articles)} new articles to {self.output_path}")
-            return True
+            # Save validation report
+            validation_report_path = self.data_dir / "scraping_validation_report.json"
+            with open(validation_report_path, 'w') as f:
+                json.dump(validation_summary, f, indent=2)
             
+            logger.info(f"Saved {len(valid_articles)} validated articles to {self.output_path}")
+            return True
         except Exception as e:
-            logger.error(f"Failed to save articles: {str(e)}")
+            logger.error(f"Failed to save validated articles: {e}")
             return False
     
     def generate_scraping_metadata(self, articles: List[Dict]) -> Dict:
@@ -435,7 +440,45 @@ class RobustNewsScraper:
             error_msg = f"Scraping process failed: {str(e)}"
             logger.error(error_msg)
             return False, error_msg
-
+    
+    def validate_scraped_articles(self, articles: List[Dict]) -> Tuple[List[Dict], Dict]:
+        """Validate scraped articles using validation schemas"""
+        if not articles:
+            return articles, {}
+        
+        validator = DataValidationPipeline()
+        
+        # Ensure required fields for validation
+        enhanced_articles = []
+        for article in articles:
+            enhanced_article = article.copy()
+            if 'source' not in enhanced_article:
+                enhanced_article['source'] = 'scraped_real'
+            if 'label' not in enhanced_article:
+                enhanced_article['label'] = 0  # Real news
+            enhanced_articles.append(enhanced_article)
+        
+        # Validate batch
+        validation_result = validator.validate_scraped_data(enhanced_articles, "web_scraping")
+        
+        # Filter valid articles
+        valid_articles = []
+        for i, result in enumerate(validation_result.validation_results):
+            if result.is_valid:
+                article = enhanced_articles[i].copy()
+                article['validation_quality_score'] = result.quality_metrics.get('overall_quality_score', 0.0)
+                valid_articles.append(article)
+        
+        validation_summary = {
+            'original_count': len(articles),
+            'valid_count': len(valid_articles),
+            'success_rate': validation_result.success_rate,
+            'overall_quality_score': validation_result.overall_quality_score
+        }
+        
+        return valid_articles, validation_summary
+    
+    
 def scrape_articles():
     """Main function for external calls"""
     scraper = RobustNewsScraper()