Update features/linguistic_analyzer.py

Adding Enhanced Feature Engineering Pipeline

features/linguistic_analyzer.py

@@ -1,534 +1,703 @@
-# features/linguistic_analyzer.py
-# Advanced Linguistic Analysis Component for Enhanced Feature Engineering
+# File: data/prepare_datasets.py (ENHANCED)
+# Enhanced version with feature engineering integration and advanced validation
 
-import numpy as np
 import pandas as pd
-import re
+import numpy as np
+from pathlib import Path
 import logging
-from typing import List, Dict, Any, Tuple
-from sklearn.base import BaseEstimator, TransformerMixin
-from collections import Counter, defaultdict
-import warnings
-warnings.filterwarnings('ignore')
+import re
+from typing import Optional, Tuple, Dict, Any
+from sklearn.model_selection import train_test_split
+import hashlib
+import json
+from datetime import datetime
+from data.data_validator import DataValidationPipeline
+from data.validation_schemas import ValidationLevel, DataSource
+from typing import Tuple, Dict
+
+# Import enhanced feature engineering components
+try:
+    from features.feature_engineer import AdvancedFeatureEngineer
+    from features.sentiment_analyzer import SentimentAnalyzer
+    from features.readability_analyzer import ReadabilityAnalyzer
+    from features.entity_analyzer import EntityAnalyzer
+    from features.linguistic_analyzer import LinguisticAnalyzer
+    ENHANCED_FEATURES_AVAILABLE = True
+except ImportError as e:
+    ENHANCED_FEATURES_AVAILABLE = False
+    logging.warning(f"Enhanced features not available in prepare_datasets.py: {e}")
 
+# Configure logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler('/tmp/data_preparation.log'),
+        logging.StreamHandler()
+    ]
+)
 logger = logging.getLogger(__name__)
 
+# Log enhanced feature availability
+if ENHANCED_FEATURES_AVAILABLE:
+    logger.info("Enhanced feature engineering components loaded for data preparation")
+else:
+    logger.warning("Enhanced features not available - standard validation only")
 
-class LinguisticAnalyzer(BaseEstimator, TransformerMixin):
-    """
-    Advanced linguistic analysis for fake news detection.
-    Analyzes syntactic patterns, discourse markers, and linguistic anomalies.
-    """
-    
-    def __init__(self):
-        self.discourse_markers = self._load_discourse_markers()
-        self.linguistic_patterns = self._load_linguistic_patterns()
-        self.pos_patterns = self._load_pos_patterns()
-        self.is_fitted_ = False
-        
-    def _load_discourse_markers(self):
-        """Load discourse markers for coherence analysis"""
-        markers = {
-            'addition': {'also', 'furthermore', 'moreover', 'additionally', 'besides', 'plus', 'and'},
-            'contrast': {'however', 'but', 'nevertheless', 'nonetheless', 'yet', 'still', 'although', 'though'},
-            'cause_effect': {'therefore', 'thus', 'consequently', 'as a result', 'because', 'since', 'so'},
-            'temporal': {'then', 'next', 'afterwards', 'meanwhile', 'subsequently', 'finally', 'first', 'second'},
-            'emphasis': {'indeed', 'certainly', 'obviously', 'clearly', 'definitely', 'absolutely', 'surely'},
-            'concession': {'admittedly', 'granted', 'to be sure', 'of course', 'naturally', 'undoubtedly'},
-            'exemplification': {'for example', 'for instance', 'such as', 'namely', 'specifically', 'particularly'},
-            'summary': {'in conclusion', 'to summarize', 'in summary', 'overall', 'in general', 'basically'}
-        }
-        return markers
-    
-    def _load_linguistic_patterns(self):
-        """Load patterns for linguistic analysis"""
-        patterns = {
-            'modal_verbs': {'can', 'could', 'may', 'might', 'must', 'shall', 'should', 'will', 'would'},
-            'hedge_words': {'probably', 'possibly', 'perhaps', 'maybe', 'likely', 'apparently', 'seemingly', 'supposedly'},
-            'boosters': {'very', 'extremely', 'highly', 'completely', 'totally', 'absolutely', 'definitely', 'certainly'},
-            'negation': {'not', 'no', 'never', 'nothing', 'nobody', 'nowhere', 'neither', 'nor'},
-            'intensifiers': {'so', 'such', 'quite', 'rather', 'pretty', 'fairly', 'really', 'truly', 'deeply'},
-            'questioning': {'why', 'how', 'what', 'when', 'where', 'who', 'which', 'whose'},
-            'personal_pronouns': {'i', 'you', 'he', 'she', 'it', 'we', 'they', 'me', 'him', 'her', 'us', 'them'},
-            'demonstratives': {'this', 'that', 'these', 'those', 'here', 'there'},
-            'quantifiers': {'all', 'every', 'each', 'some', 'any', 'many', 'few', 'several', 'most', 'much'}
-        }
-        return patterns
-    
-    def _load_pos_patterns(self):
-        """Load part-of-speech patterns (simplified without NLTK)"""
-        # Simple heuristics for POS detection
-        patterns = {
-            'verb_endings': {'ed', 'ing', 'en', 's', 'es'},
-            'noun_endings': {'tion', 'sion', 'ment', 'ness', 'ity', 'er', 'or', 'ist', 'ism'},
-            'adjective_endings': {'able', 'ible', 'ful', 'less', 'ous', 'eous', 'ious', 'ive', 'ic', 'al'},
-            'adverb_endings': {'ly', 'ward', 'wise'}
-        }
-        return patterns
-    
-    def fit(self, X, y=None):
-        """Fit the linguistic analyzer"""
-        self.is_fitted_ = True
-        return self
+
+class EnhancedDatasetPreparer:
+    """Enhanced dataset preparation with feature engineering integration and comprehensive validation"""
     
-    def transform(self, X):
-        """Extract linguistic features"""
-        if not self.is_fitted_:
-            raise ValueError("LinguisticAnalyzer must be fitted before transform")
+    def __init__(self, base_dir: Path = None, enable_feature_analysis: bool = None):
+        self.base_dir = base_dir or Path(__file__).resolve().parent
         
-        # Convert input to array if needed
-        if isinstance(X, pd.Series):
-            X = X.values
-        elif isinstance(X, list):
-            X = np.array(X)
+        # Auto-detect enhanced features if not specified
+        if enable_feature_analysis is None:
+            self.enable_feature_analysis = ENHANCED_FEATURES_AVAILABLE
+        else:
+            self.enable_feature_analysis = enable_feature_analysis and ENHANCED_FEATURES_AVAILABLE
         
-        features = []
+        self.setup_paths()
+        self.setup_feature_analyzers()
         
-        for text in X:
-            text_features = self._extract_linguistic_features(str(text))
-            features.append(text_features)
+        logger.info(f"Enhanced dataset preparer initialized with feature analysis: {self.enable_feature_analysis}")
+    
+    def setup_paths(self):
+        """Setup all necessary paths"""
+        # Input paths
+        self.kaggle_fake = self.base_dir / "kaggle" / "Fake.csv"
+        self.kaggle_real = self.base_dir / "kaggle" / "True.csv"
+        self.liar_paths = [
+            self.base_dir / "liar" / "train.tsv",
+            self.base_dir / "liar" / "test.tsv", 
+            self.base_dir / "liar" / "valid.tsv"
+        ]
         
-        return np.array(features)
+        # Output paths
+        self.output_dir = Path("/tmp/data")
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        self.output_path = self.output_dir / "combined_dataset.csv"
+        self.metadata_path = self.output_dir / "dataset_metadata.json"
+        
+        # Enhanced feature paths
+        self.feature_analysis_dir = self.output_dir / "feature_analysis"
+        self.feature_analysis_dir.mkdir(parents=True, exist_ok=True)
+        self.feature_quality_path = self.feature_analysis_dir / "feature_quality_report.json"
+        self.feature_samples_path = self.feature_analysis_dir / "feature_samples.json"
     
-    def fit_transform(self, X, y=None):
-        """Fit and transform in one step"""
-        return self.fit(X, y).transform(X)
+    def setup_feature_analyzers(self):
+        """Setup feature analyzers if enhanced features are enabled"""
+        self.feature_analyzers = {}
+        
+        if self.enable_feature_analysis:
+            try:
+                self.feature_analyzers = {
+                    'sentiment': SentimentAnalyzer(),
+                    'readability': ReadabilityAnalyzer(),
+                    'entity': EntityAnalyzer(),
+                    'linguistic': LinguisticAnalyzer()
+                }
+                
+                # Fit analyzers (they're stateless but need fit() call for API consistency)
+                for analyzer in self.feature_analyzers.values():
+                    analyzer.fit([])
+                
+                logger.info("Enhanced feature analyzers initialized successfully")
+                
+            except Exception as e:
+                logger.warning(f"Failed to initialize feature analyzers: {e}")
+                self.enable_feature_analysis = False
+                self.feature_analyzers = {}
     
-    def _extract_linguistic_features(self, text):
-        """Extract comprehensive linguistic features"""
-        text_lower = text.lower()
-        words = re.findall(r'\b\w+\b', text_lower)
-        sentences = re.split(r'[.!?]+', text)
-        sentences = [s.strip() for s in sentences if s.strip()]
-        
-        if len(words) == 0:
-            return [0.0] * 25  # Return zeros for empty text
-        
-        features = []
-        
-        # Discourse marker features
-        discourse_features = self._extract_discourse_features(text_lower, words)
-        features.extend(discourse_features)
-        
-        # Linguistic pattern features
-        pattern_features = self._extract_pattern_features(text_lower, words)
-        features.extend(pattern_features)
-        
-        # Syntactic complexity features
-        syntax_features = self._extract_syntax_features(text, sentences, words)
-        features.extend(syntax_features)
-        
-        # Coherence and flow features
-        coherence_features = self._extract_coherence_features(text, sentences)
-        features.extend(coherence_features)
-        
-        return features
+    def validate_text_quality(self, text: str) -> bool:
+        """Enhanced text quality validation with feature analysis"""
+        if not isinstance(text, str):
+            return False
+        
+        text = text.strip()
+        
+        # Basic length check
+        if len(text) < 10:
+            return False
+        
+        # Check for meaningful content
+        if not any(c.isalpha() for c in text):
+            return False
+        
+        # Check for sentence structure
+        if not any(punct in text for punct in '.!?'):
+            return False
+        
+        # Check for excessive repetition
+        words = text.lower().split()
+        if len(words) > 0:
+            most_common_word_count = max(words.count(word) for word in set(words))
+            if most_common_word_count > len(words) * 0.5:  # More than 50% repetition
+                return False
+        
+        # Check for excessive special characters
+        special_char_ratio = sum(1 for c in text if not c.isalnum() and not c.isspace()) / len(text)
+        if special_char_ratio > 0.3:  # More than 30% special characters
+            return False
+        
+        # Enhanced feature-based quality checks
+        if self.enable_feature_analysis:
+            try:
+                quality_score = self.assess_text_feature_quality(text)
+                if quality_score < 0.3:  # Minimum quality threshold
+                    return False
+            except Exception as e:
+                logger.warning(f"Feature quality check failed for text: {e}")
+        
+        return True
     
-    def _extract_discourse_features(self, text_lower, words):
-        """Extract discourse marker features"""
-        features = []
-        total_words = len(words)
+    def assess_text_feature_quality(self, text: str) -> float:
+        """Assess text quality using enhanced features"""
+        if not self.enable_feature_analysis:
+            return 1.0  # Default to high quality if features not available
         
-        # Count discourse markers by category
-        for marker_type, markers in self.discourse_markers.items():
-            marker_count = 0
+        try:
+            quality_scores = []
             
-            # Single word markers
-            marker_count += sum(1 for word in words if word in markers)
+            # Sentiment analysis quality
+            if 'sentiment' in self.feature_analyzers:
+                sentiment_analysis = self.feature_analyzers['sentiment'].analyze_text_sentiment(text)
+                # Good quality text should have reasonable emotional balance
+                emotional_balance = 1 - abs(sentiment_analysis.get('emotional_intensity', 0))
+                quality_scores.append(max(0.1, emotional_balance))
             
-            # Multi-word markers
-            for marker in markers:
-                if ' ' in marker:
-                    marker_count += text_lower.count(marker)
+            # Readability quality
+            if 'readability' in self.feature_analyzers:
+                readability_analysis = self.feature_analyzers['readability'].analyze_text_readability(text)
+                if 'interpretation' in readability_analysis:
+                    readability_level = readability_analysis['interpretation'].get('readability_level', 'standard')
+                    # Moderate readability levels indicate higher quality
+                    readability_score = {
+                        'very_easy': 0.7, 'easy': 0.8, 'fairly_easy': 0.9, 'standard': 1.0,
+                        'fairly_difficult': 0.9, 'difficult': 0.7, 'very_difficult': 0.5
+                    }.get(readability_level, 0.5)
+                    quality_scores.append(readability_score)
             
-            marker_ratio = marker_count / total_words if total_words > 0 else 0
-            features.append(marker_ratio)
-        
-        return features
-    
-    def _extract_pattern_features(self, text_lower, words):
-        """Extract linguistic pattern features"""
-        features = []
-        total_words = len(words)
-        
-        # Count linguistic patterns
-        for pattern_type, pattern_words in self.linguistic_patterns.items():
-            pattern_count = sum(1 for word in words if word in pattern_words)
-            pattern_ratio = pattern_count / total_words if total_words > 0 else 0
-            features.append(pattern_ratio)
-        
-        return features
+            # Entity analysis quality
+            if 'entity' in self.feature_analyzers:
+                entity_analysis = self.feature_analyzers['entity'].analyze_text_entities(text)
+                # Quality text should have reasonable entity density
+                entity_density = entity_analysis.get('summary', {}).get('total_entities_found', 0)
+                normalized_density = min(1.0, entity_density / max(1, len(text.split()) * 0.1))
+                quality_scores.append(0.5 + normalized_density * 0.5)
+            
+            # Linguistic analysis quality
+            if 'linguistic' in self.feature_analyzers:
+                linguistic_analysis = self.feature_analyzers['linguistic'].analyze_text_linguistics(text)
+                discourse_quality = linguistic_analysis.get('overall_assessment', {}).get('discourse_quality', 'fair')
+                discourse_score = {
+                    'poor': 0.3, 'fair': 0.6, 'good': 0.8, 'excellent': 1.0
+                }.get(discourse_quality, 0.5)
+                quality_scores.append(discourse_score)
+            
+            # Calculate overall quality score
+            if quality_scores:
+                overall_quality = sum(quality_scores) / len(quality_scores)
+            else:
+                overall_quality = 1.0  # Default if no analyzers available
+            
+            return overall_quality
+            
+        except Exception as e:
+            logger.warning(f"Feature quality assessment failed: {e}")
+            return 0.5  # Default middle-ground quality
     
-    def _extract_syntax_features(self, text, sentences, words):
-        """Extract syntactic complexity features"""
-        features = []
+    def clean_text(self, text: str) -> str:
+        """Enhanced text cleaning with feature preservation"""
+        if not isinstance(text, str):
+            return ""
         
-        # Average sentence length
-        if sentences:
-            avg_sentence_length = len(words) / len(sentences)
-        else:
-            avg_sentence_length = 0
-        features.append(avg_sentence_length)
-        
-        # Sentence length variance
-        if len(sentences) > 1:
-            sentence_lengths = [len(sentence.split()) for sentence in sentences]
-            mean_length = sum(sentence_lengths) / len(sentence_lengths)
-            variance = sum((length - mean_length) ** 2 for length in sentence_lengths) / len(sentence_lengths)
-        else:
-            variance = 0
-        features.append(variance)
-        
-        # Complex sentence indicators
-        complex_indicators = self._count_complex_sentence_indicators(text)
-        features.extend(complex_indicators)
-        
-        return features
-    
-    def _count_complex_sentence_indicators(self, text):
-        """Count indicators of complex sentence structure"""
-        indicators = []
-        
-        # Subordinating conjunctions
-        subordinating = ['although', 'because', 'since', 'while', 'whereas', 'if', 'unless', 'when', 'where']
-        sub_count = sum(text.lower().count(f' {conj} ') for conj in subordinating)
-        indicators.append(sub_count / len(text) * 1000 if text else 0)
-        
-        # Relative pronouns
-        relative_pronouns = ['that', 'which', 'who', 'whom', 'whose', 'where', 'when']
-        rel_count = sum(text.lower().count(f' {pron} ') for pron in relative_pronouns)
-        indicators.append(rel_count / len(text) * 1000 if text else 0)
-        
-        # Passive voice indicators (simplified)
-        passive_indicators = ['was', 'were', 'been', 'being']
-        passive_count = sum(text.lower().count(f' {ind} ') for ind in passive_indicators)
-        indicators.append(passive_count / len(text) * 1000 if text else 0)
-        
-        return indicators
-    
-    def _extract_coherence_features(self, text, sentences):
-        """Extract text coherence and flow features"""
-        features = []
+        # Remove excessive whitespace
+        text = re.sub(r'\s+', ' ', text)
         
-        # Paragraph structure (approximate)
-        paragraphs = text.split('\n\n')
-        paragraphs = [p.strip() for p in paragraphs if p.strip()]
+        # Remove URLs but preserve text structure
+        text = re.sub(r'http\S+|www\S+|https\S+', '[URL]', text)
         
-        # Average paragraph length
-        if paragraphs:
-            avg_paragraph_length = sum(len(p.split()) for p in paragraphs) / len(paragraphs)
-        else:
-            avg_paragraph_length = 0
-        features.append(avg_paragraph_length)
+        # Remove excessive punctuation while preserving meaning
+        text = re.sub(r'[!]{3,}', '!!', text)
+        text = re.sub(r'[?]{3,}', '??', text)
+        text = re.sub(r'[.]{4,}', '...', text)
         
-        # Topic coherence (simplified using word repetition)
-        coherence_score = self._calculate_lexical_coherence(sentences)
-        features.append(coherence_score)
+        # Remove non-printable characters
+        text = ''.join(char for char in text if ord(char) >= 32)
         
-        # Transition density
-        transition_density = self._calculate_transition_density(text)
-        features.append(transition_density)
+        # Preserve important features for analysis
+        # Don't over-clean as it might remove important linguistic patterns
         
-        return features
+        return text.strip()
     
-    def _calculate_lexical_coherence(self, sentences):
-        """Calculate lexical coherence between sentences"""
-        if len(sentences) < 2:
-            return 0
-        
-        coherence_scores = []
-        
-        for i in range(len(sentences) - 1):
-            words1 = set(re.findall(r'\b\w+\b', sentences[i].lower()))
-            words2 = set(re.findall(r'\b\w+\b', sentences[i + 1].lower()))
+    def extract_text_features_sample(self, texts: list, sample_size: int = 100) -> Dict:
+        """Extract feature samples for quality analysis"""
+        if not self.enable_feature_analysis or not texts:
+            return {}
+        
+        try:
+            # Sample texts for feature analysis
+            sample_texts = texts[:sample_size] if len(texts) > sample_size else texts
             
-            # Remove very common words
-            common_words = {'the', 'a', 'an', 'and', 'or', 'but', 'in', 'on', 'at', 'to', 'for', 'of', 'with', 'by'}
-            words1 = words1 - common_words
-            words2 = words2 - common_words
+            feature_samples = {
+                'sample_size': len(sample_texts),
+                'total_texts': len(texts),
+                'feature_analysis': {},
+                'quality_metrics': {},
+                'timestamp': datetime.now().isoformat()
+            }
             
-            if words1 and words2:
-                overlap = len(words1.intersection(words2))
-                union = len(words1.union(words2))
-                coherence = overlap / union if union > 0 else 0
-                coherence_scores.append(coherence)
-        
-        return sum(coherence_scores) / len(coherence_scores) if coherence_scores else 0
+            # Analyze features for sample texts
+            for analyzer_name, analyzer in self.feature_analyzers.items():
+                try:
+                    analyzer_samples = []
+                    
+                    for i, text in enumerate(sample_texts[:10]):  # Analyze first 10 for detailed samples
+                        if analyzer_name == 'sentiment':
+                            analysis = analyzer.analyze_text_sentiment(text)
+                        elif analyzer_name == 'readability':
+                            analysis = analyzer.analyze_text_readability(text)
+                        elif analyzer_name == 'entity':
+                            analysis = analyzer.analyze_text_entities(text)
+                        elif analyzer_name == 'linguistic':
+                            analysis = analyzer.analyze_text_linguistics(text)
+                        else:
+                            continue
+                        
+                        analyzer_samples.append({
+                            'text_index': i,
+                            'text_preview': text[:100] + "..." if len(text) > 100 else text,
+                            'analysis': analysis
+                        })
+                    
+                    feature_samples['feature_analysis'][analyzer_name] = analyzer_samples
+                    
+                except Exception as e:
+                    logger.warning(f"Feature analysis failed for {analyzer_name}: {e}")
+                    feature_samples['feature_analysis'][analyzer_name] = {'error': str(e)}
+            
+            # Calculate overall quality metrics
+            quality_scores = []
+            for text in sample_texts:
+                quality_score = self.assess_text_feature_quality(text)
+                quality_scores.append(quality_score)
+            
+            feature_samples['quality_metrics'] = {
+                'mean_quality': float(np.mean(quality_scores)),
+                'std_quality': float(np.std(quality_scores)),
+                'min_quality': float(np.min(quality_scores)),
+                'max_quality': float(np.max(quality_scores)),
+                'quality_distribution': {
+                    'high_quality': sum(1 for q in quality_scores if q > 0.7),
+                    'medium_quality': sum(1 for q in quality_scores if 0.4 <= q <= 0.7),
+                    'low_quality': sum(1 for q in quality_scores if q < 0.4)
+                }
+            }
+            
+            return feature_samples
+            
+        except Exception as e:
+            logger.error(f"Feature sample extraction failed: {e}")
+            return {'error': str(e)}
     
-    def _calculate_transition_density(self, text):
-        """Calculate density of transition words"""
-        transition_words = {
-            'however', 'therefore', 'furthermore', 'moreover', 'consequently',
-            'nevertheless', 'nonetheless', 'meanwhile', 'additionally', 'similarly',
-            'likewise', 'in contrast', 'on the other hand', 'for example', 'for instance'
-        }
-        
-        text_lower = text.lower()
-        transition_count = 0
-        
-        for transition in transition_words:
-            if ' ' in transition:
-                transition_count += text_lower.count(transition)
+    def load_kaggle_dataset(self) -> Optional[pd.DataFrame]:
+        """Load and process Kaggle dataset with enhanced validation"""
+        try:
+            logger.info("Loading Kaggle dataset with enhanced validation...")
+            
+            # Check if files exist
+            if not self.kaggle_fake.exists() or not self.kaggle_real.exists():
+                logger.warning("Kaggle dataset files not found")
+                return None
+            
+            # Load datasets
+            df_fake = pd.read_csv(self.kaggle_fake)
+            df_real = pd.read_csv(self.kaggle_real)
+            
+            logger.info(f"Loaded {len(df_fake)} fake and {len(df_real)} real articles from Kaggle")
+            
+            # Process fake news
+            df_fake['label'] = 1
+            df_fake['text'] = df_fake['title'].fillna('') + ". " + df_fake['text'].fillna('')
+            df_fake['source'] = 'kaggle_fake'
+            df_fake['original_source'] = 'kaggle'
+            
+            # Process real news
+            df_real['label'] = 0
+            df_real['text'] = df_real['title'].fillna('') + ". " + df_real['text'].fillna('')
+            df_real['source'] = 'kaggle_real'
+            df_real['original_source'] = 'kaggle'
+            
+            # Combine datasets
+            df_combined = pd.concat([
+                df_fake[['text', 'label', 'source', 'original_source']],
+                df_real[['text', 'label', 'source', 'original_source']]
+            ], ignore_index=True)
+            
+            logger.info(f"Combined Kaggle dataset: {len(df_combined)} samples")
+            
+            # Enhanced validation with feature analysis
+            validated_df, validation_report = self.validate_dataset_with_enhanced_features(
+                df_combined, 'kaggle_combined'
+            )
+            
+            return validated_df, validation_report
+            
+        except Exception as e:
+            logger.error(f"Error loading Kaggle dataset: {e}")
+            return None
+    
+    def load_liar_dataset(self) -> Optional[pd.DataFrame]:
+        """Load and process LIAR dataset with enhanced validation"""
+        try:
+            logger.info("Loading LIAR dataset with enhanced validation...")
+            
+            liar_dfs = []
+            total_processed = 0
+            
+            for path in self.liar_paths:
+                if not path.exists():
+                    logger.warning(f"LIAR file not found: {path}")
+                    continue
+                
+                try:
+                    # Read TSV with flexible parameters
+                    df = pd.read_csv(
+                        path, 
+                        sep='\t', 
+                        header=None, 
+                        quoting=3,
+                        on_bad_lines='skip',
+                        low_memory=False
+                    )
+                    
+                    # Expected columns for LIAR dataset
+                    expected_columns = [
+                        'id', 'label_text', 'statement', 'subject', 'speaker', 'job',
+                        'state', 'party', 'barely_true', 'false', 'half_true', 
+                        'mostly_true', 'pants_on_fire', 'context'
+                    ]
+                    
+                    # Handle different column counts
+                    if len(df.columns) >= 3:
+                        df.columns = expected_columns[:len(df.columns)]
+                        
+                        # Map labels to binary classification
+                        if 'label_text' in df.columns:
+                            df['label'] = df['label_text'].apply(
+                                lambda x: 1 if str(x).lower() in ['false', 'pants-fire', 'barely-true'] else 0
+                            )
+                        else:
+                            continue
+                        
+                        # Extract text
+                        if 'statement' in df.columns:
+                            df['text'] = df['statement'].astype(str)
+                        else:
+                            continue
+                        
+                        df['source'] = f'liar_{path.stem}'
+                        df['original_source'] = 'liar'
+                        
+                        processed_df = df[['text', 'label', 'source', 'original_source']].copy()
+                        liar_dfs.append(processed_df)
+                        total_processed += len(processed_df)
+                        
+                        logger.info(f"Processed {len(processed_df)} samples from {path.name}")
+                    
+                except Exception as e:
+                    logger.error(f"Error processing LIAR file {path}: {e}")
+                    continue
+            
+            if liar_dfs:
+                combined_liar = pd.concat(liar_dfs, ignore_index=True)
+                logger.info(f"Combined LIAR dataset: {len(combined_liar)} samples")
+                
+                # Enhanced validation with feature analysis
+                validated_df, validation_report = self.validate_dataset_with_enhanced_features(
+                    combined_liar, 'liar_combined'
+                )
+                
+                return validated_df, validation_report
             else:
-                transition_count += len(re.findall(rf'\b{transition}\b', text_lower))
-        
-        return transition_count / len(text) * 1000 if text else 0
+                logger.warning("No LIAR data could be processed")
+                return None
+                
+        except Exception as e:
+            logger.error(f"Error loading LIAR dataset: {e}")
+            return None
     
-    def get_feature_names(self):
-        """Get names of extracted features"""
-        feature_names = []
-        
-        # Discourse marker features
-        for marker_type in self.discourse_markers.keys():
-            feature_names.append(f'linguistic_{marker_type}_markers_ratio')
-        
-        # Linguistic pattern features
-        for pattern_type in self.linguistic_patterns.keys():
-            feature_names.append(f'linguistic_{pattern_type}_ratio')
-        
-        # Syntax features
-        syntax_features = [
-            'linguistic_avg_sentence_length',
-            'linguistic_sentence_length_variance',
-            'linguistic_subordinating_density',
-            'linguistic_relative_pronouns_density',
-            'linguistic_passive_voice_density'
-        ]
-        feature_names.extend(syntax_features)
-        
-        # Coherence features
-        coherence_features = [
-            'linguistic_avg_paragraph_length',
-            'linguistic_lexical_coherence',
-            'linguistic_transition_density'
-        ]
-        feature_names.extend(coherence_features)
-        
-        return feature_names
+    def validate_dataset(self, df: pd.DataFrame) -> pd.DataFrame:
+        """Comprehensive dataset validation with enhanced feature analysis"""
+        logger.info("Starting enhanced dataset validation...")
+        
+        initial_count = len(df)
+        
+        # Remove null texts
+        df = df.dropna(subset=['text'])
+        logger.info(f"Removed {initial_count - len(df)} null text entries")
+        
+        # Clean text with feature preservation
+        df['text'] = df['text'].apply(self.clean_text)
+        
+        # Enhanced text quality validation
+        if self.enable_feature_analysis:
+            logger.info("Performing feature-based quality validation...")
+            quality_mask = df['text'].apply(self.validate_text_quality)
+            quality_removed = len(df) - quality_mask.sum()
+            df = df[quality_mask]
+            logger.info(f"Removed {quality_removed} low-quality texts using enhanced validation")
+        else:
+            # Standard validation
+            valid_mask = df['text'].apply(self.validate_text_quality)
+            df = df[valid_mask]
+            logger.info(f"Removed {initial_count - valid_mask.sum()} low-quality texts")
+        
+        # Remove duplicates
+        before_dedup = len(df)
+        df = df.drop_duplicates(subset=['text'])
+        logger.info(f"Removed {before_dedup - len(df)} duplicate texts")
+        
+        # Validate label distribution
+        label_counts = df['label'].value_counts()
+        logger.info(f"Label distribution: {label_counts.to_dict()}")
+        
+        # Check for balance
+        if len(label_counts) > 1:
+            balance_ratio = label_counts.min() / label_counts.max()
+            if balance_ratio < 0.3:
+                logger.warning(f"Dataset is imbalanced (ratio: {balance_ratio:.2f})")
+        
+        # Add enhanced metadata
+        df['text_length'] = df['text'].str.len()
+        df['word_count'] = df['text'].str.split().str.len()
+        df['processed_timestamp'] = datetime.now().isoformat()
+        
+        # Add feature quality scores if enhanced features are available
+        if self.enable_feature_analysis:
+            logger.info("Adding feature quality scores...")
+            try:
+                df['feature_quality_score'] = df['text'].apply(self.assess_text_feature_quality)
+                quality_stats = df['feature_quality_score'].describe()
+                logger.info(f"Feature quality statistics: mean={quality_stats['mean']:.3f}, std={quality_stats['std']:.3f}")
+            except Exception as e:
+                logger.warning(f"Failed to add feature quality scores: {e}")
+        
+        return df
     
-    def analyze_text_linguistics(self, text):
-        """Detailed linguistic analysis of a single text"""
-        if not self.is_fitted_:
-            raise ValueError("LinguisticAnalyzer must be fitted before analysis")
-        
-        text_lower = text.lower()
-        words = re.findall(r'\b\w+\b', text_lower)
-        sentences = re.split(r'[.!?]+', text)
-        sentences = [s.strip() for s in sentences if s.strip()]
-        
-        analysis = {
-            'basic_stats': {
-                'text_length': len(text),
-                'word_count': len(words),
-                'sentence_count': len(sentences),
-                'avg_words_per_sentence': len(words) / len(sentences) if sentences else 0
+    def generate_dataset_metadata(self, df: pd.DataFrame, feature_samples: Dict = None) -> dict:
+        """Generate comprehensive dataset metadata with feature analysis"""
+        metadata = {
+            'total_samples': len(df),
+            'label_distribution': df['label'].value_counts().to_dict(),
+            'source_distribution': df['source'].value_counts().to_dict() if 'source' in df.columns else {},
+            'text_length_stats': {
+                'mean': float(df['text_length'].mean()),
+                'std': float(df['text_length'].std()),
+                'min': int(df['text_length'].min()),
+                'max': int(df['text_length'].max()),
+                'median': float(df['text_length'].median())
             },
-            'discourse_markers': {},
-            'linguistic_patterns': {},
-            'syntactic_complexity': {},
-            'coherence_analysis': {}
+            'word_count_stats': {
+                'mean': float(df['word_count'].mean()),
+                'std': float(df['word_count'].std()),
+                'min': int(df['word_count'].min()),
+                'max': int(df['word_count'].max()),
+                'median': float(df['word_count'].median())
+            },
+            'data_hash': hashlib.md5(df['text'].str.cat().encode()).hexdigest(),
+            'creation_timestamp': datetime.now().isoformat(),
+            'quality_score': self.calculate_quality_score(df),
+            'enhanced_features': {
+                'feature_analysis_enabled': self.enable_feature_analysis,
+                'enhanced_features_available': ENHANCED_FEATURES_AVAILABLE
+            }
         }
         
-        # Analyze discourse markers
-        for marker_type, markers in self.discourse_markers.items():
-            found_markers = []
-            for word in words:
-                if word in markers:
-                    found_markers.append(word)
-            
-            # Check multi-word markers
-            for marker in markers:
-                if ' ' in marker and marker in text_lower:
-                    found_markers.extend([marker] * text_lower.count(marker))
-            
-            analysis['discourse_markers'][marker_type] = {
-                'count': len(found_markers),
-                'ratio': len(found_markers) / len(words) if words else 0,
-                'markers_found': list(set(found_markers))[:5]  # Top 5 unique markers
+        # Add feature quality statistics if available
+        if self.enable_feature_analysis and 'feature_quality_score' in df.columns:
+            metadata['feature_quality_stats'] = {
+                'mean': float(df['feature_quality_score'].mean()),
+                'std': float(df['feature_quality_score'].std()),
+                'min': float(df['feature_quality_score'].min()),
+                'max': float(df['feature_quality_score'].max()),
+                'median': float(df['feature_quality_score'].median())
             }
-        
-        # Analyze linguistic patterns
-        for pattern_type, pattern_words in self.linguistic_patterns.items():
-            found_patterns = [word for word in words if word in pattern_words]
-            analysis['linguistic_patterns'][pattern_type] = {
-                'count': len(found_patterns),
-                'ratio': len(found_patterns) / len(words) if words else 0,
-                'patterns_found': list(set(found_patterns))[:5]
+            
+            # Quality distribution
+            quality_scores = df['feature_quality_score']
+            metadata['feature_quality_distribution'] = {
+                'high_quality': int(sum(quality_scores > 0.7)),
+                'medium_quality': int(sum((quality_scores >= 0.4) & (quality_scores <= 0.7))),
+                'low_quality': int(sum(quality_scores < 0.4))
             }
         
-        # Analyze syntactic complexity
-        complex_indicators = self._count_complex_sentence_indicators(text)
-        analysis['syntactic_complexity'] = {
-            'subordinating_conjunctions_density': complex_indicators[0],
-            'relative_pronouns_density': complex_indicators[1],
-            'passive_voice_density': complex_indicators[2],
-            'sentence_length_variance': self._extract_syntax_features(text, sentences, words)[1],
-            'complexity_score': sum(complex_indicators) / len(complex_indicators)
-        }
-        
-        # Analyze coherence
-        analysis['coherence_analysis'] = {
-            'lexical_coherence': self._calculate_lexical_coherence(sentences),
-            'transition_density': self._calculate_transition_density(text),
-            'paragraph_structure': len(text.split('\n\n')),
-            'overall_coherence_score': (self._calculate_lexical_coherence(sentences) + 
-                                      min(1.0, self._calculate_transition_density(text) / 10)) / 2
-        }
-        
-        # Overall assessment
-        analysis['overall_assessment'] = {
-            'linguistic_sophistication': self._assess_sophistication(analysis),
-            'discourse_quality': self._assess_discourse_quality(analysis),
-            'potential_anomalies': self._detect_linguistic_anomalies(analysis)
-        }
+        # Add feature samples if provided
+        if feature_samples:
+            metadata['feature_analysis_summary'] = {
+                'sample_analyzed': feature_samples.get('sample_size', 0),
+                'quality_metrics': feature_samples.get('quality_metrics', {}),
+                'analyzers_used': list(feature_samples.get('feature_analysis', {}).keys())
+            }
         
-        return analysis
+        return metadata
     
-    def _assess_sophistication(self, analysis):
-        """Assess overall linguistic sophistication"""
-        sophistication_score = 0
-        
-        # Discourse marker variety
-        marker_variety = len([mt for mt, data in analysis['discourse_markers'].items() if data['count'] > 0])
-        sophistication_score += marker_variety / len(self.discourse_markers) * 0.3
-        
-        # Complex syntax usage
-        syntax_score = analysis['syntactic_complexity']['complexity_score']
-        sophistication_score += min(syntax_score, 0.02) / 0.02 * 0.3  # Cap and normalize
-        
-        # Coherence quality
-        coherence_score = analysis['coherence_analysis']['overall_coherence_score']
-        sophistication_score += coherence_score * 0.4
-        
-        if sophistication_score > 0.7:
-            return 'high'
-        elif sophistication_score > 0.4:
-            return 'medium'
-        else:
-            return 'low'
+    def calculate_quality_score(self, df: pd.DataFrame) -> float:
+        """Calculate overall dataset quality score with feature consideration"""
+        scores = []
+        
+        # Balance score
+        label_counts = df['label'].value_counts()
+        if len(label_counts) > 1:
+            balance_score = label_counts.min() / label_counts.max()
+            scores.append(balance_score)
+        
+        # Diversity score (based on unique text ratio)
+        diversity_score = df['text'].nunique() / len(df)
+        scores.append(diversity_score)
+        
+        # Length consistency score
+        text_lengths = df['text_length']
+        length_cv = text_lengths.std() / text_lengths.mean()  # Coefficient of variation
+        length_score = max(0, 1 - length_cv / 2)  # Normalize to 0-1
+        scores.append(length_score)
+        
+        # Feature quality score if available
+        if self.enable_feature_analysis and 'feature_quality_score' in df.columns:
+            feature_quality_mean = df['feature_quality_score'].mean()
+            scores.append(feature_quality_mean)
+        
+        return float(np.mean(scores))
     
-    def _assess_discourse_quality(self, analysis):
-        """Assess discourse quality and organization"""
-        quality_indicators = []
-        
-        # Balanced use of discourse markers
-        marker_counts = [data['count'] for data in analysis['discourse_markers'].values()]
-        if marker_counts:
-            marker_balance = 1 - (max(marker_counts) - min(marker_counts)) / (sum(marker_counts) + 1)
-            quality_indicators.append(marker_balance)
-        
-        # Coherence score
-        quality_indicators.append(analysis['coherence_analysis']['overall_coherence_score'])
-        
-        # Transition usage
-        transition_score = min(1.0, analysis['coherence_analysis']['transition_density'] / 5)
-        quality_indicators.append(transition_score)
-        
-        avg_quality = sum(quality_indicators) / len(quality_indicators) if quality_indicators else 0
-        
-        if avg_quality > 0.7:
-            return 'excellent'
-        elif avg_quality > 0.5:
-            return 'good'
-        elif avg_quality > 0.3:
-            return 'fair'
+    def validate_dataset_with_enhanced_features(self, df: pd.DataFrame, source_name: str) -> Tuple[pd.DataFrame, Dict]:
+        """Validate dataset using both standard and enhanced feature validation"""
+        logger.info(f"Starting enhanced validation for {source_name}...")
+        
+        # Standard validation using existing schemas
+        validator = DataValidationPipeline()
+        
+        # Convert DataFrame to validation format
+        articles_data = []
+        for _, row in df.iterrows():
+            article_data = {
+                'text': str(row.get('text', '')),
+                'label': int(row.get('label', 0)),
+                'source': source_name
+            }
+            
+            if 'title' in row and pd.notna(row['title']):
+                article_data['title'] = str(row['title'])
+            if 'url' in row and pd.notna(row['url']):
+                article_data['url'] = str(row['url'])
+                
+            articles_data.append(article_data)
+        
+        # Perform batch validation
+        validation_result = validator.validate_batch(
+            articles_data, 
+            batch_id=f"{source_name}_{datetime.now().strftime('%Y%m%d_%H%M%S')}",
+            validation_level=ValidationLevel.MODERATE
+        )
+        
+        # Filter valid articles and add quality scores
+        valid_indices = [i for i, result in enumerate(validation_result.validation_results) if result.is_valid]
+        
+        if valid_indices:
+            valid_df = df.iloc[valid_indices].copy()
+            quality_scores = [validation_result.validation_results[i].quality_metrics.get('overall_quality_score', 0.0) 
+                             for i in valid_indices]
+            valid_df['validation_quality_score'] = quality_scores
+            valid_df['validation_timestamp'] = datetime.now().isoformat()
         else:
-            return 'poor'
-    
-    def _detect_linguistic_anomalies(self, analysis):
-        """Detect potential linguistic anomalies that might indicate manipulation"""
-        anomalies = []
-        
-        # Excessive use of boosters/intensifiers
-        booster_ratio = analysis['linguistic_patterns']['boosters']['ratio']
-        if booster_ratio > 0.05:  # More than 5% boosters
-            anomalies.append({
-                'type': 'excessive_boosters',
-                'severity': 'medium',
-                'description': f'High use of intensifying language ({booster_ratio:.1%})',
-                'examples': analysis['linguistic_patterns']['boosters']['patterns_found']
-            })
-        
-        # Unusual negation patterns
-        negation_ratio = analysis['linguistic_patterns']['negation']['ratio']
-        if negation_ratio > 0.08:  # More than 8% negation
-            anomalies.append({
-                'type': 'excessive_negation',
-                'severity': 'low',
-                'description': f'High use of negative language ({negation_ratio:.1%})',
-                'examples': analysis['linguistic_patterns']['negation']['patterns_found']
-            })
-        
-        # Low coherence with high complexity (potential obfuscation)
-        coherence = analysis['coherence_analysis']['overall_coherence_score']
-        complexity = analysis['syntactic_complexity']['complexity_score']
-        if complexity > 0.01 and coherence < 0.3:
-            anomalies.append({
-                'type': 'complexity_without_coherence',
-                'severity': 'high',
-                'description': 'Complex language structure with poor coherence (potential obfuscation)',
-                'coherence_score': coherence,
-                'complexity_score': complexity
-            })
-        
-        # Unusual question density
-        question_ratio = analysis['linguistic_patterns']['questioning']['ratio']
-        if question_ratio > 0.06:  # More than 6% question words
-            anomalies.append({
-                'type': 'excessive_questioning',
-                'severity': 'medium',
-                'description': f'High density of questioning language ({question_ratio:.1%})',
-                'examples': analysis['linguistic_patterns']['questioning']['patterns_found']
-            })
-        
-        return anomalies
-    
-    def get_manipulation_indicators(self, text):
-        """Get specific linguistic manipulation indicators"""
-        analysis = self.analyze_text_linguistics(text)
-        
-        indicators = {
-            'linguistic_manipulation_score': 0.0,
-            'specific_indicators': [],
-            'overall_risk': 'low'
+            valid_df = pd.DataFrame(columns=df.columns)
+        
+        # Enhanced feature-based validation
+        feature_quality_summary = {}
+        if self.enable_feature_analysis and not valid_df.empty:
+            logger.info(f"Performing enhanced feature analysis for {source_name}...")
+            try:
+                # Extract feature samples for analysis
+                feature_samples = self.extract_text_features_sample(valid_df['text'].tolist())
+                
+                # Add feature quality scores
+                valid_df['enhanced_quality_score'] = valid_df['text'].apply(self.assess_text_feature_quality)
+                
+                # Create feature quality summary
+                feature_quality_summary = {
+                    'enhanced_analysis_performed': True,
+                    'feature_samples': feature_samples,
+                    'enhanced_quality_stats': {
+                        'mean': float(valid_df['enhanced_quality_score'].mean()),
+                        'std': float(valid_df['enhanced_quality_score'].std()),
+                        'min': float(valid_df['enhanced_quality_score'].min()),
+                        'max': float(valid_df['enhanced_quality_score'].max())
+                    }
+                }
+                
+                # Save feature samples for reference
+                if feature_samples:
+                    sample_file = self.feature_analysis_dir / f"{source_name}_samples.json"
+                    with open(sample_file, 'w') as f:
+                        json.dump(feature_samples, f, indent=2)
+                    logger.info(f"Feature samples saved to {sample_file}")
+                
+            except Exception as e:
+                logger.warning(f"Enhanced feature analysis failed for {source_name}: {e}")
+                feature_quality_summary = {'enhanced_analysis_performed': False, 'error': str(e)}
+        
+        # Combine validation results
+        validation_summary = {
+            'source': source_name,
+            'original_count': len(df),
+            'valid_count': len(valid_df),
+            'success_rate': validation_result.success_rate,
+            'overall_quality_score': validation_result.overall_quality_score,
+            'validation_timestamp': datetime.now().isoformat(),
+            'enhanced_features': feature_quality_summary
         }
         
-        # Check for manipulation patterns
-        manipulation_score = 0
-        
-        # Excessive emphasis/boosters
-        if analysis['linguistic_patterns']['boosters']['ratio'] > 0.05:
-            manipulation_score += 0.3
-            indicators['specific_indicators'].append('excessive_emphasis')
-        
-        # Lack of hedging (overconfidence)
-        if analysis['linguistic_patterns']['hedge_words']['ratio'] < 0.01:
-            manipulation_score += 0.2
-            indicators['specific_indicators'].append('overconfident_language')
-        
-        # Poor coherence (confusion tactics)
-        if analysis['coherence_analysis']['overall_coherence_score'] < 0.3:
-            manipulation_score += 0.4
-            indicators['specific_indicators'].append('poor_coherence')
-        
-        # Excessive questioning (doubt seeding)
-        if analysis['linguistic_patterns']['questioning']['ratio'] > 0.06:
-            manipulation_score += 0.3
-            indicators['specific_indicators'].append('excessive_questioning')
-        
-        # High personal pronoun usage (false intimacy)
-        if analysis['linguistic_patterns']['personal_pronouns']['ratio'] > 0.15:
-            manipulation_score += 0.2
-            indicators['specific_indicators'].append('false_intimacy')
-        
-        indicators['linguistic_manipulation_score'] = min(1.0, manipulation_score)
-        
-        # Overall risk assessment
-        if manipulation_score > 0.7:
-            indicators['overall_risk'] = 'high'
-        elif manipulation_score > 0.4:
-            indicators['overall_risk'] = 'medium'
-        else:
-            indicators['overall_risk'] = 'low'
-        
-        return indicators
+        return valid_df, validation_summary
+    
+    def prepare_datasets(self) -> Tuple[bool, str]:
+        """Main method to prepare all datasets with enhanced feature analysis"""
+        logger.info("Starting enhanced dataset preparation with feature analysis...")
+        
+        try:
+            # Load and validate datasets with enhanced features
+            kaggle_result = self.load_kaggle_dataset()
+            liar_result = self.load_liar_dataset()
+            
+            # Handle None returns gracefully
+            if kaggle_result is None:
+                logger.warning("Kaggle dataset loading failed")
+                kaggle_df, kaggle_validation = pd.DataFrame(), {
+                    'source': 'kaggle_combined', 'original_count': 0, 'valid_count': 0, 
+                    'success_rate': 0, 'overall_quality_score': 0, 'validation_timestamp': datetime.now().isoformat(),
+                    'enhanced_features': {'enhanced_analysis_performed': False}
+                }
+            else:
+                kaggle_df, kaggle_validation = kaggle_result
+            
+            if liar_result is None:
+                logger.warning("LIAR dataset loading failed")
+                liar_df, liar_validation = pd.DataFrame(), {
+                    'source': 'liar_combined', 'original_count': 0, 'valid_count': 0, 
+                    'success_rate': 0, 'overall_quality_score': 0, 'validation_timestamp': datetime.now().isoformat(),
+                    'enhanced_features': {'enhanced_analysis_performed': False}
+                }
+            else:
+                liar_df, liar_validation = liar_result
+            
+            # Combine datasets
+            datasets_to_combine = [df for df in [kaggle_df, liar_df] if not df.empty]
+            
+            if not datasets_to_combine:
+                return False, "No datasets could be loaded and validated"
+            
+            combined_df = pd.concat(datasets_to_combine, ignore_index=True)
+            
+            # Final enhanced validation and feature analysis
+            if self.enable_feature_analysis:
+                logger.info("Performing final enhanced validation on combined dataset...")
+                combined_df = self.validate_dataset(combined_df)