Update features/feature_engineer.py

features/feature_engineer.py

@@ -1,4 +1,5 @@
-# Fixed features/feature_engineer.py addressing the IndexError and regex issues
+# File: features/feature_engineer.py
+# Enhanced Feature Engineering Pipeline for Priority 6
 
 import json
 import joblib
@@ -19,16 +20,11 @@ from sklearn.preprocessing import StandardScaler, FunctionTransformer
 import warnings
 warnings.filterwarnings('ignore')
 
-# Import feature analyzers with error handling
-try:
-    from features.sentiment_analyzer import SentimentAnalyzer
-    from features.readability_analyzer import ReadabilityAnalyzer
-    from features.entity_analyzer import EntityAnalyzer
-    from features.linguistic_analyzer import LinguisticAnalyzer
-    FEATURE_ANALYZERS_AVAILABLE = True
-except ImportError:
-    FEATURE_ANALYZERS_AVAILABLE = False
-    logging.warning("Advanced feature analyzers not available - using basic features only")
+# Import feature analyzers
+from features.sentiment_analyzer import SentimentAnalyzer
+from features.readability_analyzer import ReadabilityAnalyzer
+from features.entity_analyzer import EntityAnalyzer
+from features.linguistic_analyzer import LinguisticAnalyzer
 
 # Configure logging
 logging.basicConfig(level=logging.INFO)
@@ -37,7 +33,8 @@ logger = logging.getLogger(__name__)
 
 class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
     """
-    Fixed advanced feature engineering pipeline with proper error handling
+    Advanced feature engineering pipeline combining multiple NLP feature extractors
+    for enhanced fake news detection performance.
     """
     
     def __init__(self, 
@@ -52,39 +49,33 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
                  max_df: float = 0.95):
         """
         Initialize the advanced feature engineering pipeline.
+        
+        Args:
+            enable_sentiment: Enable sentiment analysis features
+            enable_readability: Enable readability/complexity features
+            enable_entities: Enable named entity recognition features
+            enable_linguistic: Enable advanced linguistic features
+            feature_selection_k: Number of features to select
+            tfidf_max_features: Maximum TF-IDF features
+            ngram_range: N-gram range for TF-IDF
+            min_df: Minimum document frequency for TF-IDF
+            max_df: Maximum document frequency for TF-IDF
         """
-        self.enable_sentiment = enable_sentiment and FEATURE_ANALYZERS_AVAILABLE
-        self.enable_readability = enable_readability and FEATURE_ANALYZERS_AVAILABLE
-        self.enable_entities = enable_entities and FEATURE_ANALYZERS_AVAILABLE
-        self.enable_linguistic = enable_linguistic and FEATURE_ANALYZERS_AVAILABLE
+        self.enable_sentiment = enable_sentiment
+        self.enable_readability = enable_readability
+        self.enable_entities = enable_entities
+        self.enable_linguistic = enable_linguistic
         self.feature_selection_k = feature_selection_k
         self.tfidf_max_features = tfidf_max_features
         self.ngram_range = ngram_range
         self.min_df = min_df
         self.max_df = max_df
         
-        # Initialize feature extractors only if available
-        self.sentiment_analyzer = None
-        self.readability_analyzer = None
-        self.entity_analyzer = None
-        self.linguistic_analyzer = None
-        
-        if FEATURE_ANALYZERS_AVAILABLE:
-            try:
-                if self.enable_sentiment:
-                    self.sentiment_analyzer = SentimentAnalyzer()
-                if self.enable_readability:
-                    self.readability_analyzer = ReadabilityAnalyzer()
-                if self.enable_entities:
-                    self.entity_analyzer = EntityAnalyzer()
-                if self.enable_linguistic:
-                    self.linguistic_analyzer = LinguisticAnalyzer()
-            except Exception as e:
-                logger.warning(f"Failed to initialize feature analyzers: {e}")
-                self.sentiment_analyzer = None
-                self.readability_analyzer = None
-                self.entity_analyzer = None
-                self.linguistic_analyzer = None
+        # Initialize feature extractors
+        self.sentiment_analyzer = SentimentAnalyzer() if enable_sentiment else None
+        self.readability_analyzer = ReadabilityAnalyzer() if enable_readability else None
+        self.entity_analyzer = EntityAnalyzer() if enable_entities else None
+        self.linguistic_analyzer = LinguisticAnalyzer() if enable_linguistic else None
         
         # Initialize TF-IDF components
         self.tfidf_vectorizer = None
@@ -98,7 +89,11 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
         
     def fit(self, X, y=None):
         """
-        Fit the feature engineering pipeline with proper error handling.
+        Fit the feature engineering pipeline.
+        
+        Args:
+            X: Text data (array-like of strings)
+            y: Target labels (optional, for supervised feature selection)
         """
         logger.info("Fitting advanced feature engineering pipeline...")
         
@@ -112,61 +107,41 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
         if len(X) == 0:
             raise ValueError("Cannot fit on empty data")
         
-        # Initialize TF-IDF vectorizer with safer parameters
-        actual_max_features = min(self.tfidf_max_features, len(X) * 10)
-        
+        # Initialize TF-IDF vectorizer
         self.tfidf_vectorizer = TfidfVectorizer(
-            max_features=actual_max_features,
+            max_features=self.tfidf_max_features,
             ngram_range=self.ngram_range,
-            min_df=max(1, min(self.min_df, len(X) // 10)),
+            min_df=self.min_df,
             max_df=self.max_df,
             stop_words='english',
             sublinear_tf=True,
             norm='l2',
-            lowercase=True,
-            token_pattern=r'\b[a-zA-Z][a-zA-Z]+\b'  # Fix regex pattern
+            lowercase=True
         )
         
         # Fit TF-IDF on text data
         logger.info("Fitting TF-IDF vectorizer...")
-        try:
-            tfidf_features = self.tfidf_vectorizer.fit_transform(X)
-            logger.info(f"TF-IDF features shape: {tfidf_features.shape}")
-        except Exception as e:
-            logger.error(f"TF-IDF fitting failed: {e}")
-            # Fallback to very basic TF-IDF
-            self.tfidf_vectorizer = TfidfVectorizer(
-                max_features=min(1000, len(X) * 5),
-                stop_words='english',
-                lowercase=True
-            )
-            tfidf_features = self.tfidf_vectorizer.fit_transform(X)
-            logger.info(f"Fallback TF-IDF features shape: {tfidf_features.shape}")
+        tfidf_features = self.tfidf_vectorizer.fit_transform(X)
         
-        # Extract additional features with error handling
+        # Extract additional features
         additional_features = self._extract_additional_features(X, fit=True)
         
         # Combine all features
         if additional_features.shape[1] > 0:
-            try:
-                all_features = hstack([tfidf_features, additional_features])
-            except Exception as e:
-                logger.warning(f"Failed to combine features, using TF-IDF only: {e}")
-                all_features = tfidf_features
-                additional_features = np.empty((len(X), 0))
+            all_features = hstack([tfidf_features, additional_features])
         else:
             all_features = tfidf_features
             
         logger.info(f"Total features before selection: {all_features.shape[1]}")
         
-        # Feature selection with proper bounds checking
+        # Feature selection
         if y is not None and self.feature_selection_k < all_features.shape[1]:
-            actual_k = min(self.feature_selection_k, all_features.shape[1] - 1)
-            logger.info(f"Performing feature selection (k={actual_k})...")
+            logger.info(f"Performing feature selection (k={self.feature_selection_k})...")
             
+            # Use chi2 for text features and mutual information for numerical features
             self.feature_selector = SelectKBest(
                 score_func=chi2, 
-                k=actual_k
+                k=min(self.feature_selection_k, all_features.shape[1])
             )
             
             # Ensure non-negative features for chi2
@@ -178,30 +153,22 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
             # Make features non-negative for chi2
             features_dense = np.maximum(features_dense, 0)
             
-            try:
-                self.feature_selector.fit(features_dense, y)
-                selected_features = self.feature_selector.transform(features_dense)
-                logger.info(f"Selected {selected_features.shape[1]} features")
-            except Exception as e:
-                logger.warning(f"Feature selection failed: {e}, using all features")
-                self.feature_selector = None
-                selected_features = all_features
+            self.feature_selector.fit(features_dense, y)
+            selected_features = self.feature_selector.transform(features_dense)
+            
+            logger.info(f"Selected {selected_features.shape[1]} features")
         else:
             selected_features = all_features
             
         # Scale numerical features (additional features only)
         if additional_features.shape[1] > 0:
             self.feature_scaler = StandardScaler()
-            try:
-                # Only scale the additional features part
-                additional_selected = selected_features[:, -additional_features.shape[1]:]
-                self.feature_scaler.fit(additional_selected)
-            except Exception as e:
-                logger.warning(f"Feature scaling failed: {e}")
-                self.feature_scaler = None
+            # Only scale the additional features part
+            additional_selected = selected_features[:, -additional_features.shape[1]:]
+            self.feature_scaler.fit(additional_selected)
         
-        # Generate feature names with proper bounds checking
-        self._generate_feature_names_safe()
+        # Generate feature names
+        self._generate_feature_names()
         
         # Calculate feature importance if possible
         if y is not None and self.feature_selector is not None:
@@ -215,6 +182,12 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
     def transform(self, X):
         """
         Transform text data into enhanced feature vectors.
+        
+        Args:
+            X: Text data (array-like of strings)
+            
+        Returns:
+            Transformed feature matrix
         """
         if not self.is_fitted_:
             raise ValueError("Pipeline must be fitted before transforming")
@@ -226,62 +199,42 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
             X = np.array(X)
         
         # Extract TF-IDF features
-        try:
-            tfidf_features = self.tfidf_vectorizer.transform(X)
-        except Exception as e:
-            logger.error(f"TF-IDF transform failed: {e}")
-            # Return minimal features if transform fails
-            return np.zeros((len(X), len(self.feature_names_) if self.feature_names_ else 100))
+        tfidf_features = self.tfidf_vectorizer.transform(X)
         
         # Extract additional features
         additional_features = self._extract_additional_features(X, fit=False)
         
         # Combine features
         if additional_features.shape[1] > 0:
-            try:
-                all_features = hstack([tfidf_features, additional_features])
-            except Exception as e:
-                logger.warning(f"Failed to combine features in transform: {e}")
-                all_features = tfidf_features
+            all_features = hstack([tfidf_features, additional_features])
         else:
             all_features = tfidf_features
         
         # Apply feature selection
         if self.feature_selector is not None:
-            try:
-                if hasattr(all_features, 'toarray'):
-                    features_dense = all_features.toarray()
-                else:
-                    features_dense = all_features
-                    
-                # Ensure non-negative for consistency
-                features_dense = np.maximum(features_dense, 0)
-                selected_features = self.feature_selector.transform(features_dense)
-            except Exception as e:
-                logger.warning(f"Feature selection failed in transform: {e}")
-                selected_features = all_features
+            if hasattr(all_features, 'toarray'):
+                features_dense = all_features.toarray()
+            else:
+                features_dense = all_features
+                
+            # Ensure non-negative for consistency
+            features_dense = np.maximum(features_dense, 0)
+            selected_features = self.feature_selector.transform(features_dense)
         else:
             selected_features = all_features
         
         # Scale additional features if scaler exists
         if self.feature_scaler is not None and additional_features.shape[1] > 0:
-            try:
-                # Scale only the additional features part
-                tfidf_selected = selected_features[:, :-additional_features.shape[1]]
-                additional_selected = selected_features[:, -additional_features.shape[1]:]
-                additional_scaled = self.feature_scaler.transform(additional_selected)
-                
-                # Combine back
-                if hasattr(tfidf_selected, 'toarray'):
-                    tfidf_selected = tfidf_selected.toarray()
-                
-                final_features = np.hstack([tfidf_selected, additional_scaled])
-            except Exception as e:
-                logger.warning(f"Feature scaling failed in transform: {e}")
-                if hasattr(selected_features, 'toarray'):
-                    final_features = selected_features.toarray()
-                else:
-                    final_features = selected_features
+            # Scale only the additional features part
+            tfidf_selected = selected_features[:, :-additional_features.shape[1]]
+            additional_selected = selected_features[:, -additional_features.shape[1]:]
+            additional_scaled = self.feature_scaler.transform(additional_selected)
+            
+            # Combine back
+            if hasattr(tfidf_selected, 'toarray'):
+                tfidf_selected = tfidf_selected.toarray()
+            
+            final_features = np.hstack([tfidf_selected, additional_scaled])
         else:
             if hasattr(selected_features, 'toarray'):
                 final_features = selected_features.toarray()
@@ -291,64 +244,45 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
         return final_features
     
     def _extract_additional_features(self, X, fit=False):
-        """Extract additional features with comprehensive error handling"""
+        """Extract additional features beyond TF-IDF"""
         feature_arrays = []
         
         try:
-            # Basic text features (always available)
-            basic_features = self._extract_basic_features(X)
-            if basic_features.shape[1] > 0:
-                feature_arrays.append(basic_features)
+            # Sentiment features
+            if self.sentiment_analyzer is not None:
+                logger.info("Extracting sentiment features...")
+                if fit:
+                    sentiment_features = self.sentiment_analyzer.fit_transform(X)
+                else:
+                    sentiment_features = self.sentiment_analyzer.transform(X)
+                feature_arrays.append(sentiment_features)
             
-            # Advanced features (only if analyzers available)
-            if FEATURE_ANALYZERS_AVAILABLE:
-                # Sentiment features
-                if self.sentiment_analyzer is not None:
-                    logger.info("Extracting sentiment features...")
-                    try:
-                        if fit:
-                            sentiment_features = self.sentiment_analyzer.fit_transform(X)
-                        else:
-                            sentiment_features = self.sentiment_analyzer.transform(X)
-                        feature_arrays.append(sentiment_features)
-                    except Exception as e:
-                        logger.warning(f"Sentiment analysis failed: {e}")
-                
-                # Readability features
-                if self.readability_analyzer is not None:
-                    logger.info("Extracting readability features...")
-                    try:
-                        if fit:
-                            readability_features = self.readability_analyzer.fit_transform(X)
-                        else:
-                            readability_features = self.readability_analyzer.transform(X)
-                        feature_arrays.append(readability_features)
-                    except Exception as e:
-                        logger.warning(f"Readability analysis failed: {e}")
-                
-                # Entity features
-                if self.entity_analyzer is not None:
-                    logger.info("Extracting entity features...")
-                    try:
-                        if fit:
-                            entity_features = self.entity_analyzer.fit_transform(X)
-                        else:
-                            entity_features = self.entity_analyzer.transform(X)
-                        feature_arrays.append(entity_features)
-                    except Exception as e:
-                        logger.warning(f"Entity analysis failed: {e}")
-                
-                # Linguistic features
-                if self.linguistic_analyzer is not None:
-                    logger.info("Extracting linguistic features...")
-                    try:
-                        if fit:
-                            linguistic_features = self.linguistic_analyzer.fit_transform(X)
-                        else:
-                            linguistic_features = self.linguistic_analyzer.transform(X)
-                        feature_arrays.append(linguistic_features)
-                    except Exception as e:
-                        logger.warning(f"Linguistic analysis failed: {e}")
+            # Readability features
+            if self.readability_analyzer is not None:
+                logger.info("Extracting readability features...")
+                if fit:
+                    readability_features = self.readability_analyzer.fit_transform(X)
+                else:
+                    readability_features = self.readability_analyzer.transform(X)
+                feature_arrays.append(readability_features)
+            
+            # Entity features
+            if self.entity_analyzer is not None:
+                logger.info("Extracting entity features...")
+                if fit:
+                    entity_features = self.entity_analyzer.fit_transform(X)
+                else:
+                    entity_features = self.entity_analyzer.transform(X)
+                feature_arrays.append(entity_features)
+            
+            # Linguistic features
+            if self.linguistic_analyzer is not None:
+                logger.info("Extracting linguistic features...")
+                if fit:
+                    linguistic_features = self.linguistic_analyzer.fit_transform(X)
+                else:
+                    linguistic_features = self.linguistic_analyzer.transform(X)
+                feature_arrays.append(linguistic_features)
             
             # Combine all additional features
             if feature_arrays:
@@ -363,130 +297,51 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
         
         return additional_features
     
-    def _extract_basic_features(self, X):
-        """Extract basic text features that don't require external libraries"""
-        features = []
+    def _generate_feature_names(self):
+        """Generate comprehensive feature names"""
+        self.feature_names_ = []
         
-        for text in X:
-            text_str = str(text)
-            
-            # Basic text statistics
-            word_count = len(text_str.split())
-            char_count = len(text_str)
-            sentence_count = text_str.count('.') + text_str.count('!') + text_str.count('?')
-            sentence_count = max(1, sentence_count)  # Avoid division by zero
-            
-            # Basic ratios
-            avg_word_length = char_count / max(word_count, 1)
-            avg_sentence_length = word_count / sentence_count
-            
-            # Punctuation features
-            exclamation_count = text_str.count('!')
-            question_count = text_str.count('?')
-            uppercase_ratio = sum(1 for c in text_str if c.isupper()) / max(len(text_str), 1)
-            
-            # Feature vector
-            feature_vector = [
-                word_count,
-                char_count,
-                sentence_count,
-                avg_word_length,
-                avg_sentence_length,
-                exclamation_count,
-                question_count,
-                uppercase_ratio
-            ]
-            
-            features.append(feature_vector)
+        # TF-IDF feature names
+        if self.tfidf_vectorizer is not None:
+            tfidf_names = [f"tfidf_{name}" for name in self.tfidf_vectorizer.get_feature_names_out()]
+            self.feature_names_.extend(tfidf_names)
         
-        return np.array(features)
-    
-    def _generate_feature_names_safe(self):
-        """Generate feature names with proper bounds checking"""
-        self.feature_names_ = []
+        # Additional feature names
+        if self.sentiment_analyzer is not None:
+            self.feature_names_.extend(self.sentiment_analyzer.get_feature_names())
         
-        try:
-            # TF-IDF feature names
-            if self.tfidf_vectorizer is not None:
-                tfidf_names = [f"tfidf_{name}" for name in self.tfidf_vectorizer.get_feature_names_out()]
-                self.feature_names_.extend(tfidf_names)
+        if self.readability_analyzer is not None:
+            self.feature_names_.extend(self.readability_analyzer.get_feature_names())
+        
+        if self.entity_analyzer is not None:
+            self.feature_names_.extend(self.entity_analyzer.get_feature_names())
+        
+        if self.linguistic_analyzer is not None:
+            self.feature_names_.extend(self.linguistic_analyzer.get_feature_names())
+        
+        # Apply feature selection to names if applicable
+        if self.feature_selector is not None:
+            selected_indices = self.feature_selector.get_support()
+            self.feature_names_ = [name for i, name in enumerate(self.feature_names_) if selected_indices[i]]
+    
+    def _calculate_feature_importance(self):
+        """Calculate feature importance scores"""
+        if self.feature_selector is not None:
+            scores = self.feature_selector.scores_
+            selected_indices = self.feature_selector.get_support()
             
-            # Basic feature names
-            basic_feature_names = [
-                'word_count', 'char_count', 'sentence_count', 
-                'avg_word_length', 'avg_sentence_length', 
-                'exclamation_count', 'question_count', 'uppercase_ratio'
-            ]
-            self.feature_names_.extend([f'basic_{name}' for name in basic_feature_names])
+            # Get scores for selected features
+            selected_scores = scores[selected_indices]
             
-            # Advanced feature names (only if available)
-            if FEATURE_ANALYZERS_AVAILABLE:
-                if self.sentiment_analyzer is not None:
-                    try:
-                        self.feature_names_.extend(self.sentiment_analyzer.get_feature_names())
-                    except:
-                        self.feature_names_.extend(['sentiment_compound', 'sentiment_pos', 'sentiment_neg', 'sentiment_neu'])
-                
-                if self.readability_analyzer is not None:
-                    try:
-                        self.feature_names_.extend(self.readability_analyzer.get_feature_names())
-                    except:
-                        self.feature_names_.extend(['readability_score', 'reading_ease'])
-                
-                if self.entity_analyzer is not None:
-                    try:
-                        self.feature_names_.extend(self.entity_analyzer.get_feature_names())
-                    except:
-                        self.feature_names_.extend(['entity_person', 'entity_org', 'entity_loc'])
-                
-                if self.linguistic_analyzer is not None:
-                    try:
-                        self.feature_names_.extend(self.linguistic_analyzer.get_feature_names())
-                    except:
-                        self.feature_names_.extend(['linguistic_complexity', 'pos_diversity'])
+            # Create importance dictionary
+            self.feature_importance_ = {
+                name: float(score) for name, score in zip(self.feature_names_, selected_scores)
+            }
             
-            # Apply feature selection to names if applicable
-            if self.feature_selector is not None:
-                try:
-                    selected_indices = self.feature_selector.get_support()
-                    # FIX: Ensure bounds checking
-                    if len(selected_indices) == len(self.feature_names_):
-                        self.feature_names_ = [name for i, name in enumerate(self.feature_names_) if selected_indices[i]]
-                    else:
-                        logger.warning(f"Feature selection indices mismatch: {len(selected_indices)} vs {len(self.feature_names_)}")
-                        # Keep original names if mismatch
-                except Exception as e:
-                    logger.warning(f"Failed to apply feature selection to names: {e}")
-                    
-        except Exception as e:
-            logger.warning(f"Failed to generate feature names: {e}")
-            # Generate generic names
-            self.feature_names_ = [f'feature_{i}' for i in range(100)]  # Default fallback
-    
-    def _calculate_feature_importance(self):
-        """Calculate feature importance scores with error handling"""
-        try:
-            if self.feature_selector is not None and hasattr(self.feature_selector, 'scores_'):
-                scores = self.feature_selector.scores_
-                selected_indices = self.feature_selector.get_support()
-                
-                # Get scores for selected features
-                selected_scores = scores[selected_indices]
-                
-                # Create importance dictionary with bounds checking
-                if len(selected_scores) == len(self.feature_names_):
-                    self.feature_importance_ = {
-                        name: float(score) for name, score in zip(self.feature_names_, selected_scores)
-                    }
-                    
-                    # Sort by importance
-                    self.feature_importance_ = dict(
-                        sorted(self.feature_importance_.items(), key=lambda x: x[1], reverse=True)
-                    )
-                else:
-                    logger.warning("Feature importance calculation failed due to size mismatch")
-        except Exception as e:
-            logger.warning(f"Feature importance calculation failed: {e}")
+            # Sort by importance
+            self.feature_importance_ = dict(
+                sorted(self.feature_importance_.items(), key=lambda x: x[1], reverse=True)
+            )
     
     def get_feature_names(self):
         """Get names of output features"""
@@ -509,19 +364,15 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
         if not self.is_fitted_:
             raise ValueError("Pipeline must be fitted first")
         
-        # Count feature types safely
-        feature_type_counts = {
-            'tfidf_features': sum(1 for name in self.feature_names_ if name.startswith('tfidf_')),
-            'basic_features': sum(1 for name in self.feature_names_ if name.startswith('basic_')),
-            'sentiment_features': sum(1 for name in self.feature_names_ if 'sentiment' in name),
-            'readability_features': sum(1 for name in self.feature_names_ if 'readability' in name),
-            'entity_features': sum(1 for name in self.feature_names_ if 'entity' in name),
-            'linguistic_features': sum(1 for name in self.feature_names_ if 'linguistic' in name)
-        }
-        
         metadata = {
             'total_features': len(self.feature_names_),
-            'feature_types': feature_type_counts,
+            'feature_types': {
+                'tfidf_features': sum(1 for name in self.feature_names_ if name.startswith('tfidf_')),
+                'sentiment_features': sum(1 for name in self.feature_names_ if name.startswith('sentiment_')),
+                'readability_features': sum(1 for name in self.feature_names_ if name.startswith('readability_')),
+                'entity_features': sum(1 for name in self.feature_names_ if name.startswith('entity_')),
+                'linguistic_features': sum(1 for name in self.feature_names_ if name.startswith('linguistic_'))
+            },
             'configuration': {
                 'enable_sentiment': self.enable_sentiment,
                 'enable_readability': self.enable_readability,
@@ -529,17 +380,39 @@ class AdvancedFeatureEngineer(BaseEstimator, TransformerMixin):
                 'enable_linguistic': self.enable_linguistic,
                 'feature_selection_k': self.feature_selection_k,
                 'tfidf_max_features': self.tfidf_max_features,
-                'ngram_range': self.ngram_range,
-                'analyzers_available': FEATURE_ANALYZERS_AVAILABLE
+                'ngram_range': self.ngram_range
             },
             'feature_importance_available': bool(self.feature_importance_),
             'timestamp': datetime.now().isoformat()
         }
         
         return metadata
+    
+    def save_pipeline(self, filepath):
+        """Save the fitted pipeline"""
+        if not self.is_fitted_:
+            raise ValueError("Pipeline must be fitted before saving")
+        
+        save_data = {
+            'feature_engineer': self,
+            'metadata': self.get_feature_metadata(),
+            'feature_names': self.feature_names_,
+            'feature_importance': self.feature_importance_
+        }
+        
+        joblib.dump(save_data, filepath)
+        logger.info(f"Feature engineering pipeline saved to {filepath}")
+    
+    @classmethod
+    def load_pipeline(cls, filepath):
+        """Load a fitted pipeline"""
+        save_data = joblib.load(filepath)
+        feature_engineer = save_data['feature_engineer']
+        
+        logger.info(f"Feature engineering pipeline loaded from {filepath}")
+        return feature_engineer
 
 
-# Convenience functions remain the same...
 def create_enhanced_pipeline(X_train, y_train, 
                            enable_sentiment=True,
                            enable_readability=True, 
@@ -548,18 +421,28 @@ def create_enhanced_pipeline(X_train, y_train,
                            feature_selection_k=5000):
     """
     Create and fit an enhanced feature engineering pipeline.
+    
+    Args:
+        X_train: Training text data
+        y_train: Training labels
+        enable_sentiment: Enable sentiment analysis features
+        enable_readability: Enable readability features
+        enable_entities: Enable entity features
+        enable_linguistic: Enable linguistic features
+        feature_selection_k: Number of features to select
+        
+    Returns:
+        Fitted AdvancedFeatureEngineer instance
     """
     logger.info("Creating enhanced feature engineering pipeline...")
     
-    # Create feature engineer with reduced complexity for stability
+    # Create feature engineer
     feature_engineer = AdvancedFeatureEngineer(
-        enable_sentiment=enable_sentiment and FEATURE_ANALYZERS_AVAILABLE,
-        enable_readability=enable_readability and FEATURE_ANALYZERS_AVAILABLE,
-        enable_entities=enable_entities and FEATURE_ANALYZERS_AVAILABLE,
-        enable_linguistic=enable_linguistic and FEATURE_ANALYZERS_AVAILABLE,
-        feature_selection_k=min(feature_selection_k, len(X_train) * 2),  # Safer default
-        tfidf_max_features=min(10000, len(X_train) * 5),  # Safer default
-        ngram_range=(1, 2)  # Reduced complexity
+        enable_sentiment=enable_sentiment,
+        enable_readability=enable_readability,
+        enable_entities=enable_entities,
+        enable_linguistic=enable_linguistic,
+        feature_selection_k=feature_selection_k
     )
     
     # Fit the pipeline
@@ -570,4 +453,53 @@ def create_enhanced_pipeline(X_train, y_train,
     logger.info(f"Enhanced pipeline created with {metadata['total_features']} features")
     logger.info(f"Feature breakdown: {metadata['feature_types']}")
     
-    return feature_engineer
+    return feature_engineer
+
+
+def analyze_feature_importance(feature_engineer, top_k=20):
+    """
+    Analyze and display feature importance.
+    
+    Args:
+        feature_engineer: Fitted AdvancedFeatureEngineer instance
+        top_k: Number of top features to analyze
+        
+    Returns:
+        Dictionary with feature analysis results
+    """
+    if not feature_engineer.is_fitted_:
+        raise ValueError("Feature engineer must be fitted first")
+    
+    # Get feature importance
+    importance = feature_engineer.get_feature_importance(top_k=top_k)
+    metadata = feature_engineer.get_feature_metadata()
+    
+    # Analyze feature types in top features
+    top_features = list(importance.keys())
+    feature_type_counts = {}
+    
+    for feature in top_features:
+        if feature.startswith('tfidf_'):
+            feature_type = 'tfidf'
+        elif feature.startswith('sentiment_'):
+            feature_type = 'sentiment'
+        elif feature.startswith('readability_'):
+            feature_type = 'readability'
+        elif feature.startswith('entity_'):
+            feature_type = 'entity'
+        elif feature.startswith('linguistic_'):
+            feature_type = 'linguistic'
+        else:
+            feature_type = 'other'
+        
+        feature_type_counts[feature_type] = feature_type_counts.get(feature_type, 0) + 1
+    
+    analysis = {
+        'top_features': importance,
+        'feature_type_distribution': feature_type_counts,
+        'total_features': metadata['total_features'],
+        'feature_breakdown': metadata['feature_types'],
+        'analysis_timestamp': datetime.now().isoformat()
+    }
+    
+    return analysis