import pandas as pd
import re
import logging

# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

class JobTitlePreprocessor:
    """Preprocesses job titles by converting to lowercase, removing unwanted words,
    special characters, numbers greater than 10, and content from location, states, regions, etc."""

    def __init__(self):
        # Define unwanted words and initialize counters
        self.unwanted_words = ['remote', 'hybrid', 'flexible location', 'location', 'open to work', 
                               'role', 'job', 'level', 'remot']
        self.location_removed_count = 0
        self.unwanted_words_removed_count = 0
        self.brackets_removed_count = 0
        self.state_region_removed_count = 0
        self.numbers_removed_count = 0

    def remove_location_unwanted_words_brackets(self, row):
        """Removes parts of the title based on location, unwanted words, bracketed content,
        numbers greater than 10, and also removes symbols other than alphanumeric."""
        title = row['titles_title']
        location = row['LOCATION']
        states = row.get('STATES', '')  # Get values from 'STATES' column if present
        region_state = row.get('REGION_STATE', '')  # Get values from 'REGION_STATE' column if present
        county = row.get('COUNTY', '')  # Get values from 'COUNTY' column if present
        city = row.get('city', '')  # Get values from 'city' column if present

        # Ensure title is a string
        if isinstance(title, str):
            # Remove location if present in the title
            if isinstance(location, str) and re.search(r'\b{}\b'.format(re.escape(location)), title, flags=re.IGNORECASE):
                title = re.sub(r'\b{}\b'.format(re.escape(location)), '', title, flags=re.IGNORECASE)
                self.location_removed_count += 1
            
            # Remove unwanted words
            for word in self.unwanted_words:
                pattern = r'\b{}\b'.format(re.escape(word))
                if re.search(pattern, title, flags=re.IGNORECASE):
                    title = re.sub(pattern, '', title, flags=re.IGNORECASE)
                    self.unwanted_words_removed_count += 1

            # Remove content from STATES, REGION_STATE, COUNTY, and city
            for region in [states, region_state, county, city]:
                if isinstance(region, str) and re.search(r'\b{}\b'.format(re.escape(region)), title, flags=re.IGNORECASE):
                    title = re.sub(r'\b{}\b'.format(re.escape(region)), '', title, flags=re.IGNORECASE)
                    self.state_region_removed_count += 1

            # Remove content within brackets
            if re.search(r'\[.*?\]|\(.*?\)|\{.*?\}', title):
                title = re.sub(r'\[.*?\]|\(.*?\)|\{.*?\}', '', title)
                self.brackets_removed_count += 1

            # Remove any non-alphanumeric characters (keeping spaces)
            title = re.sub(r'[^a-zA-Z0-9\s]', '', title)

            # Remove numbers greater than 10
            if re.search(r'\b(?:[1-9][0-9]+|1[1-9]|[2-9][0-9])\b', title):
                title = re.sub(r'\b(?:[1-9][0-9]+|1[1-9]|[2-9][0-9])\b', '', title)
                self.numbers_removed_count += 1

            # Clean up extra spaces
            title = re.sub(r'\s+', ' ', title).strip()

        return title

    def preprocess(self, title: str) -> str:
        """Converts title to lowercase, removes unwanted words, replaces specific terms,
        and standardizes job titles."""
        if not isinstance(title, str):
            return title
        
        # Convert to lowercase
        title = title.lower()
        
        # Replace specific terms and Roman numerals
        replacements = [
            (r'\b(?:SR|sr|Sr\.?|SR\.?|Senior|senior)\b', 'Senior'),
            (r'\b(?:JR|jr|Jr\.?|JR\.?|Junior|junior)\b', 'Junior'),
            (r'\b(?:AIML|aiml|ML|ml|MachineLearning|machinelearning|machine[_\-]learning)\b', 'Machine Learning'),
            (r'\b(?:GenAI|genai|Genai|generative[_\-]ai|GenerativeAI|generativeai)\b', 'Generative AI'),
            (r'\b(?:NLP|nlp|natural[_\-]language[_\-]processing|natural language processing)\b', 'NLP'),  
            (r'\b(?:i|I)\b', '1'),
            (r'\b(?:ii|II)\b', '2'),
            (r'\b(?:iii|III)\b', '3'),
            (r'\b(?:iv|IV)\b', '4'),
            (r'\b(?:v|V)\b', '5')
        ]
        
        for pattern, replacement in replacements:
            title = re.sub(pattern, replacement, title, flags=re.IGNORECASE)

        # Handle specific Data Scientist cases
        title = re.sub(r'\b(director|dir\.?|dir)\b.*?(data\s*scientist|data\s*science)', 'Director Data Scientist', title, flags=re.IGNORECASE)
        title = re.sub(r'\b(manager|mgr)\b.*?(data\s*scientist|data\s*science)', 'Manager Data Scientist', title, flags=re.IGNORECASE)
        title = re.sub(r'\b(lead)\b.*?(data\s*scientist|data\s*science)', 'Lead Data Scientist', title, flags=re.IGNORECASE)
        title = re.sub(r'\b(associate|associates?)\b.*?(data\s*scientist|data\s*science)', 'Associate Data Scientist', title, flags=re.IGNORECASE)
        title = re.sub(r'\b(applied)\b.*?(data\s*scientist|data\s*science)', 'Applied Data Scientist', title, flags=re.IGNORECASE)
        title = re.sub(r'\b(intern|internship|trainee)\b.*?(data\s*scientist|data\s*science)', 'Intern Data Scientist', title, flags=re.IGNORECASE)

        # Ensure "ML" or "NLP" is retained if found in the title
        if re.search(r'\bdata\s*scientist\b', title, flags=re.IGNORECASE):
            if re.search(r'\b(?:ai|artificial intelligence|ml|machine learning|deep learning|dl)\b', title, flags=re.IGNORECASE):
                return 'ML Data Scientist'
            elif re.search(r'\b(?:nlp|natural language processing)\b', title, flags=re.IGNORECASE):
                return 'NLP Data Scientist'
            return title

        # Clean up extra spaces
        title = re.sub(r'\s+', ' ', title).strip()

        return title
        
def is_title_empty(row):
    """
    Check if the 'titles_title' is effectively empty, which includes
    strings that are either empty or contain only whitespace.
    """
    title = row['titles_title']
    return pd.isna(title) or (isinstance(title, str) and title.strip() == '')

def main_preprocessing():
    try:
        # Load the dataset
        df = pd.read_csv(r"Struct Data_Data Science 100K.csv", low_memory=False)
        
        # Initialize preprocessor
        preprocessor = JobTitlePreprocessor()
        
        # Apply both the removal and standard preprocessing steps
        df['clean_title'] = df.apply(preprocessor.remove_location_unwanted_words_brackets, axis=1)
        df['clean_title'] = df['clean_title'].apply(preprocessor.preprocess)
        
        # Remove rows where 'titles_title' is empty or contains only whitespace
        df = df[~df.apply(is_title_empty, axis=1)]
        
        # Drop rows where 'clean_title' is NaN
        df = df.dropna(subset=['clean_title'])
        
        # Log some information about the dataset
        logger.info(f"Original dataset shape: {df.shape}")
        logger.info(f"Number of non-empty titles: {df['clean_title'].notna().sum()}")
        
        # Save the preprocessed data
        output_df = df[['titles_title', 'clean_title']]
        output_df.to_csv('preprocessed_job_titles.csv', index=False)
        
        logger.info(f"Preprocessed dataset shape: {output_df.shape}")
        logger.info("Job title preprocessing completed successfully.")
        logger.info(f"Total rows with part of location removed from titles: {preprocessor.location_removed_count}")
        logger.info(f"Total unwanted words removed: {preprocessor.unwanted_words_removed_count}")
        logger.info(f"Total brackets removed: {preprocessor.brackets_removed_count}")
        logger.info(f"Total states/regions removed: {preprocessor.state_region_removed_count}")
        logger.info(f"Total numbers removed: {preprocessor.numbers_removed_count}")
        
    except Exception as e:
        logger.error(f"An error occurred during preprocessing: {e}")

if __name__ == "__main__":
    main_preprocessing()