Spaces:
Sleeping
Sleeping
| import pandas as pd | |
| import re | |
| import logging | |
| # Configure logging | |
| logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') | |
| logger = logging.getLogger(__name__) | |
| class JobTitlePreprocessor: | |
| """Preprocesses job titles by converting to lowercase, removing unwanted words, | |
| special characters, numbers greater than 10, and content from location, states, regions, etc.""" | |
| def __init__(self): | |
| # Define unwanted words and initialize counters | |
| self.unwanted_words = ['remote', 'hybrid', 'flexible location', 'location', 'open to work', | |
| 'role', 'job', 'level', 'remot'] | |
| self.location_removed_count = 0 | |
| self.unwanted_words_removed_count = 0 | |
| self.brackets_removed_count = 0 | |
| self.state_region_removed_count = 0 | |
| self.numbers_removed_count = 0 | |
| def remove_location_unwanted_words_brackets(self, row): | |
| """Removes parts of the title based on location, unwanted words, bracketed content, | |
| numbers greater than 10, and also removes symbols other than alphanumeric.""" | |
| title = row['titles_title'] | |
| location = row['LOCATION'] | |
| states = row.get('STATES', '') # Get values from 'STATES' column if present | |
| region_state = row.get('REGION_STATE', '') # Get values from 'REGION_STATE' column if present | |
| county = row.get('COUNTY', '') # Get values from 'COUNTY' column if present | |
| city = row.get('city', '') # Get values from 'city' column if present | |
| # Ensure title is a string | |
| if isinstance(title, str): | |
| # Remove location if present in the title | |
| if isinstance(location, str) and re.search(r'\b{}\b'.format(re.escape(location)), title, flags=re.IGNORECASE): | |
| title = re.sub(r'\b{}\b'.format(re.escape(location)), '', title, flags=re.IGNORECASE) | |
| self.location_removed_count += 1 | |
| # Remove unwanted words | |
| for word in self.unwanted_words: | |
| pattern = r'\b{}\b'.format(re.escape(word)) | |
| if re.search(pattern, title, flags=re.IGNORECASE): | |
| title = re.sub(pattern, '', title, flags=re.IGNORECASE) | |
| self.unwanted_words_removed_count += 1 | |
| # Remove content from STATES, REGION_STATE, COUNTY, and city | |
| for region in [states, region_state, county, city]: | |
| if isinstance(region, str) and re.search(r'\b{}\b'.format(re.escape(region)), title, flags=re.IGNORECASE): | |
| title = re.sub(r'\b{}\b'.format(re.escape(region)), '', title, flags=re.IGNORECASE) | |
| self.state_region_removed_count += 1 | |
| # Remove content within brackets | |
| if re.search(r'\[.*?\]|\(.*?\)|\{.*?\}', title): | |
| title = re.sub(r'\[.*?\]|\(.*?\)|\{.*?\}', '', title) | |
| self.brackets_removed_count += 1 | |
| # Remove any non-alphanumeric characters (keeping spaces) | |
| title = re.sub(r'[^a-zA-Z0-9\s]', '', title) | |
| # Remove numbers greater than 10 | |
| if re.search(r'\b(?:[1-9][0-9]+|1[1-9]|[2-9][0-9])\b', title): | |
| title = re.sub(r'\b(?:[1-9][0-9]+|1[1-9]|[2-9][0-9])\b', '', title) | |
| self.numbers_removed_count += 1 | |
| # Clean up extra spaces | |
| title = re.sub(r'\s+', ' ', title).strip() | |
| return title | |
| def preprocess(self, title: str) -> str: | |
| """Converts title to lowercase, removes unwanted words, replaces specific terms, | |
| and standardizes job titles.""" | |
| if not isinstance(title, str): | |
| return title | |
| # Convert to lowercase | |
| title = title.lower() | |
| # Replace specific terms and Roman numerals | |
| replacements = [ | |
| (r'\b(?:SR|sr|Sr\.?|SR\.?|Senior|senior)\b', 'Senior'), | |
| (r'\b(?:JR|jr|Jr\.?|JR\.?|Junior|junior)\b', 'Junior'), | |
| (r'\b(?:AIML|aiml|ML|ml|MachineLearning|machinelearning|machine[_\-]learning)\b', 'Machine Learning'), | |
| (r'\b(?:GenAI|genai|Genai|generative[_\-]ai|GenerativeAI|generativeai)\b', 'Generative AI'), | |
| (r'\b(?:NLP|nlp|natural[_\-]language[_\-]processing|natural language processing)\b', 'NLP'), | |
| (r'\b(?:i|I)\b', '1'), | |
| (r'\b(?:ii|II)\b', '2'), | |
| (r'\b(?:iii|III)\b', '3'), | |
| (r'\b(?:iv|IV)\b', '4'), | |
| (r'\b(?:v|V)\b', '5') | |
| ] | |
| for pattern, replacement in replacements: | |
| title = re.sub(pattern, replacement, title, flags=re.IGNORECASE) | |
| # Handle specific Data Scientist cases | |
| title = re.sub(r'\b(director|dir\.?|dir)\b.*?(data\s*scientist|data\s*science)', 'Director Data Scientist', title, flags=re.IGNORECASE) | |
| title = re.sub(r'\b(manager|mgr)\b.*?(data\s*scientist|data\s*science)', 'Manager Data Scientist', title, flags=re.IGNORECASE) | |
| title = re.sub(r'\b(lead)\b.*?(data\s*scientist|data\s*science)', 'Lead Data Scientist', title, flags=re.IGNORECASE) | |
| title = re.sub(r'\b(associate|associates?)\b.*?(data\s*scientist|data\s*science)', 'Associate Data Scientist', title, flags=re.IGNORECASE) | |
| title = re.sub(r'\b(applied)\b.*?(data\s*scientist|data\s*science)', 'Applied Data Scientist', title, flags=re.IGNORECASE) | |
| title = re.sub(r'\b(intern|internship|trainee)\b.*?(data\s*scientist|data\s*science)', 'Intern Data Scientist', title, flags=re.IGNORECASE) | |
| # Ensure "ML" or "NLP" is retained if found in the title | |
| if re.search(r'\bdata\s*scientist\b', title, flags=re.IGNORECASE): | |
| if re.search(r'\b(?:ai|artificial intelligence|ml|machine learning|deep learning|dl)\b', title, flags=re.IGNORECASE): | |
| return 'ML Data Scientist' | |
| elif re.search(r'\b(?:nlp|natural language processing)\b', title, flags=re.IGNORECASE): | |
| return 'NLP Data Scientist' | |
| return title | |
| # Clean up extra spaces | |
| title = re.sub(r'\s+', ' ', title).strip() | |
| return title | |
| def is_title_empty(row): | |
| """ | |
| Check if the 'titles_title' is effectively empty, which includes | |
| strings that are either empty or contain only whitespace. | |
| """ | |
| title = row['titles_title'] | |
| return pd.isna(title) or (isinstance(title, str) and title.strip() == '') | |
| def main_preprocessing(): | |
| try: | |
| # Load the dataset | |
| df = pd.read_csv(r"Struct Data_Data Science 100K.csv", low_memory=False) | |
| # Initialize preprocessor | |
| preprocessor = JobTitlePreprocessor() | |
| # Apply both the removal and standard preprocessing steps | |
| df['clean_title'] = df.apply(preprocessor.remove_location_unwanted_words_brackets, axis=1) | |
| df['clean_title'] = df['clean_title'].apply(preprocessor.preprocess) | |
| # Remove rows where 'titles_title' is empty or contains only whitespace | |
| df = df[~df.apply(is_title_empty, axis=1)] | |
| # Drop rows where 'clean_title' is NaN | |
| df = df.dropna(subset=['clean_title']) | |
| # Log some information about the dataset | |
| logger.info(f"Original dataset shape: {df.shape}") | |
| logger.info(f"Number of non-empty titles: {df['clean_title'].notna().sum()}") | |
| # Save the preprocessed data | |
| output_df = df[['titles_title', 'clean_title']] | |
| output_df.to_csv('preprocessed_job_titles.csv', index=False) | |
| logger.info(f"Preprocessed dataset shape: {output_df.shape}") | |
| logger.info("Job title preprocessing completed successfully.") | |
| logger.info(f"Total rows with part of location removed from titles: {preprocessor.location_removed_count}") | |
| logger.info(f"Total unwanted words removed: {preprocessor.unwanted_words_removed_count}") | |
| logger.info(f"Total brackets removed: {preprocessor.brackets_removed_count}") | |
| logger.info(f"Total states/regions removed: {preprocessor.state_region_removed_count}") | |
| logger.info(f"Total numbers removed: {preprocessor.numbers_removed_count}") | |
| except Exception as e: | |
| logger.error(f"An error occurred during preprocessing: {e}") | |
| if __name__ == "__main__": | |
| main_preprocessing() | |