<reponame>wwangwe/labour-market-analysis
# -*- coding: utf-8 -*-
"""Web Scrapping.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/11iNMH2yKYzqpE7KMKaG-SC1zFRsDCToZ

# Real-time Kenyan Labour Market Analysis

## Web Scrapping
"""

import json
from random import randint

import requests
from bs4 import BeautifulSoup

headers = [
    ({
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5)AppleWebKit/605.1.15 (KHTML, like Gecko)Version/12.1.1 Safari/605.1.15',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.8 (KHTML, like Gecko)',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (iPhone; CPU iPhone OS 13_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,1;FBMD/iPhone;FBSN/iOS;FBSV/13.5.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5]'
    })
]


def header(headers: list) -> dict:
    """Generate a random header.

    Args:
        headers (list): List of headers.

    Returns:
        random_header (dict): Random header from the list of headers.
    """
    random_int = randint(0, len(headers) - 1)
    random_header = headers[random_int]
    return random_header


def prepare_soup(url: str) -> 'BeautifulSoup':
    """Process url to a Beautiful Soup object.

    Args:
        url (str): Link to jobs page.

    Raises:
        ValueError: Raised when requests.get fails.

    Returns:
        soup: Browsable bs4 object.
    """
    response = requests.get(url, header(headers), timeout=5)
    status_code = response.status_code
    if status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        return soup
    else:
        raise ValueError("Soup Not Created! Status Code: ", status_code)


def fetch_jobs(url: str) -> list:
    """Get job detail urls for all the jobs in current page(url).

    Args:
        url (str): Current page with jobs.

    Returns:
        list_data (list): List of job detail urls.
    """
    soup = prepare_soup(url)
    string_data = soup.find_all("script", type="application/ld+json")[0].text
    json_data = json.loads(string_data)['itemListElement']
    list_data = [data['url'] for data in json_data]
    pages = soup.find('ul',
                      'pagination').find_all('li')[-2].find_all(['a', 'span'
                                                                 ])[0].text
    return (list_data, pages)


def fetch_job_details(soup: 'BeautifulSoup', url: str) -> dict:
    """Fetch details for each job.

    Each dictionary contains details about only one job. Try Except 
    used to handle possible errors due to change in selectors.

    Args:
        soup (BeautifulSoup): Browsable bs4 object.

    Returns:
        dict: Dictionary of job details.
    """
    details = {}
    details['hyperlink'] = url
    try:
        details['title'] = soup.find('h1', 'job-header__title').text
    except AttributeError:
        details['title'] = 'None'
    try:
        details['function'] = soup.find('div',
                                        'hide-under-lg').find_all('h2')[1].text
    except AttributeError:
        details['function'] = 'None'
    try:
        details['location'] = soup.find('a', 'job-header__location').text
    except AttributeError:
        details['location'] = 'None'
    try:
        details['industry'] = soup.find('span',
                                        'job-header__location').find('a').text
    except AttributeError:
        details['industry'] = 'None'
    try:
        details['description'] = soup.find_all(
            'div', 'customer-card__content-segment')[0].find('p').text
    except AttributeError:
        details['description'] = 'None'
    try:
        details['qualifications'] = soup.find(
            'div', 'description-content__content').text
    except AttributeError:
        details['qualifications'] = 'None'

    return details

# -*- coding: utf-8 -*-
"""Web Scrapping.ipynb

Automatically generated by Colaboratory.

Original file is located at
    https://colab.research.google.com/drive/11iNMH2yKYzqpE7KMKaG-SC1zFRsDCToZ

# Real-time Kenyan Labour Market Analysis

## Web Scrapping
"""

import json
from random import randint

import requests
from bs4 import BeautifulSoup

headers = [
    ({
        'User-Agent':
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_5)AppleWebKit/605.1.15 (KHTML, like Gecko)Version/12.1.1 Safari/605.1.15',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/44.0.2403.157 Safari/537.36',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/601.7.8 (KHTML, like Gecko)',
    }),
    ({
        'User-Agent':
        'Mozilla/5.0 (iPhone; CPU iPhone OS 13_5_1 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Mobile/15E148 [FBAN/FBIOS;FBDV/iPhone9,1;FBMD/iPhone;FBSN/iOS;FBSV/13.5.1;FBSS/2;FBID/phone;FBLC/en_US;FBOP/5]'
    })
]


def header(headers: list) -> dict:
    """Generate a random header.

    Args:
        headers (list): List of headers.

    Returns:
        random_header (dict): Random header from the list of headers.
    """
    random_int = randint(0, len(headers) - 1)
    random_header = headers[random_int]
    return random_header


def prepare_soup(url: str) -> 'BeautifulSoup':
    """Process url to a Beautiful Soup object.

    Args:
        url (str): Link to jobs page.

    Raises:
        ValueError: Raised when requests.get fails.

    Returns:
        soup: Browsable bs4 object.
    """
    response = requests.get(url, header(headers), timeout=5)
    status_code = response.status_code
    if status_code == 200:
        soup = BeautifulSoup(response.text, 'html.parser')
        return soup
    else:
        raise ValueError("Soup Not Created! Status Code: ", status_code)


def fetch_jobs(url: str) -> list:
    """Get job detail urls for all the jobs in current page(url).

    Args:
        url (str): Current page with jobs.

    Returns:
        list_data (list): List of job detail urls.
    """
    soup = prepare_soup(url)
    string_data = soup.find_all("script", type="application/ld+json")[0].text
    json_data = json.loads(string_data)['itemListElement']
    list_data = [data['url'] for data in json_data]
    pages = soup.find('ul',
                      'pagination').find_all('li')[-2].find_all(['a', 'span'
                                                                 ])[0].text
    return (list_data, pages)


def fetch_job_details(soup: 'BeautifulSoup', url: str) -> dict:
    """Fetch details for each job.

    Each dictionary contains details about only one job. Try Except 
    used to handle possible errors due to change in selectors.

    Args:
        soup (BeautifulSoup): Browsable bs4 object.

    Returns:
        dict: Dictionary of job details.
    """
    details = {}
    details['hyperlink'] = url
    try:
        details['title'] = soup.find('h1', 'job-header__title').text
    except AttributeError:
        details['title'] = 'None'
    try:
        details['function'] = soup.find('div',
                                        'hide-under-lg').find_all('h2')[1].text
    except AttributeError:
        details['function'] = 'None'
    try:
        details['location'] = soup.find('a', 'job-header__location').text
    except AttributeError:
        details['location'] = 'None'
    try:
        details['industry'] = soup.find('span',
                                        'job-header__location').find('a').text
    except AttributeError:
        details['industry'] = 'None'
    try:
        details['description'] = soup.find_all(
            'div', 'customer-card__content-segment')[0].find('p').text
    except AttributeError:
        details['description'] = 'None'
    try:
        details['qualifications'] = soup.find(
            'div', 'description-content__content').text
    except AttributeError:
        details['qualifications'] = 'None'

    return details
# -*- coding: utf-8 -*- Web Scrapping.ipynb  Automatically generated by Colaboratory.  Original file is located at     https://colab.research.google.com/drive/11iNMH2yKYzqpE7KMKaG-SC1zFRsDCToZ  # Real-time Kenyan Labour Market Analysis  ## Web Scrapping Generate a random header.      Args:         headers (list): List of headers.      Returns:         random_header (dict): Random header from the list of headers. Process url to a Beautiful Soup object.      Args:         url (str): Link to jobs page.      Raises:         ValueError: Raised when requests.get fails.      Returns:         soup: Browsable bs4 object. Get job detail urls for all the jobs in current page(url).      Args:         url (str): Current page with jobs.      Returns:         list_data (list): List of job detail urls. Fetch details for each job.      Each dictionary contains details about only one job. Try Except      used to handle possible errors due to change in selectors.      Args:         soup (BeautifulSoup): Browsable bs4 object.      Returns:         dict: Dictionary of job details.