# -*- coding: utf-8 -*-
# standard libraries
import csv
import logging
from pathlib import Path
from typing import List

# third-party libraries
from bs4 import BeautifulSoup
import pandas as pd
import pendulum

# my libraries
import src.helpers as hlp

logger = logging


def filter_download_contents(contents):
    """
    TODO
    """
    filtered_contents = contents
    filtered_contents = filtered_contents.replace('\r', '')
    filtered_contents = filtered_contents.replace('\n', '')
    filtered_contents = filtered_contents.replace('\t', '')

    return filtered_contents


def build_journal_record(url):
    """
    TODO
    """
    # download journal
    logger.info(f'Downloading {url}')
    resp: dict = hlp.download_url(url)
    contents_raw = resp['contents'].decode('utf-8')

    # filters
    # remove characters in contents that may break CSV format
    contents = filter_download_contents(contents_raw)

    # extract important info about journal
    record = {}

    # journal download
    record = hlp.parse_download_response(resp)
    record['data'] = contents

    # session designation
    session: str = record['filename'].split('DAY')[0].split('/')[-1]
    record['session'] = session

    #  extract date and legislative day
    soup = BeautifulSoup(contents, "html.parser")
    html_title = soup.title.string
    # supplements are a special case
    if 'SUPPLEMENT —' in html_title:
        date_raw, lege_day_raw = html_title.split('SUPPLEMENT —')
    elif 'SUPPLEMENT –– ' in html_title:
        date_raw, lege_day_raw = html_title.split('SUPPLEMENT ––')
    else:
        date_raw, lege_day_raw = html_title.split('—')
    date = pendulum.parse(date_raw, strict=False).date()
    record['date'] = date
    lege_day = ''.join(ch for ch in lege_day_raw if ch.isdigit())
    record['legislative_day'] = lege_day

    return record


def acquire(**kwargs):
    """
    acquire journal data in raw format
    """
    # configuration/input
    input_filepath = kwargs['input_filepath']
    output_filepath = kwargs['output_filepath']
    field_names = tuple(kwargs['field_names'])

    logger.info('Reading URL file for journals.')
    targets: bytes = Path(input_filepath).read_bytes()
    urls: List[str] = targets.splitlines()

    logger.info('Creating journal CSV file.')
    # create parent directories for output file
    hlp.create_parent_directories(output_filepath)
    dest = Path(output_filepath)
    with dest.open(mode='w', encoding=hlp.ENCODING, newline='') as fp:
        writer = csv.DictWriter(fp,
                                fieldnames=field_names,
                                restval='',
                                delimiter=hlp.DELIMITER,
                                doublequote=hlp.DOUBLEQUOTE,
                                escapechar=hlp.ESCAPE_CHAR,
                                lineterminator=hlp.LINE_TERMINATOR,
                                quotechar=hlp.QUOTE_CHAR,
                                quoting=hlp.QUOTING)
        writer.writeheader()
        logger.info('Downloading journal URLs.')
        for url in urls:
            record = build_journal_record(url)
            writer.writerow(record)

    logger.info('Done.')


def filter_data(data):
    """
    run data through filters
    """
    clean_data = data
    # handle spanish chars
    clean_data = clean_data.replace('&#xE1;', 'á')
    clean_data = clean_data.replace('&#xF1;', 'ñ')
    # needed for picking up record votes for measures; the measures are
    # surrounded by bold tags and it messes up the algorithm
    clean_data = clean_data.replace('<b>', '')
    clean_data = clean_data.replace('</b>', '')

    return clean_data


def clean_record(record):
    """
    clean each journal record
    """
    typ = record['type']
    location = record['location']
    retrieval_datetime = record['retrieval_datetime']
    filename = record['filename']
    session = record['session']
    date = record['date']
    legislative_day = record['legislative_day']
    raw_data: str = record['data']

    # filter data
    data = filter_data(raw_data)

    # create clean record
    clean_record = {}
    clean_record['typ'] = typ
    clean_record['location'] = location
    clean_record['retrieval_datetime'] = retrieval_datetime
    clean_record['filename'] = filename
    clean_record['session'] = session
    clean_record['date'] = date
    clean_record['legislative_day'] = legislative_day
    clean_record['data'] = data

    # convert dict record to a Series
    clean_series = pd.Series(clean_record)

    return clean_series


def clean(**kwargs):
    """
    clean journal records
    """
    # configuration/input
    input_filepath = kwargs['input_filepath']
    output_filepath = kwargs['output_filepath']

    df_journals = hlp.read_tsv(input_filepath)

    clean_data = df_journals.apply(clean_record, axis=1)

    # create parent directories for output file
    hlp.create_parent_directories(output_filepath)

    # write output file
    df_clean = pd.DataFrame(clean_data)
    hlp.write_tsv(df_clean, output_filepath)

# -*- coding: utf-8 -*-
# standard libraries
import csv
import logging
from pathlib import Path
from typing import List

# third-party libraries
from bs4 import BeautifulSoup
import pandas as pd
import pendulum

# my libraries
import src.helpers as hlp

logger = logging


def filter_download_contents(contents):
    """
    TODO
    """
    filtered_contents = contents
    filtered_contents = filtered_contents.replace('\r', '')
    filtered_contents = filtered_contents.replace('\n', '')
    filtered_contents = filtered_contents.replace('\t', '')

    return filtered_contents


def build_journal_record(url):
    """
    TODO
    """
    # download journal
    logger.info(f'Downloading {url}')
    resp: dict = hlp.download_url(url)
    contents_raw = resp['contents'].decode('utf-8')

    # filters
    # remove characters in contents that may break CSV format
    contents = filter_download_contents(contents_raw)

    # extract important info about journal
    record = {}

    # journal download
    record = hlp.parse_download_response(resp)
    record['data'] = contents

    # session designation
    session: str = record['filename'].split('DAY')[0].split('/')[-1]
    record['session'] = session

    #  extract date and legislative day
    soup = BeautifulSoup(contents, "html.parser")
    html_title = soup.title.string
    # supplements are a special case
    if 'SUPPLEMENT —' in html_title:
        date_raw, lege_day_raw = html_title.split('SUPPLEMENT —')
    elif 'SUPPLEMENT –– ' in html_title:
        date_raw, lege_day_raw = html_title.split('SUPPLEMENT ––')
    else:
        date_raw, lege_day_raw = html_title.split('—')
    date = pendulum.parse(date_raw, strict=False).date()
    record['date'] = date
    lege_day = ''.join(ch for ch in lege_day_raw if ch.isdigit())
    record['legislative_day'] = lege_day

    return record


def acquire(**kwargs):
    """
    acquire journal data in raw format
    """
    # configuration/input
    input_filepath = kwargs['input_filepath']
    output_filepath = kwargs['output_filepath']
    field_names = tuple(kwargs['field_names'])

    logger.info('Reading URL file for journals.')
    targets: bytes = Path(input_filepath).read_bytes()
    urls: List[str] = targets.splitlines()

    logger.info('Creating journal CSV file.')
    # create parent directories for output file
    hlp.create_parent_directories(output_filepath)
    dest = Path(output_filepath)
    with dest.open(mode='w', encoding=hlp.ENCODING, newline='') as fp:
        writer = csv.DictWriter(fp,
                                fieldnames=field_names,
                                restval='',
                                delimiter=hlp.DELIMITER,
                                doublequote=hlp.DOUBLEQUOTE,
                                escapechar=hlp.ESCAPE_CHAR,
                                lineterminator=hlp.LINE_TERMINATOR,
                                quotechar=hlp.QUOTE_CHAR,
                                quoting=hlp.QUOTING)
        writer.writeheader()
        logger.info('Downloading journal URLs.')
        for url in urls:
            record = build_journal_record(url)
            writer.writerow(record)

    logger.info('Done.')


def filter_data(data):
    """
    run data through filters
    """
    clean_data = data
    # handle spanish chars
    clean_data = clean_data.replace('&#xE1;', 'á')
    clean_data = clean_data.replace('&#xF1;', 'ñ')
    # needed for picking up record votes for measures; the measures are
    # surrounded by bold tags and it messes up the algorithm
    clean_data = clean_data.replace('<b>', '')
    clean_data = clean_data.replace('</b>', '')

    return clean_data


def clean_record(record):
    """
    clean each journal record
    """
    typ = record['type']
    location = record['location']
    retrieval_datetime = record['retrieval_datetime']
    filename = record['filename']
    session = record['session']
    date = record['date']
    legislative_day = record['legislative_day']
    raw_data: str = record['data']

    # filter data
    data = filter_data(raw_data)

    # create clean record
    clean_record = {}
    clean_record['typ'] = typ
    clean_record['location'] = location
    clean_record['retrieval_datetime'] = retrieval_datetime
    clean_record['filename'] = filename
    clean_record['session'] = session
    clean_record['date'] = date
    clean_record['legislative_day'] = legislative_day
    clean_record['data'] = data

    # convert dict record to a Series
    clean_series = pd.Series(clean_record)

    return clean_series


def clean(**kwargs):
    """
    clean journal records
    """
    # configuration/input
    input_filepath = kwargs['input_filepath']
    output_filepath = kwargs['output_filepath']

    df_journals = hlp.read_tsv(input_filepath)

    clean_data = df_journals.apply(clean_record, axis=1)

    # create parent directories for output file
    hlp.create_parent_directories(output_filepath)

    # write output file
    df_clean = pd.DataFrame(clean_data)
    hlp.write_tsv(df_clean, output_filepath)

# -*- coding: utf-8 -*- # standard libraries # third-party libraries # my libraries TODO TODO # download journal # filters # remove characters in contents that may break CSV format # extract important info about journal # journal download # session designation #  extract date and legislative day # supplements are a special case acquire journal data in raw format # configuration/input # create parent directories for output file run data through filters # handle spanish chars #xE1;', 'á') #xF1;', 'ñ') # needed for picking up record votes for measures; the measures are # surrounded by bold tags and it messes up the algorithm clean each journal record # filter data # create clean record # convert dict record to a Series clean journal records # configuration/input # create parent directories for output file # write output file