<reponame>Jagermeister/out_of_many_one
""" Interact with web interface to capture local backups of information """

from datetime import datetime, timedelta
import logging
import json
import time

from bs4 import BeautifulSoup
import requests
from typing import Any, List, Tuple


EFD_ENDPOINT_ACCESS = 'https://efdsearch.senate.gov/search/home/'
EFD_ENDPOINT_SEARCH = 'https://efdsearch.senate.gov/search/'
EFD_ENDPOINT_DATA = 'https://efdsearch.senate.gov/search/report/data/'
EFD_ENDPOINT_REPORT = 'https://efdsearch.senate.gov/search/view/{}/{}/'

HTTP_HEADERS = {
    'User-Agent': ' '.join([
        'Mozilla/5.0 (Windows NT 6.1; Win64; x64)',
        'AppleWebKit/537.36 (KHTML, like Gecko)',
        'Chrome/74.0.3729.169 Safari/537.36']),
    'Origin': 'https://efdsearch.senate.gov',
    'Referer': 'https://efdsearch.senate.gov/search/home/',
    'DNT': '1'
}

FETCH_MINIMUM_WAIT_SECONDS = 1.25

class EFD():
    """ Manage state for access to Electronic Financial Disclosures """

    def __init__(self) -> None:
        self.is_ready = False
        self.session = requests.Session()
        self.fetched_last = datetime.now() - timedelta(seconds=FETCH_MINIMUM_WAIT_SECONDS)

    def __ensure_fetching_rate_limit(self) -> None:
        """ Ensure fetching is waiting FETCH_MINIMUM_WAIT_SECONDS
            seconds inbetween each request.
        """
        current = datetime.now()
        difference = current - self.fetched_last
        time_to_wait = FETCH_MINIMUM_WAIT_SECONDS - difference.total_seconds()
        if time_to_wait > 0:
            time.sleep(time_to_wait)

        self.fetched_last = datetime.now()

    @staticmethod
    def __parse_agreement(html: str) -> str:
        """ Produce web token value from agreement HTML.
        Args:
            html: str - Response text from fetching agreement page.
        Raises:
            ValueError: When web token input is not found.
        Returns:
            web_token: str - Value of hidden web token input.
        """
        soup = BeautifulSoup(html, features='html.parser')
        web_token = soup.find('input', {'name': 'csrfmiddlewaretoken'})
        if hasattr(web_token, 'value'):
            return web_token['value']

        raise ValueError('Expected to find web token within HTML.')

    def __fetch_web_token(self) -> str:
        """ Return web token from fetching home page. """
        self.__ensure_fetching_rate_limit()
        response = self.session.get(EFD_ENDPOINT_SEARCH)
        web_token = EFD.__parse_agreement(response.text)
        return web_token

    @staticmethod
    def __parse_search_form(html: str) -> List[str]:
        """ Return search form input names. """
        soup = BeautifulSoup(html, features='html.parser')
        form_names = [i['name'] for i in soup.find('form').findAll('input')]
        return form_names

    def __post_agreement(self, web_token: str) -> List[str]:
        """ Authenticate by agreement. Allows accessing search form. """
        payload = {
            'prohibition_agreement': 1,
            'csrfmiddlewaretoken': web_token
        }

        self.session.headers.update(HTTP_HEADERS)
        self.__ensure_fetching_rate_limit()
        response = self.session.post(EFD_ENDPOINT_ACCESS, data=payload)
        form_names = self.__parse_search_form(response.text)
        return form_names

    def login(self) -> None:
        """ Acquire tokens to authenicate and access the Search functionality. """
        web_token = self.__fetch_web_token()
        form_names = self.__post_agreement(web_token)
        assert len(form_names) == 13, 'Login unsuccessful!'
        self.is_ready = True

    def __header_update_token(self) -> None:
        """ Add required token from cookie to header """
        cookies = self.session.cookies.get_dict()
        self.session.headers.update({
            'Referer': 'https://efdsearch.senate.gov/search/',
            'X-CSRFToken': cookies['csrftoken'],
        })

    def search(self, last_name: str) -> Any:
        """ Search financial disclosures. """
        form_data = {
            'start': 0,
            'length': 100,
            'report_types': '[7]',  # Annual Report
            'filter_types': '[1]',  # Senators
            'last_name': last_name,
            'submitted_start_date': '01/01/2012 00:00:00'
        }

        self.__header_update_token()
        self.__ensure_fetching_rate_limit()
        response = self.session.post(EFD_ENDPOINT_DATA, data=form_data)
        # draw, recordsTotal, data, recordsFiltered, result
        return json.loads(response.text)

    def annual_reports_search(self) -> Any:
        """ Search financial disclosures. """
        self.__header_update_token()
        document_links = []

        page_start = 0
        page_end = 100
        is_paging_complete = False
        records_count = None
        while not is_paging_complete:
            form_data = {
                'start': page_start,
                'length': page_end,
                'report_types': '[7]',  # Annual Report
                'filter_types': '[1]',  # Senators
                'submitted_start_date': '01/01/2012 00:00:00'
            }
            self.__ensure_fetching_rate_limit()
            logging.info(f'Posting for "{page_start}" to "{page_end}" out of "{records_count}".')
            response = self.session.post(EFD_ENDPOINT_DATA, data=form_data)
            response = json.loads(response.text)
            document_links.extend(response['data'])
            records_count = response['recordsTotal']
            page_start += 100
            page_end += 100
            is_paging_complete = page_start > records_count

        return document_links

    def annual_report_view(self, document_id: str) -> Tuple[Any, Any]:
        """ View Electronic Financial Disclosure """
        self.__header_update_token()
        link = EFD_ENDPOINT_REPORT.format('annual', document_id)
        self.__ensure_fetching_rate_limit()
        response = self.session.get(link)
        soup = BeautifulSoup(response.text, features='html.parser')
        return soup.find('h1').parent, soup.findAll('section', {'class': 'card mb-2'})

""" Interact with web interface to capture local backups of information """

from datetime import datetime, timedelta
import logging
import json
import time

from bs4 import BeautifulSoup
import requests
from typing import Any, List, Tuple


EFD_ENDPOINT_ACCESS = 'https://efdsearch.senate.gov/search/home/'
EFD_ENDPOINT_SEARCH = 'https://efdsearch.senate.gov/search/'
EFD_ENDPOINT_DATA = 'https://efdsearch.senate.gov/search/report/data/'
EFD_ENDPOINT_REPORT = 'https://efdsearch.senate.gov/search/view/{}/{}/'

HTTP_HEADERS = {
    'User-Agent': ' '.join([
        'Mozilla/5.0 (Windows NT 6.1; Win64; x64)',
        'AppleWebKit/537.36 (KHTML, like Gecko)',
        'Chrome/74.0.3729.169 Safari/537.36']),
    'Origin': 'https://efdsearch.senate.gov',
    'Referer': 'https://efdsearch.senate.gov/search/home/',
    'DNT': '1'
}

FETCH_MINIMUM_WAIT_SECONDS = 1.25

class EFD():
    """ Manage state for access to Electronic Financial Disclosures """

    def __init__(self) -> None:
        self.is_ready = False
        self.session = requests.Session()
        self.fetched_last = datetime.now() - timedelta(seconds=FETCH_MINIMUM_WAIT_SECONDS)

    def __ensure_fetching_rate_limit(self) -> None:
        """ Ensure fetching is waiting FETCH_MINIMUM_WAIT_SECONDS
            seconds inbetween each request.
        """
        current = datetime.now()
        difference = current - self.fetched_last
        time_to_wait = FETCH_MINIMUM_WAIT_SECONDS - difference.total_seconds()
        if time_to_wait > 0:
            time.sleep(time_to_wait)

        self.fetched_last = datetime.now()

    @staticmethod
    def __parse_agreement(html: str) -> str:
        """ Produce web token value from agreement HTML.
        Args:
            html: str - Response text from fetching agreement page.
        Raises:
            ValueError: When web token input is not found.
        Returns:
            web_token: str - Value of hidden web token input.
        """
        soup = BeautifulSoup(html, features='html.parser')
        web_token = soup.find('input', {'name': 'csrfmiddlewaretoken'})
        if hasattr(web_token, 'value'):
            return web_token['value']

        raise ValueError('Expected to find web token within HTML.')

    def __fetch_web_token(self) -> str:
        """ Return web token from fetching home page. """
        self.__ensure_fetching_rate_limit()
        response = self.session.get(EFD_ENDPOINT_SEARCH)
        web_token = EFD.__parse_agreement(response.text)
        return web_token

    @staticmethod
    def __parse_search_form(html: str) -> List[str]:
        """ Return search form input names. """
        soup = BeautifulSoup(html, features='html.parser')
        form_names = [i['name'] for i in soup.find('form').findAll('input')]
        return form_names

    def __post_agreement(self, web_token: str) -> List[str]:
        """ Authenticate by agreement. Allows accessing search form. """
        payload = {
            'prohibition_agreement': 1,
            'csrfmiddlewaretoken': web_token
        }

        self.session.headers.update(HTTP_HEADERS)
        self.__ensure_fetching_rate_limit()
        response = self.session.post(EFD_ENDPOINT_ACCESS, data=payload)
        form_names = self.__parse_search_form(response.text)
        return form_names

    def login(self) -> None:
        """ Acquire tokens to authenicate and access the Search functionality. """
        web_token = self.__fetch_web_token()
        form_names = self.__post_agreement(web_token)
        assert len(form_names) == 13, 'Login unsuccessful!'
        self.is_ready = True

    def __header_update_token(self) -> None:
        """ Add required token from cookie to header """
        cookies = self.session.cookies.get_dict()
        self.session.headers.update({
            'Referer': 'https://efdsearch.senate.gov/search/',
            'X-CSRFToken': cookies['csrftoken'],
        })

    def search(self, last_name: str) -> Any:
        """ Search financial disclosures. """
        form_data = {
            'start': 0,
            'length': 100,
            'report_types': '[7]',  # Annual Report
            'filter_types': '[1]',  # Senators
            'last_name': last_name,
            'submitted_start_date': '01/01/2012 00:00:00'
        }

        self.__header_update_token()
        self.__ensure_fetching_rate_limit()
        response = self.session.post(EFD_ENDPOINT_DATA, data=form_data)
        # draw, recordsTotal, data, recordsFiltered, result
        return json.loads(response.text)

    def annual_reports_search(self) -> Any:
        """ Search financial disclosures. """
        self.__header_update_token()
        document_links = []

        page_start = 0
        page_end = 100
        is_paging_complete = False
        records_count = None
        while not is_paging_complete:
            form_data = {
                'start': page_start,
                'length': page_end,
                'report_types': '[7]',  # Annual Report
                'filter_types': '[1]',  # Senators
                'submitted_start_date': '01/01/2012 00:00:00'
            }
            self.__ensure_fetching_rate_limit()
            logging.info(f'Posting for "{page_start}" to "{page_end}" out of "{records_count}".')
            response = self.session.post(EFD_ENDPOINT_DATA, data=form_data)
            response = json.loads(response.text)
            document_links.extend(response['data'])
            records_count = response['recordsTotal']
            page_start += 100
            page_end += 100
            is_paging_complete = page_start > records_count

        return document_links

    def annual_report_view(self, document_id: str) -> Tuple[Any, Any]:
        """ View Electronic Financial Disclosure """
        self.__header_update_token()
        link = EFD_ENDPOINT_REPORT.format('annual', document_id)
        self.__ensure_fetching_rate_limit()
        response = self.session.get(link)
        soup = BeautifulSoup(response.text, features='html.parser')
        return soup.find('h1').parent, soup.findAll('section', {'class': 'card mb-2'})
Interact with web interface to capture local backups of information Manage state for access to Electronic Financial Disclosures Ensure fetching is waiting FETCH_MINIMUM_WAIT_SECONDS             seconds inbetween each request. Produce web token value from agreement HTML.         Args:             html: str - Response text from fetching agreement page.         Raises:             ValueError: When web token input is not found.         Returns:             web_token: str - Value of hidden web token input. Return web token from fetching home page. Return search form input names. Authenticate by agreement. Allows accessing search form. Acquire tokens to authenicate and access the Search functionality. Add required token from cookie to header Search financial disclosures. # Annual Report # Senators # draw, recordsTotal, data, recordsFiltered, result Search financial disclosures. # Annual Report # Senators View Electronic Financial Disclosure