"""Main module."""
import requests
import re
from bs4 import BeautifulSoup
import bibtexparser
from pathlib import Path
import json
from tqdm import tqdm
import multiprocessing as mp
from tqdm.contrib.concurrent import process_map
from acm_dl_searcher._utils import _ensure_data_directory_exists, SEARCH_STRING

# TODO: better logging
# TODO: parallelize data collection
def _process_venue_data_from_doi(doi, short_name, overwrite=False, verify=False, force=False):
    """
    Takes a doi of a venue in acm and get the entries and abstract. The data will be cacheed in DATA_DIRECTORY. 
    Returns a dictionary containing the doi as keys and the details as values
    """    
    doi_file = _ensure_data_directory_exists() / (doi.replace("/", "_") + ".json")
    doi_entry = requests.get(f"http://doi.org/{doi}", headers={"Accept": "application/x-bibtex"}).text
    doi_title = bibtexparser.loads(doi_entry).entries[0]["title"]
    print(f"Title: {doi_title}")
    
    _update_collection_info(doi_file.name, doi, doi_title, short_name, force)

    doi_list_details = []
    if doi_file.exists():
        if not overwrite:
            with open(doi_file) as f:
                try:
                    doi_list_details = json.load(f)
                except json.decoder.JSONDecodeError:
                    pass
            if not verify:
                return doi_list_details
            print("File exists. Verifying if all entries exist.")
    
    doi = doi.replace("/", "%2F")
    search_string = SEARCH_STRING.format(key=doi, page_id=0)
    response = requests.get(search_string)

    # For debugging
    # with open("temp.html", "w") as f:
    #     f.write(response.text)

    doi_matcher = re.compile(r">https:\/\/doi\.org\/\d{2}\.\d{4}\/\d+\.\d+<")
    # A working curl for doi meta data: curl --location --silent --header "Accept: application/x-bibtex" https://doi.org/10.1145/3313831.3376868
    doi_list = [i.rstrip("<").lstrip(">") for i in doi_matcher.findall(response.text)]
    total_hits = int(BeautifulSoup(response.text, features="html.parser").find("span", {"class": "hitsLength"}).get_text())
    print(f"Found {total_hits} hits")

    max_workers = 2 * mp.cpu_count() + 2
    
    if total_hits > 50:
        page_id = 1
        search_strings = []
        while page_id * 50 < total_hits:
            search_strings.append(SEARCH_STRING.format(key=doi, page_id=page_id))
            page_id += 1

        print(f"Getting the urls for the {total_hits} entries")
        for urls in process_map(_get_doi_urls(doi_matcher), search_strings, max_workers=max_workers):
            doi_list.extend(urls)
        print()
        
    received_doi = [details["doi"] for details in doi_list_details]

    # Asynchronusly collect entries
    manager = mp.Manager()
    q = manager.Queue()    
    pool = mp.Pool(max_workers)

    #put listener to work first
    watcher = pool.apply_async(_bib_entry_collector, (q, doi_file, doi_list_details))

    print("Getting the entries")
    try:
        jobs = []
        for doi_url in doi_list:
            if doi_url.lstrip("https://doi.org/") in received_doi:
                continue
            jobs.append(pool.apply_async(_bib_entry_worker, (doi_url, q)))

        # collect results from the workers through the pool result queue
        for job in tqdm(jobs):
            job.get()

    finally:
        #now we are done, kill the listener
        q.put('kill')
        pool.close()
        pool.join()

        with open(doi_file) as f:
            doi_list_details = json.load(f)
            return doi_list_details


class _get_doi_urls:
    def __init__(self, doi_matcher):
        self.doi_matcher = doi_matcher

    def __call__(self, search_string):
        response = requests.get(search_string)
        return [i.rstrip("<").lstrip(">") for i in self.doi_matcher.findall(response.text)]


def _bib_entry_worker(doi_url, q):
    '''Get the doi entry and abstract of from a doi_url'''
    # Try five times before giving up
    for _ in range(5):
        try:
            details = bibtexparser.loads(requests.get(doi_url, headers={"Accept": "application/x-bibtex"}).text).entries[0]
            abstract_html = requests.get("https://dl.acm.org/doi/" + details["doi"]).text
            soup = BeautifulSoup(abstract_html, features="html.parser")
            abstract = soup.findAll("div", {"class": "abstractInFull"})[0].find("p").get_text()
            details["abstract"] = abstract
            q.put(details)
            break
        except KeyboardInterrupt:
            raise
        except Exception as e:
            print(e)


def _bib_entry_collector(q, doi_file, doi_list_details):
    '''listens for messages on the q, writes to file. '''
    with open(doi_file, 'w') as f:
        idx = 0
        while True:
            details = q.get()
            if details == 'kill' or idx % 50 == 0:
                with open(doi_file, "w") as f:
                    json.dump(doi_list_details, f, indent=4)
                if details == 'kill':
                    break
            doi_list_details.append(details)
            idx += 1
    

def _get_collection_info():
    """
    Reads the global information file and return the contents as adictionary and the path to the file itself.
    """
    info_file = _ensure_data_directory_exists() / "info.json"
    if info_file.exists():
        with open(info_file) as f:
            info = json.load(f)
    else:
        info = {}
    return info, info_file
        

def _update_collection_info(file_name, doi, title, short_name, force):
    """
    Adds an entry to the gloabl infomation file. If the entry exists, will be overwritten.
    """
    info, info_file = _get_collection_info()

    if file_name in info:
        if not force and short_name != info[file_name]["short_name"]:
            raise ValueError("Error: short-name provided does not match the one in database. " \
                             "If making sure all entries exist, run without the --short-name option to use the name in databse.\n"\
                             "To force the new name, use the option --force: acm-dl-searcher get <doi> --short-name <new-short-name> --force")
    else:
        if short_name is None:
            raise ValueError("Error: New entry: need paramter value for `short-name`. Pass value with option --short-name.")

    if short_name is not None and len(short_name) > 10:
        raise ValueError("Error: value for `--short-name` is too long (max 10 charachters).")
        
    info[file_name] = {"doi": doi, "title": title, "short_name": short_name}

    with open(info_file, "w") as f:
        json.dump(info, f, indent=4)


def _get_entry_count(doi_file):
    """
    Given a file path, will return the number of entries in that file. If the file reading files, returns None.
    """
    try:
        with open(doi_file) as f:
            content = json.load(f)
        return len(content)
    except:
        return None


def _search(search_fn, venue_filter=None):
    """
    Takes a two functions as parameters. And returns a set of entries from the complete databases that return true for both functions.
    
    :search_fn: A callable that takes one parameter and returns true or false. 
                The paramter passed to this callable will be the content of an entry.
    :venue_filter: A callable that takes three paramters: (short_name, title, doi) and return a boolean.
    :return: Returns a list of entries.
    """
    info, info_file = _get_collection_info()

    if venue_filter is None:
        venue_filter = lambda short_name, title, doi:True

    entries = []
    root_dir = _ensure_data_directory_exists()
    print(f"Searching in `{root_dir}`")
    for doi_file, entry in info.items():
        if venue_filter(entry["short_name"], entry["title"], entry["doi"]):
            with open(root_dir / doi_file) as f:
                try:
                    full_content_list = json.load(f)
                except json.decoder.JSONDecodeError:
                    print(f"Warning: {entry['doi']} is empty.")
                    continue
            for content_dict in full_content_list:
                content = ":: ".join(content_dict.values())
                if search_fn(content):
                    entries.append(content_dict)

    return entries
                

"""Main module."""
import requests
import re
from bs4 import BeautifulSoup
import bibtexparser
from pathlib import Path
import json
from tqdm import tqdm
import multiprocessing as mp
from tqdm.contrib.concurrent import process_map
from acm_dl_searcher._utils import _ensure_data_directory_exists, SEARCH_STRING

# TODO: better logging
# TODO: parallelize data collection
def _process_venue_data_from_doi(doi, short_name, overwrite=False, verify=False, force=False):
    """
    Takes a doi of a venue in acm and get the entries and abstract. The data will be cacheed in DATA_DIRECTORY. 
    Returns a dictionary containing the doi as keys and the details as values
    """    
    doi_file = _ensure_data_directory_exists() / (doi.replace("/", "_") + ".json")
    doi_entry = requests.get(f"http://doi.org/{doi}", headers={"Accept": "application/x-bibtex"}).text
    doi_title = bibtexparser.loads(doi_entry).entries[0]["title"]
    print(f"Title: {doi_title}")
    
    _update_collection_info(doi_file.name, doi, doi_title, short_name, force)

    doi_list_details = []
    if doi_file.exists():
        if not overwrite:
            with open(doi_file) as f:
                try:
                    doi_list_details = json.load(f)
                except json.decoder.JSONDecodeError:
                    pass
            if not verify:
                return doi_list_details
            print("File exists. Verifying if all entries exist.")
    
    doi = doi.replace("/", "%2F")
    search_string = SEARCH_STRING.format(key=doi, page_id=0)
    response = requests.get(search_string)

    # For debugging
    # with open("temp.html", "w") as f:
    #     f.write(response.text)

    doi_matcher = re.compile(r">https:\/\/doi\.org\/\d{2}\.\d{4}\/\d+\.\d+<")
    # A working curl for doi meta data: curl --location --silent --header "Accept: application/x-bibtex" https://doi.org/10.1145/3313831.3376868
    doi_list = [i.rstrip("<").lstrip(">") for i in doi_matcher.findall(response.text)]
    total_hits = int(BeautifulSoup(response.text, features="html.parser").find("span", {"class": "hitsLength"}).get_text())
    print(f"Found {total_hits} hits")

    max_workers = 2 * mp.cpu_count() + 2
    
    if total_hits > 50:
        page_id = 1
        search_strings = []
        while page_id * 50 < total_hits:
            search_strings.append(SEARCH_STRING.format(key=doi, page_id=page_id))
            page_id += 1

        print(f"Getting the urls for the {total_hits} entries")
        for urls in process_map(_get_doi_urls(doi_matcher), search_strings, max_workers=max_workers):
            doi_list.extend(urls)
        print()
        
    received_doi = [details["doi"] for details in doi_list_details]

    # Asynchronusly collect entries
    manager = mp.Manager()
    q = manager.Queue()    
    pool = mp.Pool(max_workers)

    #put listener to work first
    watcher = pool.apply_async(_bib_entry_collector, (q, doi_file, doi_list_details))

    print("Getting the entries")
    try:
        jobs = []
        for doi_url in doi_list:
            if doi_url.lstrip("https://doi.org/") in received_doi:
                continue
            jobs.append(pool.apply_async(_bib_entry_worker, (doi_url, q)))

        # collect results from the workers through the pool result queue
        for job in tqdm(jobs):
            job.get()

    finally:
        #now we are done, kill the listener
        q.put('kill')
        pool.close()
        pool.join()

        with open(doi_file) as f:
            doi_list_details = json.load(f)
            return doi_list_details


class _get_doi_urls:
    def __init__(self, doi_matcher):
        self.doi_matcher = doi_matcher

    def __call__(self, search_string):
        response = requests.get(search_string)
        return [i.rstrip("<").lstrip(">") for i in self.doi_matcher.findall(response.text)]


def _bib_entry_worker(doi_url, q):
    '''Get the doi entry and abstract of from a doi_url'''
    # Try five times before giving up
    for _ in range(5):
        try:
            details = bibtexparser.loads(requests.get(doi_url, headers={"Accept": "application/x-bibtex"}).text).entries[0]
            abstract_html = requests.get("https://dl.acm.org/doi/" + details["doi"]).text
            soup = BeautifulSoup(abstract_html, features="html.parser")
            abstract = soup.findAll("div", {"class": "abstractInFull"})[0].find("p").get_text()
            details["abstract"] = abstract
            q.put(details)
            break
        except KeyboardInterrupt:
            raise
        except Exception as e:
            print(e)


def _bib_entry_collector(q, doi_file, doi_list_details):
    '''listens for messages on the q, writes to file. '''
    with open(doi_file, 'w') as f:
        idx = 0
        while True:
            details = q.get()
            if details == 'kill' or idx % 50 == 0:
                with open(doi_file, "w") as f:
                    json.dump(doi_list_details, f, indent=4)
                if details == 'kill':
                    break
            doi_list_details.append(details)
            idx += 1
    

def _get_collection_info():
    """
    Reads the global information file and return the contents as adictionary and the path to the file itself.
    """
    info_file = _ensure_data_directory_exists() / "info.json"
    if info_file.exists():
        with open(info_file) as f:
            info = json.load(f)
    else:
        info = {}
    return info, info_file
        

def _update_collection_info(file_name, doi, title, short_name, force):
    """
    Adds an entry to the gloabl infomation file. If the entry exists, will be overwritten.
    """
    info, info_file = _get_collection_info()

    if file_name in info:
        if not force and short_name != info[file_name]["short_name"]:
            raise ValueError("Error: short-name provided does not match the one in database. " \
                             "If making sure all entries exist, run without the --short-name option to use the name in databse.\n"\
                             "To force the new name, use the option --force: acm-dl-searcher get <doi> --short-name <new-short-name> --force")
    else:
        if short_name is None:
            raise ValueError("Error: New entry: need paramter value for `short-name`. Pass value with option --short-name.")

    if short_name is not None and len(short_name) > 10:
        raise ValueError("Error: value for `--short-name` is too long (max 10 charachters).")
        
    info[file_name] = {"doi": doi, "title": title, "short_name": short_name}

    with open(info_file, "w") as f:
        json.dump(info, f, indent=4)


def _get_entry_count(doi_file):
    """
    Given a file path, will return the number of entries in that file. If the file reading files, returns None.
    """
    try:
        with open(doi_file) as f:
            content = json.load(f)
        return len(content)
    except:
        return None


def _search(search_fn, venue_filter=None):
    """
    Takes a two functions as parameters. And returns a set of entries from the complete databases that return true for both functions.
    
    :search_fn: A callable that takes one parameter and returns true or false. 
                The paramter passed to this callable will be the content of an entry.
    :venue_filter: A callable that takes three paramters: (short_name, title, doi) and return a boolean.
    :return: Returns a list of entries.
    """
    info, info_file = _get_collection_info()

    if venue_filter is None:
        venue_filter = lambda short_name, title, doi:True

    entries = []
    root_dir = _ensure_data_directory_exists()
    print(f"Searching in `{root_dir}`")
    for doi_file, entry in info.items():
        if venue_filter(entry["short_name"], entry["title"], entry["doi"]):
            with open(root_dir / doi_file) as f:
                try:
                    full_content_list = json.load(f)
                except json.decoder.JSONDecodeError:
                    print(f"Warning: {entry['doi']} is empty.")
                    continue
            for content_dict in full_content_list:
                content = ":: ".join(content_dict.values())
                if search_fn(content):
                    entries.append(content_dict)

    return entries
                

Main module. # TODO: better logging # TODO: parallelize data collection Takes a doi of a venue in acm and get the entries and abstract. The data will be cacheed in DATA_DIRECTORY.      Returns a dictionary containing the doi as keys and the details as values # For debugging # with open("temp.html", "w") as f: #     f.write(response.text) # A working curl for doi meta data: curl --location --silent --header "Accept: application/x-bibtex" https://doi.org/10.1145/3313831.3376868 # Asynchronusly collect entries #put listener to work first # collect results from the workers through the pool result queue #now we are done, kill the listener Get the doi entry and abstract of from a doi_url # Try five times before giving up listens for messages on the q, writes to file. Reads the global information file and return the contents as adictionary and the path to the file itself. Adds an entry to the gloabl infomation file. If the entry exists, will be overwritten. Given a file path, will return the number of entries in that file. If the file reading files, returns None. Takes a two functions as parameters. And returns a set of entries from the complete databases that return true for both functions.          :search_fn: A callable that takes one parameter and returns true or false.                  The paramter passed to this callable will be the content of an entry.     :venue_filter: A callable that takes three paramters: (short_name, title, doi) and return a boolean.     :return: Returns a list of entries.