#  Copyright (c) 2021. Universidad de Pinar del Rio
#  This file is part of SCEIBA (sceiba.cu).
#  SCEIBA is free software; you can redistribute it and/or modify it
#  under the terms of the MIT License; see LICENSE file for more details.
#

import binascii
import json
import os
import time
from random import randint

import requests
from flask import current_app
from invenio_db import db
from lxml import html

from iroko.harvester.api import BaseHarvester
from iroko.harvester.utils import get_iroko_harvester_agent
from iroko.sources.api import SourceRecord
from iroko.sources.models import SourceRawData, SourceStatus, SourceType
from iroko.utils import get_default_user


def is_issn(code: str) -> bool:
    """
    return true if code is a valid issn
    :param code:
    :return:
    """
    try:
        # TODO: replace this by a regular expression...
        parts = code.split('-')
        if len(parts[0]) == 4 and len(parts[1]) == 4:
            return True
        return False
    except Exception:
        return False


def encode_multipart_formdata(fields):
    boundary = binascii.hexlify(os.urandom(16)).decode('ascii')

    body = (
        "".join(
            "--%s\r\n"
            "Content-Disposition: form-data; name=\"%s\"\r\n"
            "\r\n"
            "%s\r\n" % (boundary, field, value)
            for field, value in fields.items()
            ) +
        "--%s--\r\n" % boundary
    )

    content_type = "multipart/form-data; boundary=%s" % boundary

    return body, content_type


def get_issn_count(res: requests.Response):
    doc1 = html.fromstring(res.text)
    element = doc1.xpath('.//h2[@class="page-items-count search-results-count"]')
    return int(element[0].text.split()[0])


def get_info_issn(issn: str, sess: requests.Session):
    sess.headers.update(get_iroko_harvester_agent())
    url = 'https://portal.issn.org/resource/ISSN/' + issn + '?format=json'
    response = sess.get(url)

    # text = response.content.decode('UTF-8').replace('\\n', '')
    # return text.replace('\\"', '"')

    # TODO: see if we need to do something if response.status_code != 200:

    return json.loads(response.content.decode('UTF-8'))


def getissn(res: requests.Response):
    doc1 = html.fromstring(res.text)
    result = []
    element = doc1.xpath('.//div[@class="item-result-content-text flex-zero"]//p[1]')
    for v in element:
        result.append(v.text_content().split()[1])
    return result


def request_advanced_search(country_code):
    url = 'https://portal.issn.org/advancedsearch'

    sess = requests.Session()

    sess.headers.update(get_iroko_harvester_agent())

    response = sess.get(url)

    doc1 = html.fromstring(response.content.decode('UTF-8'))
    variable = ["form_build_id", "form_id", "honeypot_time"]
    variable_id = "advances-search-issn-final-form"
    dictionary = {}

    for v in variable:
        element = doc1.xpath('.//form[@id="' + variable_id + '"]//input[@name="' + v + '"]')
        dictionary[v] = element[0].value

    print(dictionary)
    sleep_time = randint(3, 9)
    print('sleep {0} seconds'.format(sleep_time))
    time.sleep(sleep_time)

    body = {
        'operator': 'SHOULD',
        'selectTitle': 'all',
        'opSpeTitle': '0',
        'issnOpt': 'all',
        'globalField[fld][0][operator]': 'MUST',
        'globalField[fld][0][selector]': 'country',
        'globalField[fld][0][countryDiv][countrylist][' + country_code + ']': country_code,
        'op': 'Search',
        'form_id': dictionary['form_id'],
        'honeypot_time': dictionary['honeypot_time'],
        'form_build_id': dictionary['form_build_id']
        }

    data, content_type = encode_multipart_formdata(body)

    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 '
                      'Firefox/64.0',
        'Content-Type': content_type,
        'Cache-Control': 'no-cache',
        }

    resp = sess.post(url, data=data, headers=headers)

    return resp, sess


def collect_issn_list(country_code):
    """get all cuban ISSNs """

    issn_list = []
    fres, session = request_advanced_search(country_code)
    issn_list = getissn(fres)

    session.headers.update(get_iroko_harvester_agent())

    sleep_time = randint(3, 9)
    print('sleep {0} seconds'.format(sleep_time))
    time.sleep(sleep_time)

    page = 2
    size = 100
    count = (get_issn_count(fres) - 10) / size

    while page < count:
        print('getting: {0}'.format(fres.url + '&currentpage={0}&size={1}'.format(page, size)))
        resp = session.get(fres.url + '&currentpage={0}&size={1}'.format(page, size))
        issn_list.extend(getissn(resp))

        sleep_time = randint(3, 9)
        print('sleep {0} seconds'.format(sleep_time))
        time.sleep(sleep_time)

        page += 1

    return issn_list


def collect_issn_info(issn_list):
    session = requests.Session()

    session.headers.update(get_iroko_harvester_agent())

    result = dict()

    for issn in issn_list:
        try:
            print('try getting {0} info'.format(issn))
            result[issn] = get_info_issn(issn, session)
        except Exception as e:
            print('error, getting {0} info, error: {1}'.format(issn, e))
            result[issn] = {"error": str(e)}
            pass
        finally:
            print('ok, saving to file')
            # with open(self.issn_info_file, 'w+', encoding=('UTF-8')) as file_issn:
            #     print('writing to file {0}'.format(self.cuban_issn_info_file))
            #     json.dump(result, file_issn, ensure_ascii=False)

            sleep_time = randint(4, 7)
            print('finally, sleep {0} seconds'.format(sleep_time))
            time.sleep(sleep_time)
    return result


def collect_issn_info_single(issn):
    session = requests.Session()

    session.headers.update(get_iroko_harvester_agent())

    result = dict()

    try:
        print('try getting {0} info'.format(issn))
        result[issn] = get_info_issn(issn, session)
    except Exception as e:
        print('error, getting {0} info, error: {1}'.format(issn, e))
        result[issn] = {"error": str(e)}
        pass
    finally:
        print('ok')
        pass
    return result


class IssnDataParser:
    """
    adhoc class for issn.org data,
    assume in
    source = SourceRawData.query.filter_by(identifier=identifier).first()
    data = source.get_data_field('issn')
    jsonschema-LD format
    """

    @classmethod
    def parse(cls, identifier) -> dict:
        """

        :param identifier: to SourceRawData.query.filter_by(identifier=identifier).first()
        :return:
        [list, str, list]
        1- a list with identifiers, valid for a SourceRecord
        2- the mainTitle
        3- all known aliases

        """
        return cls._parse_data(identifier, parse_relations=True)

    @classmethod
    def _parse_data(cls, identifier, parse_relations=False) -> dict:
        """
        helper func
        :param
        :param parse_relations: if True, will analize 'isFormatOf' and 'otherPhysicalFormat' fields
        :return:
        """

        if not is_issn(identifier):
            return None

        main_title = ''
        identifiers = list()
        aliases = list()

        source = SourceRawData.query.filter_by(identifier=identifier).first()
        if not source:
            return None
            # esto significa que no se recolecto el issn, recolectarlo entonces!!!!
            print('NO SOURCE', identifier)
            issndata = collect_issn_info_single(identifier)
            obj_issn = SourceRawData()
            obj_issn.identifier = identifier
            obj_issn.set_data_field('issn', issndata)
            db.session.add(obj_issn)
            db.session.commit()
            source = SourceRawData.query.filter_by(identifier=identifier).first()

        data = source.get_data_field('issn')

        if '@graph' not in data:
            print('NO DATA', identifier, data)
            return None

        for item in data["@graph"]:

            # ISSN Link (issn_l) is the main issn
            if '@type' in item and item['@type'] == 'http://id.loc.gov/ontologies/bibframe/IssnL':
                cls._append_to_identifier_list(
                    identifiers, {'idtype': 'issn_l', 'value': item['value']}
                    )

            if '@id' in item and item['@id'] == 'resource/ISSN/' + identifier:
                cls._append_to_identifier_list(identifiers, cls._get_issn(item))

                if 'mainTitle' in item:
                    main_title = item['mainTitle']
                    pass
                if 'name' in item:
                    name = item['name']
                    if type(name) == list:
                        for n in name:
                            cls._append_to_alias_list(aliases, n)
                    else:
                        cls._append_to_alias_list(aliases, name)
                else:
                    print('no name!!!!', identifier)
                    pass

                # issn relations.
                if parse_relations:
                    try:
                        rel = cls._get_relations(item)
                        if main_title == '' and rel.title != '':
                            main_title = rel.title
                        for _id in rel.identifiers:
                            cls._append_to_identifier_list(identifiers, _id)
                        for a in rel.aliases:
                            cls._append_to_alias_list(aliases, a)
                    except Exception as e:
                        # print(e)
                        pass
        if main_title == '':
            print('no mainTitle', identifier)
            if len(aliases) > 0:
                main_title = aliases[0]
        record = dict()
        record['identifiers'] = identifiers
        record['name'] = main_title
        record['title'] = main_title
        record['aliases'] = aliases

        return record

    @classmethod
    def _append_to_identifier_list(cls, id_list: list, id_item):
        append = True
        for id in id_list:
            if id['idtype'] == id_item['idtype']:
                append = False
        if append:
            id_list.append(id_item)

    @classmethod
    def _append_to_alias_list(cls, alias_list: list, alias_item):
        append = True
        for alias in alias_list:
            if alias == alias_item:
                append = False
        if append:
            alias_list.append(alias_item)

    @classmethod
    def _get_relations(cls, item):
        # TODO: 'isFormatOf' is the same as 'otherPhysicalFormat'?
        relations = None
        if 'isFormatOf' in item:
            relations = item['isFormatOf']
        elif 'otherPhysicalFormat' in item:
            relations = item['otherPhysicalFormat']
        else:
            # print('no relations', item)
            pass
        if type(relations) == str:
            issn_rel = relations.split('/').pop()
            return cls._parse_data(issn_rel)
        if type(relations) == list:
            result_ids = list()
            result_alias = list()
            result_title = ''
            for rel in relations:
                issn_rel = rel.split('/').pop()
                rel = cls._parse_data(issn_rel)
                if result_title == '':
                    result_title = rel.title
                result_ids.extend(rel.identifiers)
                result_alias.extend(rel.aliases)

            record = dict()
            record['identifiers'] = result_ids
            record['title'] = result_title
            record['aliases'] = result_alias

            return record

    @classmethod
    def _get_issn(cls, item):
        # desambiguate all issn types
        if item['format'] == 'vocabularies/medium#Online':
            return {'idtype': 'issn_e', 'value': item['issn']}
        if item['format'] == 'vocabularies/medium#Print':
            return {'idtype': 'issn_p', 'value': item['issn']}
        if item['format'] == 'vocabularies/medium#DigitalCarrier':
            return {'idtype': 'issn_c', 'value': item['issn']}
        if item['format'] == 'vocabularies/medium#Other':
            return {'idtype': 'issn_o', 'value': item['issn']}


class IssnHarvester(BaseHarvester):
    """
    TODO: Document this better!!!
    Harvest all cuban ISSN from issn.org"""

    # TODO: all functions private, except process_pipeline

    def __init__(
        self, work_dir, country_code='CUB', country_id='http://id.loc.gov/vocabulary/countries/cu'
        ):

        self.work_dir = os.path.join(work_dir, 'issn')
        if not os.path.exists(self.work_dir):
            os.mkdir(self.work_dir)

        self.country_code = country_code
        self.country_id = country_id

        self.issn_list_file = os.path.join(self.work_dir, self.country_code + '.list.json')
        self.issn_info_file = os.path.join(self.work_dir, self.country_code + '.info.json')

    def process_pipeline(self):
        """get all cuban issn and its info
        return pair issns(array of all issns), infos(dict with issns info)
        """

        issn_list = self.get_issn_list(remote=True)
        issn_info = self.get_issn_info(remote=True)
        return issn_list, issn_info

    def get_issn_list(self, remote):
        if remote:
            issn_list = collect_issn_list(self.country_code)
            with open(self.issn_list_file, 'w+', encoding='UTF-8') as file_issn:
                if file_issn:
                    print(issn_list)
                    json.dump(issn_list, file_issn, ensure_ascii=False)
        else:
            with open(self.issn_list_file, 'r', encoding='UTF-8') as file_issn:
                issn_list = json.load(file_issn)

        return issn_list

    def get_issn_info(self, remote):
        issn_list = self.get_issn_list(remote=False)
        if remote and issn_list:
            issn_info = collect_issn_info(issn_list)
            with open(self.issn_info_file, 'w+', encoding='UTF-8') as file_issn:
                if file_issn:
                    json.dump(issn_info, file_issn, ensure_ascii=False)
        else:
            with open(self.issn_info_file, 'r', encoding='UTF-8') as file_issn:
                issn_info = json.load(file_issn)

        return issn_info


class IssnHarvesterManager:
    @classmethod
    def collect_issn(cls):
        work_dir = current_app.config['IROKO_DATA_DIRECTORY']
        harvester = IssnHarvester(work_dir)

        return harvester.process_pipeline()

        # issns = harvester.get_issn_list(remoteissns)
        #
        # if info:
        #     infos = harvester.get_cuban_issns_info_json(issns, remoteinfo)
        #     # con lo que hay en el dic, crear/actualizar, versiones de source cuyo comentario
        #     sea issn...

    # @classmethod
    # def get_cuban_issns(cls):
    #     work_dir = current_app.config['IROKO_DATA_DIRECTORY']
    #     harvester = IssnHarvester(work_dir)
    #
    #     issns = harvester.get_issn_list(False)
    #     infos = harvester.get_cuban_issns_info_json(issns, False)
    #
    #     return infos

    @classmethod
    def sync_db(cls):
        work_dir = current_app.config['IROKO_DATA_DIRECTORY']
        harvester = IssnHarvester(work_dir)
        issn_list = harvester.get_issn_list(remote=False)
        issn_info = harvester.get_issn_info(remote=False)

        for issn in issn_list:
            data = issn_info[issn]
            issn_model = SourceRawData.query.filter_by(identifier=issn).first()
            print(issn_model)
            if not issn_model:
                obj_issn = SourceRawData()
                obj_issn.identifier = issn
                obj_issn.set_data_field('issn', data)
                db.session.add(obj_issn)
                print('NEW')
            else:
                issn_model.set_data_field('issn', data)
                print('UPDATED')
            db.session.flush()
        db.session.commit()
        print('DB COMMIT')

    @classmethod
    def sync_records(cls):

        issn_list = SourceRawData.query.all()
        for issn in issn_list:
            record = IssnDataParser.parse(issn.identifier)
            if record:
                user = get_default_user()
                record['source_status'] = SourceStatus.UNOFFICIAL.value
                record['source_type'] = SourceType.JOURNAL.value
                print(record)
                SourceRecord.new_source_revision(record, user_id=user.id, comment='issn.org import')
                # SourceRecord.create_or_update(record.dump(record))

#  Copyright (c) 2021. Universidad de Pinar del Rio
#  This file is part of SCEIBA (sceiba.cu).
#  SCEIBA is free software; you can redistribute it and/or modify it
#  under the terms of the MIT License; see LICENSE file for more details.
#

import binascii
import json
import os
import time
from random import randint

import requests
from flask import current_app
from invenio_db import db
from lxml import html

from iroko.harvester.api import BaseHarvester
from iroko.harvester.utils import get_iroko_harvester_agent
from iroko.sources.api import SourceRecord
from iroko.sources.models import SourceRawData, SourceStatus, SourceType
from iroko.utils import get_default_user


def is_issn(code: str) -> bool:
    """
    return true if code is a valid issn
    :param code:
    :return:
    """
    try:
        # TODO: replace this by a regular expression...
        parts = code.split('-')
        if len(parts[0]) == 4 and len(parts[1]) == 4:
            return True
        return False
    except Exception:
        return False


def encode_multipart_formdata(fields):
    boundary = binascii.hexlify(os.urandom(16)).decode('ascii')

    body = (
        "".join(
            "--%s\r\n"
            "Content-Disposition: form-data; name=\"%s\"\r\n"
            "\r\n"
            "%s\r\n" % (boundary, field, value)
            for field, value in fields.items()
            ) +
        "--%s--\r\n" % boundary
    )

    content_type = "multipart/form-data; boundary=%s" % boundary

    return body, content_type


def get_issn_count(res: requests.Response):
    doc1 = html.fromstring(res.text)
    element = doc1.xpath('.//h2[@class="page-items-count search-results-count"]')
    return int(element[0].text.split()[0])


def get_info_issn(issn: str, sess: requests.Session):
    sess.headers.update(get_iroko_harvester_agent())
    url = 'https://portal.issn.org/resource/ISSN/' + issn + '?format=json'
    response = sess.get(url)

    # text = response.content.decode('UTF-8').replace('\\n', '')
    # return text.replace('\\"', '"')

    # TODO: see if we need to do something if response.status_code != 200:

    return json.loads(response.content.decode('UTF-8'))


def getissn(res: requests.Response):
    doc1 = html.fromstring(res.text)
    result = []
    element = doc1.xpath('.//div[@class="item-result-content-text flex-zero"]//p[1]')
    for v in element:
        result.append(v.text_content().split()[1])
    return result


def request_advanced_search(country_code):
    url = 'https://portal.issn.org/advancedsearch'

    sess = requests.Session()

    sess.headers.update(get_iroko_harvester_agent())

    response = sess.get(url)

    doc1 = html.fromstring(response.content.decode('UTF-8'))
    variable = ["form_build_id", "form_id", "honeypot_time"]
    variable_id = "advances-search-issn-final-form"
    dictionary = {}

    for v in variable:
        element = doc1.xpath('.//form[@id="' + variable_id + '"]//input[@name="' + v + '"]')
        dictionary[v] = element[0].value

    print(dictionary)
    sleep_time = randint(3, 9)
    print('sleep {0} seconds'.format(sleep_time))
    time.sleep(sleep_time)

    body = {
        'operator': 'SHOULD',
        'selectTitle': 'all',
        'opSpeTitle': '0',
        'issnOpt': 'all',
        'globalField[fld][0][operator]': 'MUST',
        'globalField[fld][0][selector]': 'country',
        'globalField[fld][0][countryDiv][countrylist][' + country_code + ']': country_code,
        'op': 'Search',
        'form_id': dictionary['form_id'],
        'honeypot_time': dictionary['honeypot_time'],
        'form_build_id': dictionary['form_build_id']
        }

    data, content_type = encode_multipart_formdata(body)

    headers = {
        'User-Agent': 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:64.0) Gecko/20100101 '
                      'Firefox/64.0',
        'Content-Type': content_type,
        'Cache-Control': 'no-cache',
        }

    resp = sess.post(url, data=data, headers=headers)

    return resp, sess


def collect_issn_list(country_code):
    """get all cuban ISSNs """

    issn_list = []
    fres, session = request_advanced_search(country_code)
    issn_list = getissn(fres)

    session.headers.update(get_iroko_harvester_agent())

    sleep_time = randint(3, 9)
    print('sleep {0} seconds'.format(sleep_time))
    time.sleep(sleep_time)

    page = 2
    size = 100
    count = (get_issn_count(fres) - 10) / size

    while page < count:
        print('getting: {0}'.format(fres.url + '&currentpage={0}&size={1}'.format(page, size)))
        resp = session.get(fres.url + '&currentpage={0}&size={1}'.format(page, size))
        issn_list.extend(getissn(resp))

        sleep_time = randint(3, 9)
        print('sleep {0} seconds'.format(sleep_time))
        time.sleep(sleep_time)

        page += 1

    return issn_list


def collect_issn_info(issn_list):
    session = requests.Session()

    session.headers.update(get_iroko_harvester_agent())

    result = dict()

    for issn in issn_list:
        try:
            print('try getting {0} info'.format(issn))
            result[issn] = get_info_issn(issn, session)
        except Exception as e:
            print('error, getting {0} info, error: {1}'.format(issn, e))
            result[issn] = {"error": str(e)}
            pass
        finally:
            print('ok, saving to file')
            # with open(self.issn_info_file, 'w+', encoding=('UTF-8')) as file_issn:
            #     print('writing to file {0}'.format(self.cuban_issn_info_file))
            #     json.dump(result, file_issn, ensure_ascii=False)

            sleep_time = randint(4, 7)
            print('finally, sleep {0} seconds'.format(sleep_time))
            time.sleep(sleep_time)
    return result


def collect_issn_info_single(issn):
    session = requests.Session()

    session.headers.update(get_iroko_harvester_agent())

    result = dict()

    try:
        print('try getting {0} info'.format(issn))
        result[issn] = get_info_issn(issn, session)
    except Exception as e:
        print('error, getting {0} info, error: {1}'.format(issn, e))
        result[issn] = {"error": str(e)}
        pass
    finally:
        print('ok')
        pass
    return result


class IssnDataParser:
    """
    adhoc class for issn.org data,
    assume in
    source = SourceRawData.query.filter_by(identifier=identifier).first()
    data = source.get_data_field('issn')
    jsonschema-LD format
    """

    @classmethod
    def parse(cls, identifier) -> dict:
        """

        :param identifier: to SourceRawData.query.filter_by(identifier=identifier).first()
        :return:
        [list, str, list]
        1- a list with identifiers, valid for a SourceRecord
        2- the mainTitle
        3- all known aliases

        """
        return cls._parse_data(identifier, parse_relations=True)

    @classmethod
    def _parse_data(cls, identifier, parse_relations=False) -> dict:
        """
        helper func
        :param
        :param parse_relations: if True, will analize 'isFormatOf' and 'otherPhysicalFormat' fields
        :return:
        """

        if not is_issn(identifier):
            return None

        main_title = ''
        identifiers = list()
        aliases = list()

        source = SourceRawData.query.filter_by(identifier=identifier).first()
        if not source:
            return None
            # esto significa que no se recolecto el issn, recolectarlo entonces!!!!
            print('NO SOURCE', identifier)
            issndata = collect_issn_info_single(identifier)
            obj_issn = SourceRawData()
            obj_issn.identifier = identifier
            obj_issn.set_data_field('issn', issndata)
            db.session.add(obj_issn)
            db.session.commit()
            source = SourceRawData.query.filter_by(identifier=identifier).first()

        data = source.get_data_field('issn')

        if '@graph' not in data:
            print('NO DATA', identifier, data)
            return None

        for item in data["@graph"]:

            # ISSN Link (issn_l) is the main issn
            if '@type' in item and item['@type'] == 'http://id.loc.gov/ontologies/bibframe/IssnL':
                cls._append_to_identifier_list(
                    identifiers, {'idtype': 'issn_l', 'value': item['value']}
                    )

            if '@id' in item and item['@id'] == 'resource/ISSN/' + identifier:
                cls._append_to_identifier_list(identifiers, cls._get_issn(item))

                if 'mainTitle' in item:
                    main_title = item['mainTitle']
                    pass
                if 'name' in item:
                    name = item['name']
                    if type(name) == list:
                        for n in name:
                            cls._append_to_alias_list(aliases, n)
                    else:
                        cls._append_to_alias_list(aliases, name)
                else:
                    print('no name!!!!', identifier)
                    pass

                # issn relations.
                if parse_relations:
                    try:
                        rel = cls._get_relations(item)
                        if main_title == '' and rel.title != '':
                            main_title = rel.title
                        for _id in rel.identifiers:
                            cls._append_to_identifier_list(identifiers, _id)
                        for a in rel.aliases:
                            cls._append_to_alias_list(aliases, a)
                    except Exception as e:
                        # print(e)
                        pass
        if main_title == '':
            print('no mainTitle', identifier)
            if len(aliases) > 0:
                main_title = aliases[0]
        record = dict()
        record['identifiers'] = identifiers
        record['name'] = main_title
        record['title'] = main_title
        record['aliases'] = aliases

        return record

    @classmethod
    def _append_to_identifier_list(cls, id_list: list, id_item):
        append = True
        for id in id_list:
            if id['idtype'] == id_item['idtype']:
                append = False
        if append:
            id_list.append(id_item)

    @classmethod
    def _append_to_alias_list(cls, alias_list: list, alias_item):
        append = True
        for alias in alias_list:
            if alias == alias_item:
                append = False
        if append:
            alias_list.append(alias_item)

    @classmethod
    def _get_relations(cls, item):
        # TODO: 'isFormatOf' is the same as 'otherPhysicalFormat'?
        relations = None
        if 'isFormatOf' in item:
            relations = item['isFormatOf']
        elif 'otherPhysicalFormat' in item:
            relations = item['otherPhysicalFormat']
        else:
            # print('no relations', item)
            pass
        if type(relations) == str:
            issn_rel = relations.split('/').pop()
            return cls._parse_data(issn_rel)
        if type(relations) == list:
            result_ids = list()
            result_alias = list()
            result_title = ''
            for rel in relations:
                issn_rel = rel.split('/').pop()
                rel = cls._parse_data(issn_rel)
                if result_title == '':
                    result_title = rel.title
                result_ids.extend(rel.identifiers)
                result_alias.extend(rel.aliases)

            record = dict()
            record['identifiers'] = result_ids
            record['title'] = result_title
            record['aliases'] = result_alias

            return record

    @classmethod
    def _get_issn(cls, item):
        # desambiguate all issn types
        if item['format'] == 'vocabularies/medium#Online':
            return {'idtype': 'issn_e', 'value': item['issn']}
        if item['format'] == 'vocabularies/medium#Print':
            return {'idtype': 'issn_p', 'value': item['issn']}
        if item['format'] == 'vocabularies/medium#DigitalCarrier':
            return {'idtype': 'issn_c', 'value': item['issn']}
        if item['format'] == 'vocabularies/medium#Other':
            return {'idtype': 'issn_o', 'value': item['issn']}


class IssnHarvester(BaseHarvester):
    """
    TODO: Document this better!!!
    Harvest all cuban ISSN from issn.org"""

    # TODO: all functions private, except process_pipeline

    def __init__(
        self, work_dir, country_code='CUB', country_id='http://id.loc.gov/vocabulary/countries/cu'
        ):

        self.work_dir = os.path.join(work_dir, 'issn')
        if not os.path.exists(self.work_dir):
            os.mkdir(self.work_dir)

        self.country_code = country_code
        self.country_id = country_id

        self.issn_list_file = os.path.join(self.work_dir, self.country_code + '.list.json')
        self.issn_info_file = os.path.join(self.work_dir, self.country_code + '.info.json')

    def process_pipeline(self):
        """get all cuban issn and its info
        return pair issns(array of all issns), infos(dict with issns info)
        """

        issn_list = self.get_issn_list(remote=True)
        issn_info = self.get_issn_info(remote=True)
        return issn_list, issn_info

    def get_issn_list(self, remote):
        if remote:
            issn_list = collect_issn_list(self.country_code)
            with open(self.issn_list_file, 'w+', encoding='UTF-8') as file_issn:
                if file_issn:
                    print(issn_list)
                    json.dump(issn_list, file_issn, ensure_ascii=False)
        else:
            with open(self.issn_list_file, 'r', encoding='UTF-8') as file_issn:
                issn_list = json.load(file_issn)

        return issn_list

    def get_issn_info(self, remote):
        issn_list = self.get_issn_list(remote=False)
        if remote and issn_list:
            issn_info = collect_issn_info(issn_list)
            with open(self.issn_info_file, 'w+', encoding='UTF-8') as file_issn:
                if file_issn:
                    json.dump(issn_info, file_issn, ensure_ascii=False)
        else:
            with open(self.issn_info_file, 'r', encoding='UTF-8') as file_issn:
                issn_info = json.load(file_issn)

        return issn_info


class IssnHarvesterManager:
    @classmethod
    def collect_issn(cls):
        work_dir = current_app.config['IROKO_DATA_DIRECTORY']
        harvester = IssnHarvester(work_dir)

        return harvester.process_pipeline()

        # issns = harvester.get_issn_list(remoteissns)
        #
        # if info:
        #     infos = harvester.get_cuban_issns_info_json(issns, remoteinfo)
        #     # con lo que hay en el dic, crear/actualizar, versiones de source cuyo comentario
        #     sea issn...

    # @classmethod
    # def get_cuban_issns(cls):
    #     work_dir = current_app.config['IROKO_DATA_DIRECTORY']
    #     harvester = IssnHarvester(work_dir)
    #
    #     issns = harvester.get_issn_list(False)
    #     infos = harvester.get_cuban_issns_info_json(issns, False)
    #
    #     return infos

    @classmethod
    def sync_db(cls):
        work_dir = current_app.config['IROKO_DATA_DIRECTORY']
        harvester = IssnHarvester(work_dir)
        issn_list = harvester.get_issn_list(remote=False)
        issn_info = harvester.get_issn_info(remote=False)

        for issn in issn_list:
            data = issn_info[issn]
            issn_model = SourceRawData.query.filter_by(identifier=issn).first()
            print(issn_model)
            if not issn_model:
                obj_issn = SourceRawData()
                obj_issn.identifier = issn
                obj_issn.set_data_field('issn', data)
                db.session.add(obj_issn)
                print('NEW')
            else:
                issn_model.set_data_field('issn', data)
                print('UPDATED')
            db.session.flush()
        db.session.commit()
        print('DB COMMIT')

    @classmethod
    def sync_records(cls):

        issn_list = SourceRawData.query.all()
        for issn in issn_list:
            record = IssnDataParser.parse(issn.identifier)
            if record:
                user = get_default_user()
                record['source_status'] = SourceStatus.UNOFFICIAL.value
                record['source_type'] = SourceType.JOURNAL.value
                print(record)
                SourceRecord.new_source_revision(record, user_id=user.id, comment='issn.org import')
                # SourceRecord.create_or_update(record.dump(record))

#  Copyright (c) 2021. Universidad de Pinar del Rio #  This file is part of SCEIBA (sceiba.cu). #  SCEIBA is free software; you can redistribute it and/or modify it #  under the terms of the MIT License; see LICENSE file for more details. # return true if code is a valid issn     :param code:     :return: # TODO: replace this by a regular expression... # text = response.content.decode('UTF-8').replace('\\n', '') # return text.replace('\\"', '"') # TODO: see if we need to do something if response.status_code != 200: get all cuban ISSNs # with open(self.issn_info_file, 'w+', encoding=('UTF-8')) as file_issn: #     print('writing to file {0}'.format(self.cuban_issn_info_file)) #     json.dump(result, file_issn, ensure_ascii=False) adhoc class for issn.org data,     assume in     source = SourceRawData.query.filter_by(identifier=identifier).first()     data = source.get_data_field('issn')     jsonschema-LD format :param identifier: to SourceRawData.query.filter_by(identifier=identifier).first()         :return:         [list, str, list]         1- a list with identifiers, valid for a SourceRecord         2- the mainTitle         3- all known aliases helper func         :param         :param parse_relations: if True, will analize 'isFormatOf' and 'otherPhysicalFormat' fields         :return: # esto significa que no se recolecto el issn, recolectarlo entonces!!!! # ISSN Link (issn_l) is the main issn # issn relations. # print(e) # TODO: 'isFormatOf' is the same as 'otherPhysicalFormat'? # print('no relations', item) # desambiguate all issn types #Online': #Print': #DigitalCarrier': #Other': TODO: Document this better!!!     Harvest all cuban ISSN from issn.org # TODO: all functions private, except process_pipeline get all cuban issn and its info         return pair issns(array of all issns), infos(dict with issns info) # issns = harvester.get_issn_list(remoteissns) # # if info: #     infos = harvester.get_cuban_issns_info_json(issns, remoteinfo) #     # con lo que hay en el dic, crear/actualizar, versiones de source cuyo comentario #     sea issn... # @classmethod # def get_cuban_issns(cls): #     work_dir = current_app.config['IROKO_DATA_DIRECTORY'] #     harvester = IssnHarvester(work_dir) # #     issns = harvester.get_issn_list(False) #     infos = harvester.get_cuban_issns_info_json(issns, False) # #     return infos # SourceRecord.create_or_update(record.dump(record))