import os
import sys
import json
import gzip

import pandas as pd
import numpy as np
from nameparser import HumanName
import requests
from lxml import etree
from io import BytesIO

# determine if we are loading from a jupyter notebook (to make pretty progress bars)
if 'ipykernel' in sys.modules:
    from tqdm.notebook import tqdm
else:
    from tqdm import tqdm

from pyscisci.datasource.readwrite import load_preprocessed_data, load_int, load_float, load_html_str, load_xml_text
from pyscisci.database import BibDataBase


class WOS(BibDataBase):
    """
    Base class for Web of Science interface.

    """

    def __init__(self, path2database='', keep_in_memory=False, global_filter=None, show_progress=True):

        self._default_init(path2database, keep_in_memory, global_filter, show_progress)

        self.PublicationIdType = int
        self.AffiliationIdType = str
        self.AuthorIdType = str

    def download_from_source(self):

        raise NotImplementedError("The Web of Science (WOS) is proprietary owned by Clarivate Analytics.  Contact their sales team to aquire access to the data.")

    def parse_affiliations(self, preprocess = False, show_progress=False):
        raise NotImplementedError("WOS is stored as a xml archive.  Run preprocess to parse the archive.")

    def parse_authors(self, preprocess = False, process_name = True, num_file_lines = 5*10**6, show_progress=False):
        raise NotImplementedError("WOS does not contain disambiguated author information.")

    def _blank_wos_publication(self, PublicationId = 0):
        record = {}
        record['PublicationId'] = PublicationId
        record['Title'] = ''
        record['Year'] = 0
        record['Volume'] = 0
        record['Issue'] = ''
        record['Pages'] = ''
        record['JournalId'] = ''
        record['TeamSize'] = 0
        record['Date'] = 1
        record['DocType'] = ''
        record['ISSN'] = ''
        record['Doi'] = ''

        return record

    def _blank_wos_author(self, AuthorId = None):
        record = {}
        record['AuthorId'] = AuthorId
        record['FullName'] = ''
        record['FirstName'] = ''
        record['LastName'] = ''
        record['OrigAuthorName'] = ''
        return record

    def _blank_wos_affiliation(self):
        record = {}
        record['FullAddress'] = ''
        record['Organizations'] = ''
        record['SubOrganizations'] = ''
        record['City'] = ''
        record['Country'] = ''
        return record

    def _save_dataframes(self, ifile, publication_df, pub_column_names, author_df, author_columns, paa_df, pub2ref_df, affiliation_df, field_df):

        publication_df = pd.DataFrame(publication_df, columns=pub_column_names)
        publication_df['PublicationId'] = publication_df['PublicationId']
        publication_df['Year'] = publication_df['Year'].astype(int)
        publication_df['Volume'] = pd.to_numeric(publication_df['Volume'])
        publication_df['TeamSize'] = publication_df['TeamSize'].astype(int)
        publication_df.to_hdf( os.path.join(self.path2database, 'publication', 'publication{}.hdf'.format(ifile)), key = 'pub', mode='w')


        author_df = pd.DataFrame(author_df, columns = author_columns)
        author_df['AuthorId'] = author_df['AuthorId'].astype(int)
        author_df.to_hdf( os.path.join(self.path2database, 'author', 'author{}.hdf'.format(ifile)), key = 'author', mode='w')

        paa_df = pd.DataFrame(paa_df, columns = ['PublicationId', 'AuthorId', 'AffiliationId', 'AuthorSequence', 'OrigAuthorName'])
        paa_df.to_hdf( os.path.join(self.path2database, 'publicationauthoraffiliation', 'publicationauthoraffiliation{}.hdf'.format(ifile)), key = 'pa', mode='w')

        pub2ref_df = pd.DataFrame(pub2ref_df, columns = ['CitingPublicationId', 'CitedPubliationId'])
        pub2ref_df.to_hdf(os.path.join(self.path2database, 'pub2ref', 'pub2ref{}.hdf'.format(ifile)), key = 'pub2ref', mode='w')

        affiliation_df = pd.DataFrame(affiliation_df, columns = ['PublicationId', 'AffiliationId', 'AffiliationString'])
        affiliation_df.to_hdf(os.path.join(self.path2database, 'affiliation', 'affiliation{}.hdf'.format(ifile)), key = 'affiliation', mode='w')

        field_df = pd.DataFrame(field_df, columns = ['PublicationId', 'FieldId', 'FieldType'])
        field_df.to_hdf(os.path.join(self.path2database, 'pub2field', 'pub2field{}.hdf'.format(ifile)), key = 'pub2field', mode='w')

    def preprocess(self, xml_directory = 'RawXML', name_space = 'http://scientific.thomsonreuters.com/schema/wok5.4/public/FullRecord',
        process_name=True, num_file_lines=10**6, show_progress=True):
        """
        Bulk preprocess of the Web of Science raw data.

        Parameters
        ----------
        process_name: bool, default True
            If True, then when processing the raw file, the package `NameParser <https://nameparser.readthedocs.io/en/latest/>`_
            will be used to split author FullNames.

        xml_file_name: str, default 'dblp.xml.gz'
            The xml file name.

        num_file_lines: int, default 10**6
            The processed data will be saved into smaller DataFrames, each with `num_file_lines` rows.

        show_progress: bool, default True
            Show progress with processing of the data.

        """

        pub_column_names = ['PublicationId', 'Year', 'JournalId', 'Doi', 'ISSN', 'Title', 'Date', 'Volume', 'Issue', 'Pages', 'DocType', 'TeamSize']
        author_column_names = ['AuthorId', 'FullName', 'FirstName', 'LastName']

        if show_progress:
            print("Starting to preprocess the WOS database.")


        for hier_dir_type in ['publication', 'author', 'publicationauthoraffiliation', 'pub2field', 'pub2ref', 'affiliation']:

            if not os.path.exists(os.path.join(self.path2database, hier_dir_type)):
                os.mkdir(os.path.join(self.path2database, hier_dir_type))


        pub2year = {}
        pub2doctype = {}

        found_aids = set([])

        found_affiliations = {}


        ns = {"ns": name_space}
        xmlfiles = sorted([fname for fname in os.listdir(os.path.join(self.path2database, xml_directory)) if '.xml' in fname])

        ifile = 0
        for xml_file_name in tqdm(xmlfiles, desc='WOS xml files', leave=True, disable=not show_progress):

            publication_df = []
            author_df = []
            paa_df = []
            pub2field_df = []
            pub2ref_df = []
            affiliation_df = []
            field_df = []

            name, extension = os.path.splitext(xml_file_name)

            if extension == '.gz':
                with gzip.open(os.path.join(self.path2database, xml_directory, xml_file_name), 'r') as infile:
                    xml_file = infile.read()
                bytesxml = BytesIO(xml_file)

            elif extension == '.xml':
                with open(os.path.join(self.path2database, xml_directory, xml_file_name), 'r') as infile:
                    xml_file = infile.read()

            # extract the desired fields from the XML tree  #
            
            xmltree = etree.iterparse(bytesxml, events=('end',), tag="{{{0}}}REC".format(name_space))

            if show_progress:
                print("{} Xml tree parsed, iterating through elements.".format(xml_file_name))

            last_position = 0

            for event, elem in xmltree:

                # scrape the publication information
                PublicationId = load_html_str(elem.xpath('./ns:UID', namespaces=ns)[0].text)

                pub_record = self._blank_wos_publication(PublicationId)

                pub_record['Title'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:titles/ns:title[@type="item"]', namespaces=ns)))
                pub_record['JournalId'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:titles/ns:title[@type="source"]', namespaces=ns)))

                pub_info = elem.xpath('./ns:static_data/ns:summary/ns:pub_info', namespaces=ns)[0]
                pub_record['Year'] = load_int(pub_info.get('pubyear', ''))
                pub_record['Date'] = load_html_str(pub_info.get('sortdate', ''))
                pub_record['Volume'] = load_int(pub_info.get('vol', ''))
                pub_record['Issue'] = load_int(pub_info.get('issue', ''))

                pub2year[PublicationId] = pub_record['Year']

                pub_record['Pages'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:pub_info/ns:page', namespaces=ns), default=''))

                for ident in ['ISSN', 'Doi']:
                    identobject = elem.xpath('./ns:dynamic_data/ns:cluster_related/ns:identifiers/ns:identifier[@type="{}"]'.format(ident.lower()), namespaces=ns)
                    if len(identobject) > 0:
                        pub_record[ident] =load_html_str( identobject[0].get('value', ''))


                #load_html_str(load_xml_text(elem.xpath('./ns:dynamic_data/ns:cluster_related/ns:identifiers/ns:identifier[@type="doi"]', namespaces=ns)))

                pub_record['DocType'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:doctypes/ns:doctype', namespaces=ns)))

                pub2doctype[PublicationId] = pub_record['DocType']

                # now scrape the authors
                pub_authors = {}
                author_objects = elem.xpath('./ns:static_data/ns:summary/ns:names/ns:name[@role="author"]', namespaces=ns)
                pub_record['TeamSize'] = len(author_objects)

                for author_obj in author_objects:
                    author_record = self._blank_wos_author(None)
                    author_record['AuthorId'] = author_obj.get('dais_id', None)

                    author_record['FullName'] = load_html_str(load_xml_text(author_obj.xpath('./ns:full_name', namespaces=ns)))
                    author_record['FirstName'] = load_html_str(load_xml_text(author_obj.xpath('./ns:first_name', namespaces=ns)))
                    author_record['LastName'] = load_html_str(load_xml_text(author_obj.xpath('./ns:last_name', namespaces=ns)))

                    author_record['Affiliations'] = author_obj.get('addr_no', '')
                    author_record['Affiliations'] = [int(single_addr_no) for single_addr_no in author_record['Affiliations'].split(' ') if len(single_addr_no) > 0]

                    author_record['AuthorOrder'] = int(author_obj.get('seq_no', None))

                    pub_authors[author_record['AuthorOrder']] = author_record


                #contributor_objects = elem.xpath('./ns:static_data/ns:contributors/ns:contributor/ns:name[@role="researcher_id"]', namespaces=ns)

                address_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:addresses/ns:address_name/ns:address_spec', namespaces=ns)
                for addr_obj in address_objects:
                    addr_record = self._blank_wos_affiliation()

                    organization_objects = addr_obj.xpath('./ns:organizations/ns:organization[@pref="Y"]', namespaces=ns)
                    if len(organization_objects) == 0:
                        organization_objects = addr_obj.xpath('./ns:organizations/ns:organization', namespaces=ns)

                    if len(organization_objects) == 0:
                        orgtext = ''
                    else:
                        orgtext = organization_objects[0].text
                    
                    address_no = int(addr_obj.get('addr_no'))

                    affiliation_df.append([PublicationId, addr_no, orgtext])

                    #if found_affiliations

                    #article['addresses'][address_no] = address_info


                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:headings/ns:heading', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'heading'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:subheadings/ns:subheading', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'subheading'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:subjects/ns:subject[@ascatype="traditional"]', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'ASCA traditional subject'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:subjects/ns:subject[@ascatype="extended"]', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'ASCA extended subject'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:keywords/ns:keyword', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'keyword'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:item/ns:keywords_plus/ns:keyword', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'keyword plus'] for field_obj in field_objects if field_obj is not None])

                reference_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:references/ns:reference', namespaces=ns)
                for ref_obj in reference_objects:
                    for ref_elem in ref_obj:
                        if ref_elem.tag == "{{{0}}}uid".format(name_space):
                            refid = load_html_str(ref_elem.text.replace('WOS:', ''))
                            pub2ref_df.append([PublicationId, refid])
                        elif ref_elem.tag == "{{{0}}}year".format(name_space):
                            pub2year[refid] = load_int(ref_elem.text)

                publication_df.append([pub_record[k] for k in pub_column_names])

                for aorder, author_record in pub_authors.items():
                    if not author_record['AuthorId'] is None and not author_record['AuthorId'] in found_aids:
                        found_aids.add(author_record['AuthorId'])
                        author_df.append([author_record[k] for k in author_column_names])

                    paa_df.append([PublicationId, author_record['AuthorId'], aorder, author_record['FullName']])


            self._save_dataframes(ifile, publication_df, pub_column_names, author_df, author_column_names, paa_df, pub2ref_df, affiliation_df, field_df)
            ifile += 1

        with gzip.open(os.path.join(self.path2database, 'pub2year.json.gz'), 'w') as outfile:
            outfile.write(json.dumps(pub2year).encode('utf8'))

        with gzip.open(os.path.join(self.path2database, 'pub2doctype.json.gz'), 'w') as outfile:
            outfile.write(json.dumps(pub2doctype).encode('utf8'))


import os
import sys
import json
import gzip

import pandas as pd
import numpy as np
from nameparser import HumanName
import requests
from lxml import etree
from io import BytesIO

# determine if we are loading from a jupyter notebook (to make pretty progress bars)
if 'ipykernel' in sys.modules:
    from tqdm.notebook import tqdm
else:
    from tqdm import tqdm

from pyscisci.datasource.readwrite import load_preprocessed_data, load_int, load_float, load_html_str, load_xml_text
from pyscisci.database import BibDataBase


class WOS(BibDataBase):
    """
    Base class for Web of Science interface.

    """

    def __init__(self, path2database='', keep_in_memory=False, global_filter=None, show_progress=True):

        self._default_init(path2database, keep_in_memory, global_filter, show_progress)

        self.PublicationIdType = int
        self.AffiliationIdType = str
        self.AuthorIdType = str

    def download_from_source(self):

        raise NotImplementedError("The Web of Science (WOS) is proprietary owned by Clarivate Analytics.  Contact their sales team to aquire access to the data.")

    def parse_affiliations(self, preprocess = False, show_progress=False):
        raise NotImplementedError("WOS is stored as a xml archive.  Run preprocess to parse the archive.")

    def parse_authors(self, preprocess = False, process_name = True, num_file_lines = 5*10**6, show_progress=False):
        raise NotImplementedError("WOS does not contain disambiguated author information.")

    def _blank_wos_publication(self, PublicationId = 0):
        record = {}
        record['PublicationId'] = PublicationId
        record['Title'] = ''
        record['Year'] = 0
        record['Volume'] = 0
        record['Issue'] = ''
        record['Pages'] = ''
        record['JournalId'] = ''
        record['TeamSize'] = 0
        record['Date'] = 1
        record['DocType'] = ''
        record['ISSN'] = ''
        record['Doi'] = ''

        return record

    def _blank_wos_author(self, AuthorId = None):
        record = {}
        record['AuthorId'] = AuthorId
        record['FullName'] = ''
        record['FirstName'] = ''
        record['LastName'] = ''
        record['OrigAuthorName'] = ''
        return record

    def _blank_wos_affiliation(self):
        record = {}
        record['FullAddress'] = ''
        record['Organizations'] = ''
        record['SubOrganizations'] = ''
        record['City'] = ''
        record['Country'] = ''
        return record

    def _save_dataframes(self, ifile, publication_df, pub_column_names, author_df, author_columns, paa_df, pub2ref_df, affiliation_df, field_df):

        publication_df = pd.DataFrame(publication_df, columns=pub_column_names)
        publication_df['PublicationId'] = publication_df['PublicationId']
        publication_df['Year'] = publication_df['Year'].astype(int)
        publication_df['Volume'] = pd.to_numeric(publication_df['Volume'])
        publication_df['TeamSize'] = publication_df['TeamSize'].astype(int)
        publication_df.to_hdf( os.path.join(self.path2database, 'publication', 'publication{}.hdf'.format(ifile)), key = 'pub', mode='w')


        author_df = pd.DataFrame(author_df, columns = author_columns)
        author_df['AuthorId'] = author_df['AuthorId'].astype(int)
        author_df.to_hdf( os.path.join(self.path2database, 'author', 'author{}.hdf'.format(ifile)), key = 'author', mode='w')

        paa_df = pd.DataFrame(paa_df, columns = ['PublicationId', 'AuthorId', 'AffiliationId', 'AuthorSequence', 'OrigAuthorName'])
        paa_df.to_hdf( os.path.join(self.path2database, 'publicationauthoraffiliation', 'publicationauthoraffiliation{}.hdf'.format(ifile)), key = 'pa', mode='w')

        pub2ref_df = pd.DataFrame(pub2ref_df, columns = ['CitingPublicationId', 'CitedPubliationId'])
        pub2ref_df.to_hdf(os.path.join(self.path2database, 'pub2ref', 'pub2ref{}.hdf'.format(ifile)), key = 'pub2ref', mode='w')

        affiliation_df = pd.DataFrame(affiliation_df, columns = ['PublicationId', 'AffiliationId', 'AffiliationString'])
        affiliation_df.to_hdf(os.path.join(self.path2database, 'affiliation', 'affiliation{}.hdf'.format(ifile)), key = 'affiliation', mode='w')

        field_df = pd.DataFrame(field_df, columns = ['PublicationId', 'FieldId', 'FieldType'])
        field_df.to_hdf(os.path.join(self.path2database, 'pub2field', 'pub2field{}.hdf'.format(ifile)), key = 'pub2field', mode='w')

    def preprocess(self, xml_directory = 'RawXML', name_space = 'http://scientific.thomsonreuters.com/schema/wok5.4/public/FullRecord',
        process_name=True, num_file_lines=10**6, show_progress=True):
        """
        Bulk preprocess of the Web of Science raw data.

        Parameters
        ----------
        process_name: bool, default True
            If True, then when processing the raw file, the package `NameParser <https://nameparser.readthedocs.io/en/latest/>`_
            will be used to split author FullNames.

        xml_file_name: str, default 'dblp.xml.gz'
            The xml file name.

        num_file_lines: int, default 10**6
            The processed data will be saved into smaller DataFrames, each with `num_file_lines` rows.

        show_progress: bool, default True
            Show progress with processing of the data.

        """

        pub_column_names = ['PublicationId', 'Year', 'JournalId', 'Doi', 'ISSN', 'Title', 'Date', 'Volume', 'Issue', 'Pages', 'DocType', 'TeamSize']
        author_column_names = ['AuthorId', 'FullName', 'FirstName', 'LastName']

        if show_progress:
            print("Starting to preprocess the WOS database.")


        for hier_dir_type in ['publication', 'author', 'publicationauthoraffiliation', 'pub2field', 'pub2ref', 'affiliation']:

            if not os.path.exists(os.path.join(self.path2database, hier_dir_type)):
                os.mkdir(os.path.join(self.path2database, hier_dir_type))


        pub2year = {}
        pub2doctype = {}

        found_aids = set([])

        found_affiliations = {}


        ns = {"ns": name_space}
        xmlfiles = sorted([fname for fname in os.listdir(os.path.join(self.path2database, xml_directory)) if '.xml' in fname])

        ifile = 0
        for xml_file_name in tqdm(xmlfiles, desc='WOS xml files', leave=True, disable=not show_progress):

            publication_df = []
            author_df = []
            paa_df = []
            pub2field_df = []
            pub2ref_df = []
            affiliation_df = []
            field_df = []

            name, extension = os.path.splitext(xml_file_name)

            if extension == '.gz':
                with gzip.open(os.path.join(self.path2database, xml_directory, xml_file_name), 'r') as infile:
                    xml_file = infile.read()
                bytesxml = BytesIO(xml_file)

            elif extension == '.xml':
                with open(os.path.join(self.path2database, xml_directory, xml_file_name), 'r') as infile:
                    xml_file = infile.read()

            # extract the desired fields from the XML tree  #
            
            xmltree = etree.iterparse(bytesxml, events=('end',), tag="{{{0}}}REC".format(name_space))

            if show_progress:
                print("{} Xml tree parsed, iterating through elements.".format(xml_file_name))

            last_position = 0

            for event, elem in xmltree:

                # scrape the publication information
                PublicationId = load_html_str(elem.xpath('./ns:UID', namespaces=ns)[0].text)

                pub_record = self._blank_wos_publication(PublicationId)

                pub_record['Title'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:titles/ns:title[@type="item"]', namespaces=ns)))
                pub_record['JournalId'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:titles/ns:title[@type="source"]', namespaces=ns)))

                pub_info = elem.xpath('./ns:static_data/ns:summary/ns:pub_info', namespaces=ns)[0]
                pub_record['Year'] = load_int(pub_info.get('pubyear', ''))
                pub_record['Date'] = load_html_str(pub_info.get('sortdate', ''))
                pub_record['Volume'] = load_int(pub_info.get('vol', ''))
                pub_record['Issue'] = load_int(pub_info.get('issue', ''))

                pub2year[PublicationId] = pub_record['Year']

                pub_record['Pages'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:pub_info/ns:page', namespaces=ns), default=''))

                for ident in ['ISSN', 'Doi']:
                    identobject = elem.xpath('./ns:dynamic_data/ns:cluster_related/ns:identifiers/ns:identifier[@type="{}"]'.format(ident.lower()), namespaces=ns)
                    if len(identobject) > 0:
                        pub_record[ident] =load_html_str( identobject[0].get('value', ''))


                #load_html_str(load_xml_text(elem.xpath('./ns:dynamic_data/ns:cluster_related/ns:identifiers/ns:identifier[@type="doi"]', namespaces=ns)))

                pub_record['DocType'] = load_html_str(load_xml_text(elem.xpath('./ns:static_data/ns:summary/ns:doctypes/ns:doctype', namespaces=ns)))

                pub2doctype[PublicationId] = pub_record['DocType']

                # now scrape the authors
                pub_authors = {}
                author_objects = elem.xpath('./ns:static_data/ns:summary/ns:names/ns:name[@role="author"]', namespaces=ns)
                pub_record['TeamSize'] = len(author_objects)

                for author_obj in author_objects:
                    author_record = self._blank_wos_author(None)
                    author_record['AuthorId'] = author_obj.get('dais_id', None)

                    author_record['FullName'] = load_html_str(load_xml_text(author_obj.xpath('./ns:full_name', namespaces=ns)))
                    author_record['FirstName'] = load_html_str(load_xml_text(author_obj.xpath('./ns:first_name', namespaces=ns)))
                    author_record['LastName'] = load_html_str(load_xml_text(author_obj.xpath('./ns:last_name', namespaces=ns)))

                    author_record['Affiliations'] = author_obj.get('addr_no', '')
                    author_record['Affiliations'] = [int(single_addr_no) for single_addr_no in author_record['Affiliations'].split(' ') if len(single_addr_no) > 0]

                    author_record['AuthorOrder'] = int(author_obj.get('seq_no', None))

                    pub_authors[author_record['AuthorOrder']] = author_record


                #contributor_objects = elem.xpath('./ns:static_data/ns:contributors/ns:contributor/ns:name[@role="researcher_id"]', namespaces=ns)

                address_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:addresses/ns:address_name/ns:address_spec', namespaces=ns)
                for addr_obj in address_objects:
                    addr_record = self._blank_wos_affiliation()

                    organization_objects = addr_obj.xpath('./ns:organizations/ns:organization[@pref="Y"]', namespaces=ns)
                    if len(organization_objects) == 0:
                        organization_objects = addr_obj.xpath('./ns:organizations/ns:organization', namespaces=ns)

                    if len(organization_objects) == 0:
                        orgtext = ''
                    else:
                        orgtext = organization_objects[0].text
                    
                    address_no = int(addr_obj.get('addr_no'))

                    affiliation_df.append([PublicationId, addr_no, orgtext])

                    #if found_affiliations

                    #article['addresses'][address_no] = address_info


                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:headings/ns:heading', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'heading'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:subheadings/ns:subheading', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'subheading'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:subjects/ns:subject[@ascatype="traditional"]', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'ASCA traditional subject'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:category_info/ns:subjects/ns:subject[@ascatype="extended"]', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'ASCA extended subject'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:keywords/ns:keyword', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'keyword'] for field_obj in field_objects if field_obj is not None])

                field_objects = elem.xpath('./ns:static_data/ns:item/ns:keywords_plus/ns:keyword', namespaces=ns)
                field_df.extend([[PublicationId, field_obj.text, 'keyword plus'] for field_obj in field_objects if field_obj is not None])

                reference_objects = elem.xpath('./ns:static_data/ns:fullrecord_metadata/ns:references/ns:reference', namespaces=ns)
                for ref_obj in reference_objects:
                    for ref_elem in ref_obj:
                        if ref_elem.tag == "{{{0}}}uid".format(name_space):
                            refid = load_html_str(ref_elem.text.replace('WOS:', ''))
                            pub2ref_df.append([PublicationId, refid])
                        elif ref_elem.tag == "{{{0}}}year".format(name_space):
                            pub2year[refid] = load_int(ref_elem.text)

                publication_df.append([pub_record[k] for k in pub_column_names])

                for aorder, author_record in pub_authors.items():
                    if not author_record['AuthorId'] is None and not author_record['AuthorId'] in found_aids:
                        found_aids.add(author_record['AuthorId'])
                        author_df.append([author_record[k] for k in author_column_names])

                    paa_df.append([PublicationId, author_record['AuthorId'], aorder, author_record['FullName']])


            self._save_dataframes(ifile, publication_df, pub_column_names, author_df, author_column_names, paa_df, pub2ref_df, affiliation_df, field_df)
            ifile += 1

        with gzip.open(os.path.join(self.path2database, 'pub2year.json.gz'), 'w') as outfile:
            outfile.write(json.dumps(pub2year).encode('utf8'))

        with gzip.open(os.path.join(self.path2database, 'pub2doctype.json.gz'), 'w') as outfile:
            outfile.write(json.dumps(pub2doctype).encode('utf8'))


# determine if we are loading from a jupyter notebook (to make pretty progress bars) Base class for Web of Science interface. Bulk preprocess of the Web of Science raw data.          Parameters         ----------         process_name: bool, default True             If True, then when processing the raw file, the package `NameParser <https://nameparser.readthedocs.io/en/latest/>`_             will be used to split author FullNames.          xml_file_name: str, default 'dblp.xml.gz'             The xml file name.          num_file_lines: int, default 10**6             The processed data will be saved into smaller DataFrames, each with `num_file_lines` rows.          show_progress: bool, default True             Show progress with processing of the data. # extract the desired fields from the XML tree  # # scrape the publication information #load_html_str(load_xml_text(elem.xpath('./ns:dynamic_data/ns:cluster_related/ns:identifiers/ns:identifier[@type="doi"]', namespaces=ns))) # now scrape the authors #contributor_objects = elem.xpath('./ns:static_data/ns:contributors/ns:contributor/ns:name[@role="researcher_id"]', namespaces=ns) #if found_affiliations #article['addresses'][address_no] = address_info