<reponame>marissa-graham/network-similarity
import smtplib
import gspread
import csv
import re
import glob
import time
import urllib
import requests
import numpy as np
from oauth2client.service_account import ServiceAccountCredentials

def check_crossref(citation):
    """
    Get a single CrossRef item manually for a given citation.
    """
    base_url = "https://api.crossref.org/works?query.bibliographic="
    url = base_url + urllib.parse.quote(citation, errors='surrogateescape')
    response = requests.get(url, timeout=100).json()
    item = response['message']['items'][0]
    return item

def set_property(attribute, item, property):
    """Generic safe dictionary lookup setter."""
    try:
        attribute = item[property]
    except KeyError:
        pass

class Paper:
    """
    Store metadata for a specific paper.

    freeform_citation: lookup info we initialize the paper with

    doi : DOI number, if it exists
    url : DOI in URL form if it exists, generic link otherwise
    year : year of publication
    title : title of paper
    author : "author1 and author2 and author3", etc. TODO: crossref format?
    reference_count : number of references
    citation_count : number of times cited
    subjects : list of subjects given by CrossRef
    verified : Are we sure that our metadata is correct?
    
    item : full CrossRef item thing
    bibtex : nice string to put in BibTeX

    type: original, survey, or application?
    includes_alg : does it include an algorithm?
    prob_attrs : list of problem attributes
    strategies : strategies used in the paper to approach the problem
    notes : any notes I made while reading
    """
    def __init__(self):

        # Initialized with 
        self.freeform_citation = None

        # Can be done either manually or automatically
        self.doi = None
        self.url = None
        self.year = None
        self.title = ""
        self.author = []
        self.reference_count = 0
        self.citation_count = 0
        self.subjects = []
        self.verified = False

        # Non-spreadsheet
        self.item = None
        self.bibtex = None
        
        # Must always be done manually
        self.type = None
        self.includes_alg = False
        self.prob_attrs = []
        self.strategies = []
        self.notes = ""   

    def lookup(self, citation, base_url):
        """
        Look up the citation in CrossRef and assign values accordingly.
        """

        # Make the CrossRef API request
        self.freeform_citation = citation 
        url = base_url + urllib.parse.quote(citation, errors='surrogateescape')
        r = requests.get(url, timeout=100).json()
        self.crossref_item = r['message']['items'][0]

        # Get the straightforward properties
        set_property(self.doi, self.crossref_item, 'DOI')
        set_property(self.url, self.crossref_item, 'URL')
        set_property(self.reference_count, self.crossref_item, 'reference_count')
        set_property(self.citation_count, self.crossref_item, 'citation_count')
        set_property(self.subjects, self.crossref_item, 'subject')
            
        # Get the title
        try:
            self.title = self.crossref_item['title'][0]#.encode('ascii','ignore').decode()
        except KeyError: pass

        # Get the year of publication
        try:
            self.year = self.crossref_item['issued']['date_parts'][0][0]
        except KeyError: pass

        # Get the author list
        try:
            authors = self.crossref_item['author']
            n = len(authors)
            alist = [authors[i]['given']+" "+authors[i]['family'] for i in range(n)]
            self.author = " and ".join(alist)
        except KeyError: pass
    
    def __str__(self):
        """
        Print out in a nice format.
        """
        out = "\nTitle: " + self.title
        out += "\n\tAuthor(s): " + str(self.author)
        out += "\n\tReference count: " + str(self.reference_count)
        out += "\n\tCitation count: " + str(self.citation_count)
        out += "\n\tDOI number: " + self.doi
        out += "\n\tYear of Publication: " + self.year
        out += "\n\tSubject(s): " + self.field
        out += "\n\tType: " + str(self.original_work)
        out += "\n\tIncludes algorithm?: " + str(self.includes_alg)
        out += "\n\tProblem attributes: " + str(self.prob_attrs)
        out += "\n\tStrategies in approach: " + str(self.strategies)
        out += "\n\tNotes: " + self.notes
        return out

    def to_csv(self):
        """
        Return a list of attributes to be joined into a row in a .csv file
        """
        return [self.doi, self.year, self.title, self.author, self.citation, 
            self.bibtex, self.crossref_score, self.verified, self.index, 
            self.in_spreadsheet, self.field, self.original_work, self.includes_alg, 
            self.prob_attrs, self.strategies, self.notes]

class Database:
    """
    Database for all the papers.

    sheet : Google sheets spreadsheet with data in it
    papers : list of Paper objects in the database
    titles : set with all the titles of the papers in the database
    bibs : list of filenames for all the bibliographies of stuff

    base_url : base URL for the API request to CrossRef
    doi_url : base URL for a DOI number
    """

    def __init__(self):
    
        self.papers = []
        self.titles = dict()


    def add_from_spreadsheet(self, sheet, row, bool_inds, str_inds):
        """
        Add the possible entries from the spreadsheet
        """
        row_vals = sheet.row_values(row)

        if sheet.cell(row, bool_inds[0]).value == "TRUE":
            self.verified = 1
        if sheet.cell(row, bool_inds[1]).value == "Survey":
            self.original_work = False
        if sheet.cell(row, bool_inds[2]).value == "Yes":
            self.includes_alg = True

        str_vals = [sheet.cell(row, str_inds[i]).value for i in range(len(str_inds))]

        self.doi, self.year, self.author, self.field = str_vals[:4]
        self.title, self.prob_attrs, self.strategies, self.notes = str_vals[4:]


    def add_paper(self, citation, base_url, get_bib=True):
        """
        Add paper to the database and return it.
        """

        p = Paper()
        p.lookup(citation, base_url, get_bib=get_bib)

        # Check if the title matches what we expect from the citation
        strip_citation = citation.lower().replace('-','')
        strip_ptitle = p.title.replace(',','').replace('-','').lower()

        if strip_citation.find(strip_ptitle) > 0:
            p.verified = 1
        else:
            print("Could not verify", strip_ptitle, "from", strip_citation)

        # Check if it's a duplicate and add/return accordingly
        if p.title in self.titles:

            matches_p = self.papers[titles[p.title]]
            
            if p.doi != matches_p.doi:
                raise ValueError("WE HAVE RUN INTO A DUPLICATE TITLE:", p.title,
                 "matches DOI numbers", p.doi, "and", matches_p.doi)

            return self.papers[titles[p.title]]

        else:
            p.index = len(self.papers)
            self.titles[p.title] = p.index
            self.papers.append(p)
            return p

    def make_paper_list(self):
        """
        Convenience function to get a list of all the papers we have a properly
        formatted reference list for.
        """
        with open('allpapers.txt','w') as output:
        
            for i in range(num_papers):

                bibfile = "bibliographies\\database" + str(i+1) + ".txt"

                with open(bibfile) as input:

                    splits = re.split('\[[0-9]+\] ', input.read())
                    head = splits[1].replace('\n',' ').strip(' ')
                    p, unique = self.add_paper(head)

                    output.write(p.title+'\n')

                    for j in range(2,len(splits)):

                        ref = splits[j].replace('\n',' ').strip(' ')
                        print('(', j-1, 'of', len(splits)-2, ')', ref[:80])
                        q, unique = self.add_paper(ref, get_bib=False)
                        output.write(q.title + '\n')

    def concat_bibtxt(self):
        """
        Convenience function to smush all the reference lists together.
        """
        headers_list = []

        with open("ALL_CITATIONS.txt", 'w', errors="surrogateescape") as output:

            for i in range(len(self.bibs)):

                with open(self.bibs[i], errors="surrogateescape") as input:
                    
                    inputstr = input.read()
                    splits = re.split('\[[0-9]+\] ', inputstr)

                    output.write('\nTHIS ONE IS THE HEAD OF THE FILE '+self.bibs[i]+':\n\n')
                    output.write(splits[1])
                    headers_list.append(splits[1])
                    output.write('\nHERE ARE ALL ITS CITED THINGS:\n\n')

                    for j in range(2,len(splits)):
                        output.write(splits[j]+'\n')

        with open("ALL_HAVEBIBS.txt", 'w', errors="surrogateescape") as output:
            for i in range(len(headers_list)):
                output.write(headers_list[i])
                output.write('\n')

    # Main thing from 643 project, haven't touched since
    def import_bibliographies(self, num_papers):
        """
        Go through the .txt file bibliographies we made, add all the papers
        to the database, and make an adjacency list for all the citations
        """

        with open('bib_edges.txt','w') as output:
        
            for i in range(num_papers):

                bibfile = "bibliographies\\database" + str(i+1) + ".txt"

                with open(bibfile) as input:

                    inputstr = input.read()
                    splits = re.split('\[[0-9]+\] ', inputstr)
                    head = splits[1].replace('\n',' ').strip(' ')
                    
                    print("\nRead bibliography for paper", i+1, "of", num_papers,
                        ": ", len(splits)-2, " references")
                    print('*'*100,'\n')

                    p, unique = self.add_paper(head)

                    for j in range(2,len(splits)):

                        ref = splits[j].replace('\n',' ').strip(' ')
                        print('(', j-1, 'of', len(splits)-2, ')', ref[:80])
                        q, unique = self.add_paper(ref, get_bib=False)
                        output.write(p.title + ' <-> ' + q.title + '\n')

    # Kinda useless, it's slower than just using CrossRef
    def read_from_spreadsheet(self, sheet):
        """
        Read the spreadsheet into a database.
        """
        n = sheet.row_count
        col_names = sheet.row_values(1)

        lookup_cols = ['CrossRef Verified', 'Original?', 'DOI number', 'Year', 
                'Authors', 'Field of Study of Primary Author', 'Algorithm?', 
                'Title', 'Problem Attributes', 'Strategies', 'Notes'] 

        bool_cols = ['CrossRef Verified', 'Original?', 'Algorithm?']

        str_cols =  ['DOI number', 'Year', 'Authors', 'Field of Study of Primary Author',
                'Title', 'Problem Attributes', 'Strategies', 'Notes']

        bool_inds = [col_names.index(col) + 1 for col in bool_cols]
        str_inds = [col_names.index(col) + 1 for col in str_cols]

        num_requests = 2

        for i in range(2, n+1):

            if num_requests > 80:
                print("\nSLEEP FOR 100 SECONDS SO THE API DOESN'T GET MAD")
                for i in range(10):
                    time.sleep(10)
                    print(10*i, "seconds done")
                print()
                num_requests = 0

            if len(sheet.row_values(i)) > 0:
                p = Paper()
                p.add_from_spreadsheet(sheet, i, bool_inds, str_inds)
                print(p)
                num_requests += 12

                p.index = len(self.papers)
                self.titles[p.title] = p.index
                self.papers.append(p)

            else:
                num_requests += 1

    def read_from_bibheaders(self):
        """
        Go through the bibliography .txt files we have and add their
        corresponding papers.
        """
        for i in range(len(self.bibs)):
            with open(self.bibs[i]) as input:
                print("Add", i, "of", len(self.bibs), "from", self.bibs[i])
                splits = re.split('\[[0-9]+\] ', input.read())
                head = splits[1].replace('\n',' ').strip(' ')
                p = self.add_paper(head)
        
    def update_spreadsheet(self, sheet):
        """
        Update the entries in the spreadsheet with CrossRef entries from the
        existing database.
        """
        col_names = sheet.row_values(1)
        update_cols = ['CrossRef Verified', 'DOI number', 'Year', 'Authors']
        col_inds = [col_names.index(col) + 1 for col in update_cols]

        num_requests = 1

        for i in range(len(self.papers)):
            
            p = self.papers[i]
            
            if num_requests > 90:
                print("\nSLEEP FOR 100 SECONDS SO THE API DOESN'T GET MAD")
                for i in range(10):
                    time.sleep(10)
                    print(10*i, "seconds done")
                print()
                num_requests = 0
            
            if p.verified:
                
                # Already in the spreadsheet
                try:
                    row = sheet.find(p.title).row
                    p.in_spreadsheet = True
                    num_requests += 1
                    
                    if sheet.cell(row, col_inds[0]).value != "TRUE":
                        sheet.update_cell(row, col_inds[0], 'True')
                        num_requests += 1

                        if p.doi is not None:
                            sheet.update_cell(row, col_inds[1], 
                                            self.doi_url + p.doi)
                            num_requests += 1

                        if p.year is not None:
                            sheet.update_cell(row, col_inds[2], p.year)
                            num_requests += 1

                        if p.author is not None:
                            sheet.update_cell(row, col_inds[3], p.author)
                            num_requests += 1


                # Not already in the spreadsheet
                except:

                    # Add known values and TODOS
                    newrow = ['', 'TRUE', '', 'TODO', self.doi_url + p.doi, 
                              p.year, p.author, 'TODO', 'TODO', p.title, 'TODO', 
                              'TODO', '']
                    sheet.append_row(newrow)
                    p.in_spreadsheet = True
                    num_requests += 1
                
            # Otherwise, put the citation as the 'title' and TODO everything else
            else:
                print("NOT VERIFIED: do not add '"+p.title+"' from citation\
                notes", p.citation)

    # TODO
    def make_pandas(self):
        """
        Make a nice pandas database.
        """
        pass

    # TODO
    def read_from_file(self, filename):
        """
        Read in an existing paper database from file.
        """
        pass

    # TODO
    def append_to_file(self, filename):
        """
        Write new database entries to an existing file.
        """
        pass

    # TODO
    def to_file(self, filename):
        """
        Write the database to a file.
        """

        field_delim = "<<"
        element_delim = "@@@"
        firstline_items = ['doi','year','title','author','citation','bibtex',
            'crossref_score','in_spreadsheet','field','original_work',
            'includes_alg','prob_attrs','strategies','notes']

        """
        TODO: Include a thing to check if the filename already has stuff in it,
        and if so, go through and check. For now, the 'w' argument guarantees
        a fresh, blank file.
        """

        with open(filename,'w'):

            file.write(field_delim.join(firstline_items) + element_delim)

            for i in range(len(papers)):
                file.write(field_delim.join(paper[i].to_csv) + element_delim)

class Library:
    """
    WE HAVE TO GO DEEPER.
    """

    def __init__(self):
        self.bibs = glob.glob('./bibliographies/*.txt')
        self.base_url = "https://api.crossref.org/works?query.bibliographic="
        self.doi_url = "http://dx.doi.org/"  

        # Get the Google drive spreadsheet
        scope = ['https://spreadsheets.google.com/feeds',
                 'https://www.googleapis.com/auth/drive']
        creds = ServiceAccountCredentials.from_json_keyfile_name(
            'client_secret.json', scope)
        client = gspread.authorize(creds)

        self.sheet = client.open("Paper Database").sheet1
import smtplib
import gspread
import csv
import re
import glob
import time
import urllib
import requests
import numpy as np
from oauth2client.service_account import ServiceAccountCredentials

def check_crossref(citation):
    """
    Get a single CrossRef item manually for a given citation.
    """
    base_url = "https://api.crossref.org/works?query.bibliographic="
    url = base_url + urllib.parse.quote(citation, errors='surrogateescape')
    response = requests.get(url, timeout=100).json()
    item = response['message']['items'][0]
    return item

def set_property(attribute, item, property):
    """Generic safe dictionary lookup setter."""
    try:
        attribute = item[property]
    except KeyError:
        pass

class Paper:
    """
    Store metadata for a specific paper.

    freeform_citation: lookup info we initialize the paper with

    doi : DOI number, if it exists
    url : DOI in URL form if it exists, generic link otherwise
    year : year of publication
    title : title of paper
    author : "author1 and author2 and author3", etc. TODO: crossref format?
    reference_count : number of references
    citation_count : number of times cited
    subjects : list of subjects given by CrossRef
    verified : Are we sure that our metadata is correct?
    
    item : full CrossRef item thing
    bibtex : nice string to put in BibTeX

    type: original, survey, or application?
    includes_alg : does it include an algorithm?
    prob_attrs : list of problem attributes
    strategies : strategies used in the paper to approach the problem
    notes : any notes I made while reading
    """
    def __init__(self):

        # Initialized with 
        self.freeform_citation = None

        # Can be done either manually or automatically
        self.doi = None
        self.url = None
        self.year = None
        self.title = ""
        self.author = []
        self.reference_count = 0
        self.citation_count = 0
        self.subjects = []
        self.verified = False

        # Non-spreadsheet
        self.item = None
        self.bibtex = None
        
        # Must always be done manually
        self.type = None
        self.includes_alg = False
        self.prob_attrs = []
        self.strategies = []
        self.notes = ""   

    def lookup(self, citation, base_url):
        """
        Look up the citation in CrossRef and assign values accordingly.
        """

        # Make the CrossRef API request
        self.freeform_citation = citation 
        url = base_url + urllib.parse.quote(citation, errors='surrogateescape')
        r = requests.get(url, timeout=100).json()
        self.crossref_item = r['message']['items'][0]

        # Get the straightforward properties
        set_property(self.doi, self.crossref_item, 'DOI')
        set_property(self.url, self.crossref_item, 'URL')
        set_property(self.reference_count, self.crossref_item, 'reference_count')
        set_property(self.citation_count, self.crossref_item, 'citation_count')
        set_property(self.subjects, self.crossref_item, 'subject')
            
        # Get the title
        try:
            self.title = self.crossref_item['title'][0]#.encode('ascii','ignore').decode()
        except KeyError: pass

        # Get the year of publication
        try:
            self.year = self.crossref_item['issued']['date_parts'][0][0]
        except KeyError: pass

        # Get the author list
        try:
            authors = self.crossref_item['author']
            n = len(authors)
            alist = [authors[i]['given']+" "+authors[i]['family'] for i in range(n)]
            self.author = " and ".join(alist)
        except KeyError: pass
    
    def __str__(self):
        """
        Print out in a nice format.
        """
        out = "\nTitle: " + self.title
        out += "\n\tAuthor(s): " + str(self.author)
        out += "\n\tReference count: " + str(self.reference_count)
        out += "\n\tCitation count: " + str(self.citation_count)
        out += "\n\tDOI number: " + self.doi
        out += "\n\tYear of Publication: " + self.year
        out += "\n\tSubject(s): " + self.field
        out += "\n\tType: " + str(self.original_work)
        out += "\n\tIncludes algorithm?: " + str(self.includes_alg)
        out += "\n\tProblem attributes: " + str(self.prob_attrs)
        out += "\n\tStrategies in approach: " + str(self.strategies)
        out += "\n\tNotes: " + self.notes
        return out

    def to_csv(self):
        """
        Return a list of attributes to be joined into a row in a .csv file
        """
        return [self.doi, self.year, self.title, self.author, self.citation, 
            self.bibtex, self.crossref_score, self.verified, self.index, 
            self.in_spreadsheet, self.field, self.original_work, self.includes_alg, 
            self.prob_attrs, self.strategies, self.notes]

class Database:
    """
    Database for all the papers.

    sheet : Google sheets spreadsheet with data in it
    papers : list of Paper objects in the database
    titles : set with all the titles of the papers in the database
    bibs : list of filenames for all the bibliographies of stuff

    base_url : base URL for the API request to CrossRef
    doi_url : base URL for a DOI number
    """

    def __init__(self):
    
        self.papers = []
        self.titles = dict()


    def add_from_spreadsheet(self, sheet, row, bool_inds, str_inds):
        """
        Add the possible entries from the spreadsheet
        """
        row_vals = sheet.row_values(row)

        if sheet.cell(row, bool_inds[0]).value == "TRUE":
            self.verified = 1
        if sheet.cell(row, bool_inds[1]).value == "Survey":
            self.original_work = False
        if sheet.cell(row, bool_inds[2]).value == "Yes":
            self.includes_alg = True

        str_vals = [sheet.cell(row, str_inds[i]).value for i in range(len(str_inds))]

        self.doi, self.year, self.author, self.field = str_vals[:4]
        self.title, self.prob_attrs, self.strategies, self.notes = str_vals[4:]


    def add_paper(self, citation, base_url, get_bib=True):
        """
        Add paper to the database and return it.
        """

        p = Paper()
        p.lookup(citation, base_url, get_bib=get_bib)

        # Check if the title matches what we expect from the citation
        strip_citation = citation.lower().replace('-','')
        strip_ptitle = p.title.replace(',','').replace('-','').lower()

        if strip_citation.find(strip_ptitle) > 0:
            p.verified = 1
        else:
            print("Could not verify", strip_ptitle, "from", strip_citation)

        # Check if it's a duplicate and add/return accordingly
        if p.title in self.titles:

            matches_p = self.papers[titles[p.title]]
            
            if p.doi != matches_p.doi:
                raise ValueError("WE HAVE RUN INTO A DUPLICATE TITLE:", p.title,
                 "matches DOI numbers", p.doi, "and", matches_p.doi)

            return self.papers[titles[p.title]]

        else:
            p.index = len(self.papers)
            self.titles[p.title] = p.index
            self.papers.append(p)
            return p

    def make_paper_list(self):
        """
        Convenience function to get a list of all the papers we have a properly
        formatted reference list for.
        """
        with open('allpapers.txt','w') as output:
        
            for i in range(num_papers):

                bibfile = "bibliographies\\database" + str(i+1) + ".txt"

                with open(bibfile) as input:

                    splits = re.split('\[[0-9]+\] ', input.read())
                    head = splits[1].replace('\n',' ').strip(' ')
                    p, unique = self.add_paper(head)

                    output.write(p.title+'\n')

                    for j in range(2,len(splits)):

                        ref = splits[j].replace('\n',' ').strip(' ')
                        print('(', j-1, 'of', len(splits)-2, ')', ref[:80])
                        q, unique = self.add_paper(ref, get_bib=False)
                        output.write(q.title + '\n')

    def concat_bibtxt(self):
        """
        Convenience function to smush all the reference lists together.
        """
        headers_list = []

        with open("ALL_CITATIONS.txt", 'w', errors="surrogateescape") as output:

            for i in range(len(self.bibs)):

                with open(self.bibs[i], errors="surrogateescape") as input:
                    
                    inputstr = input.read()
                    splits = re.split('\[[0-9]+\] ', inputstr)

                    output.write('\nTHIS ONE IS THE HEAD OF THE FILE '+self.bibs[i]+':\n\n')
                    output.write(splits[1])
                    headers_list.append(splits[1])
                    output.write('\nHERE ARE ALL ITS CITED THINGS:\n\n')

                    for j in range(2,len(splits)):
                        output.write(splits[j]+'\n')

        with open("ALL_HAVEBIBS.txt", 'w', errors="surrogateescape") as output:
            for i in range(len(headers_list)):
                output.write(headers_list[i])
                output.write('\n')

    # Main thing from 643 project, haven't touched since
    def import_bibliographies(self, num_papers):
        """
        Go through the .txt file bibliographies we made, add all the papers
        to the database, and make an adjacency list for all the citations
        """

        with open('bib_edges.txt','w') as output:
        
            for i in range(num_papers):

                bibfile = "bibliographies\\database" + str(i+1) + ".txt"

                with open(bibfile) as input:

                    inputstr = input.read()
                    splits = re.split('\[[0-9]+\] ', inputstr)
                    head = splits[1].replace('\n',' ').strip(' ')
                    
                    print("\nRead bibliography for paper", i+1, "of", num_papers,
                        ": ", len(splits)-2, " references")
                    print('*'*100,'\n')

                    p, unique = self.add_paper(head)

                    for j in range(2,len(splits)):

                        ref = splits[j].replace('\n',' ').strip(' ')
                        print('(', j-1, 'of', len(splits)-2, ')', ref[:80])
                        q, unique = self.add_paper(ref, get_bib=False)
                        output.write(p.title + ' <-> ' + q.title + '\n')

    # Kinda useless, it's slower than just using CrossRef
    def read_from_spreadsheet(self, sheet):
        """
        Read the spreadsheet into a database.
        """
        n = sheet.row_count
        col_names = sheet.row_values(1)

        lookup_cols = ['CrossRef Verified', 'Original?', 'DOI number', 'Year', 
                'Authors', 'Field of Study of Primary Author', 'Algorithm?', 
                'Title', 'Problem Attributes', 'Strategies', 'Notes'] 

        bool_cols = ['CrossRef Verified', 'Original?', 'Algorithm?']

        str_cols =  ['DOI number', 'Year', 'Authors', 'Field of Study of Primary Author',
                'Title', 'Problem Attributes', 'Strategies', 'Notes']

        bool_inds = [col_names.index(col) + 1 for col in bool_cols]
        str_inds = [col_names.index(col) + 1 for col in str_cols]

        num_requests = 2

        for i in range(2, n+1):

            if num_requests > 80:
                print("\nSLEEP FOR 100 SECONDS SO THE API DOESN'T GET MAD")
                for i in range(10):
                    time.sleep(10)
                    print(10*i, "seconds done")
                print()
                num_requests = 0

            if len(sheet.row_values(i)) > 0:
                p = Paper()
                p.add_from_spreadsheet(sheet, i, bool_inds, str_inds)
                print(p)
                num_requests += 12

                p.index = len(self.papers)
                self.titles[p.title] = p.index
                self.papers.append(p)

            else:
                num_requests += 1

    def read_from_bibheaders(self):
        """
        Go through the bibliography .txt files we have and add their
        corresponding papers.
        """
        for i in range(len(self.bibs)):
            with open(self.bibs[i]) as input:
                print("Add", i, "of", len(self.bibs), "from", self.bibs[i])
                splits = re.split('\[[0-9]+\] ', input.read())
                head = splits[1].replace('\n',' ').strip(' ')
                p = self.add_paper(head)
        
    def update_spreadsheet(self, sheet):
        """
        Update the entries in the spreadsheet with CrossRef entries from the
        existing database.
        """
        col_names = sheet.row_values(1)
        update_cols = ['CrossRef Verified', 'DOI number', 'Year', 'Authors']
        col_inds = [col_names.index(col) + 1 for col in update_cols]

        num_requests = 1

        for i in range(len(self.papers)):
            
            p = self.papers[i]
            
            if num_requests > 90:
                print("\nSLEEP FOR 100 SECONDS SO THE API DOESN'T GET MAD")
                for i in range(10):
                    time.sleep(10)
                    print(10*i, "seconds done")
                print()
                num_requests = 0
            
            if p.verified:
                
                # Already in the spreadsheet
                try:
                    row = sheet.find(p.title).row
                    p.in_spreadsheet = True
                    num_requests += 1
                    
                    if sheet.cell(row, col_inds[0]).value != "TRUE":
                        sheet.update_cell(row, col_inds[0], 'True')
                        num_requests += 1

                        if p.doi is not None:
                            sheet.update_cell(row, col_inds[1], 
                                            self.doi_url + p.doi)
                            num_requests += 1

                        if p.year is not None:
                            sheet.update_cell(row, col_inds[2], p.year)
                            num_requests += 1

                        if p.author is not None:
                            sheet.update_cell(row, col_inds[3], p.author)
                            num_requests += 1


                # Not already in the spreadsheet
                except:

                    # Add known values and TODOS
                    newrow = ['', 'TRUE', '', 'TODO', self.doi_url + p.doi, 
                              p.year, p.author, 'TODO', 'TODO', p.title, 'TODO', 
                              'TODO', '']
                    sheet.append_row(newrow)
                    p.in_spreadsheet = True
                    num_requests += 1
                
            # Otherwise, put the citation as the 'title' and TODO everything else
            else:
                print("NOT VERIFIED: do not add '"+p.title+"' from citation\
                notes", p.citation)

    # TODO
    def make_pandas(self):
        """
        Make a nice pandas database.
        """
        pass

    # TODO
    def read_from_file(self, filename):
        """
        Read in an existing paper database from file.
        """
        pass

    # TODO
    def append_to_file(self, filename):
        """
        Write new database entries to an existing file.
        """
        pass

    # TODO
    def to_file(self, filename):
        """
        Write the database to a file.
        """

        field_delim = "<<"
        element_delim = "@@@"
        firstline_items = ['doi','year','title','author','citation','bibtex',
            'crossref_score','in_spreadsheet','field','original_work',
            'includes_alg','prob_attrs','strategies','notes']

        """
        TODO: Include a thing to check if the filename already has stuff in it,
        and if so, go through and check. For now, the 'w' argument guarantees
        a fresh, blank file.
        """

        with open(filename,'w'):

            file.write(field_delim.join(firstline_items) + element_delim)

            for i in range(len(papers)):
                file.write(field_delim.join(paper[i].to_csv) + element_delim)

class Library:
    """
    WE HAVE TO GO DEEPER.
    """

    def __init__(self):
        self.bibs = glob.glob('./bibliographies/*.txt')
        self.base_url = "https://api.crossref.org/works?query.bibliographic="
        self.doi_url = "http://dx.doi.org/"  

        # Get the Google drive spreadsheet
        scope = ['https://spreadsheets.google.com/feeds',
                 'https://www.googleapis.com/auth/drive']
        creds = ServiceAccountCredentials.from_json_keyfile_name(
            'client_secret.json', scope)
        client = gspread.authorize(creds)

        self.sheet = client.open("Paper Database").sheet1
Get a single CrossRef item manually for a given citation. Generic safe dictionary lookup setter. Store metadata for a specific paper.      freeform_citation: lookup info we initialize the paper with      doi : DOI number, if it exists     url : DOI in URL form if it exists, generic link otherwise     year : year of publication     title : title of paper     author : "author1 and author2 and author3", etc. TODO: crossref format?     reference_count : number of references     citation_count : number of times cited     subjects : list of subjects given by CrossRef     verified : Are we sure that our metadata is correct?          item : full CrossRef item thing     bibtex : nice string to put in BibTeX      type: original, survey, or application?     includes_alg : does it include an algorithm?     prob_attrs : list of problem attributes     strategies : strategies used in the paper to approach the problem     notes : any notes I made while reading # Initialized with # Can be done either manually or automatically # Non-spreadsheet # Must always be done manually Look up the citation in CrossRef and assign values accordingly. # Make the CrossRef API request # Get the straightforward properties # Get the title #.encode('ascii','ignore').decode() # Get the year of publication # Get the author list Print out in a nice format. Return a list of attributes to be joined into a row in a .csv file Database for all the papers.      sheet : Google sheets spreadsheet with data in it     papers : list of Paper objects in the database     titles : set with all the titles of the papers in the database     bibs : list of filenames for all the bibliographies of stuff      base_url : base URL for the API request to CrossRef     doi_url : base URL for a DOI number Add the possible entries from the spreadsheet Add paper to the database and return it. # Check if the title matches what we expect from the citation # Check if it's a duplicate and add/return accordingly Convenience function to get a list of all the papers we have a properly         formatted reference list for. Convenience function to smush all the reference lists together. # Main thing from 643 project, haven't touched since Go through the .txt file bibliographies we made, add all the papers         to the database, and make an adjacency list for all the citations # Kinda useless, it's slower than just using CrossRef Read the spreadsheet into a database. Go through the bibliography .txt files we have and add their         corresponding papers. Update the entries in the spreadsheet with CrossRef entries from the         existing database. # Already in the spreadsheet # Not already in the spreadsheet # Add known values and TODOS # Otherwise, put the citation as the 'title' and TODO everything else # TODO Make a nice pandas database. # TODO Read in an existing paper database from file. # TODO Write new database entries to an existing file. # TODO Write the database to a file. TODO: Include a thing to check if the filename already has stuff in it,         and if so, go through and check. For now, the 'w' argument guarantees         a fresh, blank file. WE HAVE TO GO DEEPER. # Get the Google drive spreadsheet