<reponame>MLGB3/mlgb.cron
# -*- coding: utf-8 -*-
"""
Create a preliminary version of output from <NAME>'s list.master.
By <NAME>
"""
##=====================================================================================

import sys
import MySQLdb

import connectToMLGB as c

##=====================================================================================

output_filename = 'set_catalogue_dates.sql'

newline = '\n'
full_stop = '.'
comma = ','
semi_colon = ';'
question_mark = '?'
open_bracket = '('
close_bracket = ')'
dash = '-'
single_quote = "'"

months = {
  'January': 1,
  'Jan.': 1,
  'Jan': 1,
  'February': 2,
  'Feb.': 2,
  'Feb': 2,
  'March': 3,
  'April': 4,
  'May': 5,
  'June': 6,
  'July': 7,
  'August': 8,
  'September': 9,
  'Sept.': 9,
  'Sept': 9,
  'October': 10,
  'Oct.': 10,
  'Oct': 10,
  'November': 11,
  'Nov.': 11,
  'Nov': 11,
  'December': 12,
  'Dec.': 12,
  'Dec': 12,
}

##=====================================================================================

def process_all_documents(): #{

  the_database_connection = None
  the_cursor = None

  try:
    outfile_handle = file
    outfile_handle = open( output_filename, 'wb' ) # 'wb' allows entry of UTF-8
    outfile_handle.write( newline + '-- This script was generated by dates.py ' + newline + newline )

    the_database_connection = c.get_database_connection()
    the_cursor = the_database_connection.cursor() 

    statement = "select document_id, document_code, document_name " 
    statement += " from index_medieval_documents where document_code > ''"
    statement += " order by document_code_sort"

    the_cursor.execute( statement )
    results = the_cursor.fetchall()

    for row in results: #{
      output = process_document( row )
      outfile_handle.write( output.encode( 'utf-8' ) )
      outfile_handle.write( newline )
    #}

    outfile_handle.close()

    the_cursor.close()
    the_database_connection.close()

  except:
    if isinstance( outfile_handle, file ):
      if not outfile_handle.closed : outfile_handle.close()
    if the_cursor: the_cursor.close()
    if the_database_connection: the_database_connection.close()
    raise
#}

##=====================================================================================
def get_month_from_word( word ): #{

  month = 0
  if months.has_key( word ): month = months[ word ]
  return month
#}
##=====================================================================================
def get_day_from_word( word ): #{

  day = 0
  if word.isdigit(): #{
    if int( word ) > 0 and int( word ) <= 31 : #{
      day = int( word )
    #}
  #}
  return day
#}
##=====================================================================================
def get_last_day_of_month( month ): #{

  day = 31
  if month == 2:
    day = 28
  elif month in [ 9, 4, 6, 11 ]: # 30 days hath September, April, June and November
    day = 30
  return day
#}
##=====================================================================================


def process_document( row ): #{

    document_id   = row[ 0 ]
    document_code = row[ 1 ].strip()
    document_name = row[ 2 ].strip()

    print document_code, document_name
    if len( document_code ) == 0: return ''
    if len( document_name ) == 0: return ''

    # We don't need all the little nuances like 'circa'.
    # We're just trying to extract rough dates for sorting
    document_name = document_name.replace( '--c.', ' ' )
    document_name = document_name.replace( ' c.', ' ' )
    document_name = document_name.replace( '{x}', ' ' )
    document_name = document_name.replace( '/', '-' )  # treat 'or' same as ranges
    document_name = document_name.replace( '--', '-' ) # double dash is same as dash

    # punctuation too can go
    document_name = document_name.replace( question_mark, '' )
    document_name = document_name.replace( full_stop, '' )
    document_name = document_name.replace( comma, '' )
    document_name = document_name.replace( semi_colon, '' )
    document_name = document_name.replace( open_bracket, '' )
    document_name = document_name.replace( close_bracket, '' )
    document_name = document_name.replace( single_quote, ' ' )

    # check for little oddities that will throw us off track
    if 'late 1th or early 16th' in document_name: # this must be a typo
      document_name = document_name.replace( 'late 1th or early 16th', 'late 15th or early 16th' )
    if 'mid-1' in document_name:
      document_name = document_name.replace( 'mid-1', 'mid 1' )
    if ' d 984' in document_name:
      document_name = document_name.replace( ' d 984', ' d 0984' )
    if '6th Lord' in document_name:
      document_name = document_name.replace( '6th Lord', 'sixth Lord' )
    if "1339 16th cent" in document_name:
      document_name = document_name.replace( "1339 16th cent", "1339" )
    
    output = ''

    start_date = ''  # for ordering - will include month and day if known
    end_date = ''

    date_in_words = '' # for display

    start_year = '' # for querying
    end_year = ''

    words = document_name.split()
    num_words = len( words )
    last_word = num_words - 1

    for i in range( last_word, -1, -1 ): #{  # go through in reverse order
      word = words[ i ]

      # Word is a simple 4-digit year, e.g. 1486
      if word.isdigit() and len(word) == 4: #{ 
        year = word
        date_string = year
        month = 0
        day = 0
  
        # Month, or month and day, given as well as year
        if i > 0: #{
          earlier_word = words[ i - 1 ]
          month = get_month_from_word( earlier_word )
        #}

        if month > 0 and i > 1: #{
          earlier_word = words[ i - 2 ]
          day = get_day_from_word( earlier_word )
        #}

        if not month: month = 12
        date_string += '-%02d' % month

        if not day: day = get_last_day_of_month( month )
        date_string += '-%02d' % day
        
        if not end_date: #{
          end_date = date_string
          end_year = year
        #}
        elif not start_date: #{
          start_date = date_string
          start_year = year
        #}
      #}

      # Word refers to a century e.g. 14th century
      elif word.endswith( 'th' ) : #{
        without_th = word.replace( 'th', '', 1 )
        if without_th.isdigit(): #{
          date_in_words = "%s century" % word

          century = int( without_th )
          century = century * 100 # 14th to 1400
          century = century - 100 # 1400 to 1300

          if i > 0: #{
            if words[ i - 1 ] in [ 'late', 'mid', 'early' ]: 
              date_in_words = "%s %s" % (words[ i - 1 ], date_in_words)

            if words[ i - 1 ] == 'late': #{
              start_year = century + 75
              end_year = century + 99
              century += 99
            #}
            elif words[ i - 1 ] == 'mid':  #{
              start_year = century + 25
              end_year = century + 75
              century += 50
            #}
            elif words[ i - 1 ] == 'early':  #{
              start_year = century 
              end_year = century + 25
            #}
            elif words[ i - 1 ] != 'early': #{
              start_year = century 
              end_year = century + 99
              century += 98 # sort vaguer dates to the end
            #}
          #}
          else: #{
            start_year = century 
            end_year = century + 99
            century += 99 # sort vaguer dates to the end
          #}

          date_string = str( century ) + '-12-31'
          if not end_date: 
            end_date = date_string
          elif not start_date: 
            start_date = date_string
        #}
      #}

      # Word is a range of years e.g. 1336-40
      elif dash in word: #{
        without_dash = word.replace( dash, '', 1 )
        if without_dash.isdigit(): #{
          range_parts = word.split( dash )
          range_start = range_parts[ 0 ] # e.g. 1336
          range_end   = range_parts[ 1 ] # e.g. 40
          if len( range_start ) == 4 and len( range_end ) < 4:  #{  # convert, e.g., 40 to 1340
            first_chars = range_start[ 0 : 0 - len( range_end ) ]
            range_end = first_chars + range_end
          #}

          start_date = range_start + '-12-31'
          end_date = range_end + '-12-31'

          date_in_words = '%s to %s' % (range_start, range_end)
          start_year = range_start
          end_year = range_end
        #}
      #}

      if start_date and end_date: break
    #} 

    if end_date and not start_date: #{
      start_date = end_date
    #}
    if end_year and not start_year: #{
      start_year = end_year
    #}

    # Manually correct records with too complex a name to extract the date properly
    if 'late 15th or early 16th cent' in document_name: #{
       date_in_words = 'late 15th or early 16th century'
       end_year = '1525'
       end_date = '1525-12-31'
    #}
    elif 'late 14th or early 15th cent' in document_name: #{
       date_in_words = 'late 14th or early 15th century'
       end_year = '1425'
       end_date = '1425-12-31'
    #}
    elif 'late 12th to early 13th cent' in document_name: #{
       date_in_words = 'late 12th to early 13th century'
       end_year = '1225'
       end_date = '1225-12-31'
    #}
    elif '1548-9 to 1558-9' in document_name: #{
       date_in_words = '1548-9 to 1558-9'
       start_year = '1548'
       start_date = '1548-12-31'
    #}
    elif '1443-4 to 1498-9' in document_name: #{
       date_in_words = '1443-4 to 1498-9'
       start_year = '1443'
       start_date = '1443-12-31'
    #}
    elif '1528-9 to 1555-6' in document_name: #{
       date_in_words = '1528-9 to 1555-6'
       start_year = '1528'
       start_date = '1528-12-31'
    #}
    elif '1363-4 to 1383-4' in document_name: #{
       date_in_words = '1363-4 to 1383-4'
       start_year = '1363'
       start_date = '1363-12-31'
    #}
    elif '1385-6 to 1408-9' in document_name: #{
       date_in_words = '1385-6 to 1408-9'
       start_year = '1385'
       start_date = '1385-12-31'
    #}
    elif '1413-14 to 1428-9' in document_name: #{
       date_in_words = '1413-14 to 1428-9'
       start_year = '1413'
       start_date = '1413-12-31'
    #}
    elif '1430-31 to 1492-3' in document_name: #{
       date_in_words = '1430-31 to 1492-3'
       start_year = '1430'
       start_date = '1430-12-31'
    #}
    elif '1522-3 to 1546-7' in document_name: #{
       date_in_words = '1522-3 to 1546-7'
       start_year = '1522'
       start_date = '1522-12-31'
    #}
    elif '1553-4 to 1557-8' in document_name: #{
       date_in_words = '1553-4 to 1557-8'
       start_year = '1553'
       start_date = '1553-12-31'
    #}

    if not end_date:
      date_in_words = 'undated'
      print 'NO DATES FOUND FOR', document_code, document_name
    elif start_date == end_date:
      print document_code, document_name, ' -- single date: ', end_date
    else:
      print document_code, document_name, ' -- start/end: ', start_date, end_date

    if not date_in_words: #{
      if start_date == end_date:
        date_in_words = start_year
      else:
        date_in_words = '%s to %s' % (start_year, end_year)
    #}

    print 'Date in words:', date_in_words
    print 'Range covered: ', start_year, end_year
    print ""

    if start_date and end_date: #{
      output = "update index_medieval_documents "
      output += " set start_date = '%s', end_date = '%s', " % (start_date, end_date)
      output += " start_year = %s, end_year = %s, " % (start_year, end_year)
      output += " date_in_words = '%s' " % date_in_words
      output += " where document_code = '%s' and document_id = %d;%s" \
             %  (document_code, document_id, newline)
    #}
    else: #{
      output = "update index_medieval_documents "
      output += " set date_in_words = '%s' " % date_in_words
      output += " where document_code = '%s' and document_id = %d;%s" \
             %  (document_code, document_id, newline)
    #}

    return output
#}

##=====================================================================================

if __name__ == '__main__':


  # These two lines are hacks (copied from Mat's clever hack, thanks Mat). 
  # They switch the default encoding to utf8 so that the command line will convert UTF8 + Ascii to UTF8
  reload(sys)
  sys.setdefaultencoding("utf8")

  process_all_documents()

##=====================================================================================

# -*- coding: utf-8 -*-
"""
Create a preliminary version of output from <NAME>'s list.master.
By <NAME>
"""
##=====================================================================================

import sys
import MySQLdb

import connectToMLGB as c

##=====================================================================================

output_filename = 'set_catalogue_dates.sql'

newline = '\n'
full_stop = '.'
comma = ','
semi_colon = ';'
question_mark = '?'
open_bracket = '('
close_bracket = ')'
dash = '-'
single_quote = "'"

months = {
  'January': 1,
  'Jan.': 1,
  'Jan': 1,
  'February': 2,
  'Feb.': 2,
  'Feb': 2,
  'March': 3,
  'April': 4,
  'May': 5,
  'June': 6,
  'July': 7,
  'August': 8,
  'September': 9,
  'Sept.': 9,
  'Sept': 9,
  'October': 10,
  'Oct.': 10,
  'Oct': 10,
  'November': 11,
  'Nov.': 11,
  'Nov': 11,
  'December': 12,
  'Dec.': 12,
  'Dec': 12,
}

##=====================================================================================

def process_all_documents(): #{

  the_database_connection = None
  the_cursor = None

  try:
    outfile_handle = file
    outfile_handle = open( output_filename, 'wb' ) # 'wb' allows entry of UTF-8
    outfile_handle.write( newline + '-- This script was generated by dates.py ' + newline + newline )

    the_database_connection = c.get_database_connection()
    the_cursor = the_database_connection.cursor() 

    statement = "select document_id, document_code, document_name " 
    statement += " from index_medieval_documents where document_code > ''"
    statement += " order by document_code_sort"

    the_cursor.execute( statement )
    results = the_cursor.fetchall()

    for row in results: #{
      output = process_document( row )
      outfile_handle.write( output.encode( 'utf-8' ) )
      outfile_handle.write( newline )
    #}

    outfile_handle.close()

    the_cursor.close()
    the_database_connection.close()

  except:
    if isinstance( outfile_handle, file ):
      if not outfile_handle.closed : outfile_handle.close()
    if the_cursor: the_cursor.close()
    if the_database_connection: the_database_connection.close()
    raise
#}

##=====================================================================================
def get_month_from_word( word ): #{

  month = 0
  if months.has_key( word ): month = months[ word ]
  return month
#}
##=====================================================================================
def get_day_from_word( word ): #{

  day = 0
  if word.isdigit(): #{
    if int( word ) > 0 and int( word ) <= 31 : #{
      day = int( word )
    #}
  #}
  return day
#}
##=====================================================================================
def get_last_day_of_month( month ): #{

  day = 31
  if month == 2:
    day = 28
  elif month in [ 9, 4, 6, 11 ]: # 30 days hath September, April, June and November
    day = 30
  return day
#}
##=====================================================================================


def process_document( row ): #{

    document_id   = row[ 0 ]
    document_code = row[ 1 ].strip()
    document_name = row[ 2 ].strip()

    print document_code, document_name
    if len( document_code ) == 0: return ''
    if len( document_name ) == 0: return ''

    # We don't need all the little nuances like 'circa'.
    # We're just trying to extract rough dates for sorting
    document_name = document_name.replace( '--c.', ' ' )
    document_name = document_name.replace( ' c.', ' ' )
    document_name = document_name.replace( '{x}', ' ' )
    document_name = document_name.replace( '/', '-' )  # treat 'or' same as ranges
    document_name = document_name.replace( '--', '-' ) # double dash is same as dash

    # punctuation too can go
    document_name = document_name.replace( question_mark, '' )
    document_name = document_name.replace( full_stop, '' )
    document_name = document_name.replace( comma, '' )
    document_name = document_name.replace( semi_colon, '' )
    document_name = document_name.replace( open_bracket, '' )
    document_name = document_name.replace( close_bracket, '' )
    document_name = document_name.replace( single_quote, ' ' )

    # check for little oddities that will throw us off track
    if 'late 1th or early 16th' in document_name: # this must be a typo
      document_name = document_name.replace( 'late 1th or early 16th', 'late 15th or early 16th' )
    if 'mid-1' in document_name:
      document_name = document_name.replace( 'mid-1', 'mid 1' )
    if ' d 984' in document_name:
      document_name = document_name.replace( ' d 984', ' d 0984' )
    if '6th Lord' in document_name:
      document_name = document_name.replace( '6th Lord', 'sixth Lord' )
    if "1339 16th cent" in document_name:
      document_name = document_name.replace( "1339 16th cent", "1339" )
    
    output = ''

    start_date = ''  # for ordering - will include month and day if known
    end_date = ''

    date_in_words = '' # for display

    start_year = '' # for querying
    end_year = ''

    words = document_name.split()
    num_words = len( words )
    last_word = num_words - 1

    for i in range( last_word, -1, -1 ): #{  # go through in reverse order
      word = words[ i ]

      # Word is a simple 4-digit year, e.g. 1486
      if word.isdigit() and len(word) == 4: #{ 
        year = word
        date_string = year
        month = 0
        day = 0
  
        # Month, or month and day, given as well as year
        if i > 0: #{
          earlier_word = words[ i - 1 ]
          month = get_month_from_word( earlier_word )
        #}

        if month > 0 and i > 1: #{
          earlier_word = words[ i - 2 ]
          day = get_day_from_word( earlier_word )
        #}

        if not month: month = 12
        date_string += '-%02d' % month

        if not day: day = get_last_day_of_month( month )
        date_string += '-%02d' % day
        
        if not end_date: #{
          end_date = date_string
          end_year = year
        #}
        elif not start_date: #{
          start_date = date_string
          start_year = year
        #}
      #}

      # Word refers to a century e.g. 14th century
      elif word.endswith( 'th' ) : #{
        without_th = word.replace( 'th', '', 1 )
        if without_th.isdigit(): #{
          date_in_words = "%s century" % word

          century = int( without_th )
          century = century * 100 # 14th to 1400
          century = century - 100 # 1400 to 1300

          if i > 0: #{
            if words[ i - 1 ] in [ 'late', 'mid', 'early' ]: 
              date_in_words = "%s %s" % (words[ i - 1 ], date_in_words)

            if words[ i - 1 ] == 'late': #{
              start_year = century + 75
              end_year = century + 99
              century += 99
            #}
            elif words[ i - 1 ] == 'mid':  #{
              start_year = century + 25
              end_year = century + 75
              century += 50
            #}
            elif words[ i - 1 ] == 'early':  #{
              start_year = century 
              end_year = century + 25
            #}
            elif words[ i - 1 ] != 'early': #{
              start_year = century 
              end_year = century + 99
              century += 98 # sort vaguer dates to the end
            #}
          #}
          else: #{
            start_year = century 
            end_year = century + 99
            century += 99 # sort vaguer dates to the end
          #}

          date_string = str( century ) + '-12-31'
          if not end_date: 
            end_date = date_string
          elif not start_date: 
            start_date = date_string
        #}
      #}

      # Word is a range of years e.g. 1336-40
      elif dash in word: #{
        without_dash = word.replace( dash, '', 1 )
        if without_dash.isdigit(): #{
          range_parts = word.split( dash )
          range_start = range_parts[ 0 ] # e.g. 1336
          range_end   = range_parts[ 1 ] # e.g. 40
          if len( range_start ) == 4 and len( range_end ) < 4:  #{  # convert, e.g., 40 to 1340
            first_chars = range_start[ 0 : 0 - len( range_end ) ]
            range_end = first_chars + range_end
          #}

          start_date = range_start + '-12-31'
          end_date = range_end + '-12-31'

          date_in_words = '%s to %s' % (range_start, range_end)
          start_year = range_start
          end_year = range_end
        #}
      #}

      if start_date and end_date: break
    #} 

    if end_date and not start_date: #{
      start_date = end_date
    #}
    if end_year and not start_year: #{
      start_year = end_year
    #}

    # Manually correct records with too complex a name to extract the date properly
    if 'late 15th or early 16th cent' in document_name: #{
       date_in_words = 'late 15th or early 16th century'
       end_year = '1525'
       end_date = '1525-12-31'
    #}
    elif 'late 14th or early 15th cent' in document_name: #{
       date_in_words = 'late 14th or early 15th century'
       end_year = '1425'
       end_date = '1425-12-31'
    #}
    elif 'late 12th to early 13th cent' in document_name: #{
       date_in_words = 'late 12th to early 13th century'
       end_year = '1225'
       end_date = '1225-12-31'
    #}
    elif '1548-9 to 1558-9' in document_name: #{
       date_in_words = '1548-9 to 1558-9'
       start_year = '1548'
       start_date = '1548-12-31'
    #}
    elif '1443-4 to 1498-9' in document_name: #{
       date_in_words = '1443-4 to 1498-9'
       start_year = '1443'
       start_date = '1443-12-31'
    #}
    elif '1528-9 to 1555-6' in document_name: #{
       date_in_words = '1528-9 to 1555-6'
       start_year = '1528'
       start_date = '1528-12-31'
    #}
    elif '1363-4 to 1383-4' in document_name: #{
       date_in_words = '1363-4 to 1383-4'
       start_year = '1363'
       start_date = '1363-12-31'
    #}
    elif '1385-6 to 1408-9' in document_name: #{
       date_in_words = '1385-6 to 1408-9'
       start_year = '1385'
       start_date = '1385-12-31'
    #}
    elif '1413-14 to 1428-9' in document_name: #{
       date_in_words = '1413-14 to 1428-9'
       start_year = '1413'
       start_date = '1413-12-31'
    #}
    elif '1430-31 to 1492-3' in document_name: #{
       date_in_words = '1430-31 to 1492-3'
       start_year = '1430'
       start_date = '1430-12-31'
    #}
    elif '1522-3 to 1546-7' in document_name: #{
       date_in_words = '1522-3 to 1546-7'
       start_year = '1522'
       start_date = '1522-12-31'
    #}
    elif '1553-4 to 1557-8' in document_name: #{
       date_in_words = '1553-4 to 1557-8'
       start_year = '1553'
       start_date = '1553-12-31'
    #}

    if not end_date:
      date_in_words = 'undated'
      print 'NO DATES FOUND FOR', document_code, document_name
    elif start_date == end_date:
      print document_code, document_name, ' -- single date: ', end_date
    else:
      print document_code, document_name, ' -- start/end: ', start_date, end_date

    if not date_in_words: #{
      if start_date == end_date:
        date_in_words = start_year
      else:
        date_in_words = '%s to %s' % (start_year, end_year)
    #}

    print 'Date in words:', date_in_words
    print 'Range covered: ', start_year, end_year
    print ""

    if start_date and end_date: #{
      output = "update index_medieval_documents "
      output += " set start_date = '%s', end_date = '%s', " % (start_date, end_date)
      output += " start_year = %s, end_year = %s, " % (start_year, end_year)
      output += " date_in_words = '%s' " % date_in_words
      output += " where document_code = '%s' and document_id = %d;%s" \
             %  (document_code, document_id, newline)
    #}
    else: #{
      output = "update index_medieval_documents "
      output += " set date_in_words = '%s' " % date_in_words
      output += " where document_code = '%s' and document_id = %d;%s" \
             %  (document_code, document_id, newline)
    #}

    return output
#}

##=====================================================================================

if __name__ == '__main__':


  # These two lines are hacks (copied from Mat's clever hack, thanks Mat). 
  # They switch the default encoding to utf8 so that the command line will convert UTF8 + Ascii to UTF8
  reload(sys)
  sys.setdefaultencoding("utf8")

  process_all_documents()

##=====================================================================================
# -*- coding: utf-8 -*- Create a preliminary version of output from <NAME>'s list.master. By <NAME> ##===================================================================================== ##===================================================================================== ##===================================================================================== #{ # 'wb' allows entry of UTF-8 #{ #} #} ##===================================================================================== #{ #} ##===================================================================================== #{ #{ #{ #} #} #} ##===================================================================================== #{ # 30 days hath September, April, June and November #} ##===================================================================================== #{ # We don't need all the little nuances like 'circa'. # We're just trying to extract rough dates for sorting # treat 'or' same as ranges # double dash is same as dash # punctuation too can go # check for little oddities that will throw us off track # this must be a typo # for ordering - will include month and day if known # for display # for querying #{  # go through in reverse order # Word is a simple 4-digit year, e.g. 1486 #{ # Month, or month and day, given as well as year #{ #} #{ #} #{ #} #{ #} #} # Word refers to a century e.g. 14th century #{ #{ # 14th to 1400 # 1400 to 1300 #{ #{ #} #{ #} #{ #} #{ # sort vaguer dates to the end #} #} #{ # sort vaguer dates to the end #} #} #} # Word is a range of years e.g. 1336-40 #{ #{ # e.g. 1336 # e.g. 40 #{  # convert, e.g., 40 to 1340 #} #} #} #} #{ #} #{ #} # Manually correct records with too complex a name to extract the date properly #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #{ #} #} ##===================================================================================== # These two lines are hacks (copied from Mat's clever hack, thanks Mat). # They switch the default encoding to utf8 so that the command line will convert UTF8 + Ascii to UTF8 ##=====================================================================================