<s> import os <EOL> import os . path <EOL> import re <EOL> import json <EOL> import time <EOL> import hashlib <EOL> import logging <EOL> import tarfile <EOL> import textwrap <EOL> import urlparse <EOL> from cStringIO import StringIO <EOL> from rfc3339 import rfc3339 <EOL> from lxml import etree <EOL> from urllib import url2pathname <EOL> from django . db import models <EOL> from django . db . models import permalink , Q <EOL> from django . conf import settings <EOL> from chronam . core . utils import strftime <EOL> from django . core import urlresolvers <EOL> class Awardee ( models . Model ) : <EOL> org_code = models . CharField ( max_length = <NUM_LIT:50> , primary_key = True ) <EOL> name = models . CharField ( max_length = <NUM_LIT:100> ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> @ property <EOL> def batch_count ( self ) : <EOL> return Batch . objects . filter ( awardee__org_code = self . org_code ) . count ( ) <EOL> @ property <EOL> def page_count ( self ) : <EOL> return Page . objects . filter ( issue__batch__awardee__org_code = self . org_code ) . count ( ) <EOL> @ property <EOL> @ permalink <EOL> def url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . org_code } ) <EOL> @ property <EOL> @ permalink <EOL> def json_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . org_code } ) <EOL> @ property <EOL> def abstract_url ( self ) : <EOL> return self . url . rstrip ( '<STR_LIT:/>' ) + '<STR_LIT>' <EOL> def json ( self , host = "<STR_LIT>" , serialize = True ) : <EOL> j = { <EOL> "<STR_LIT:name>" : self . name , <EOL> "<STR_LIT:url>" : '<STR_LIT>' + host + self . json_url <EOL> } <EOL> if serialize : <EOL> return json . dumps ( j , indent = <NUM_LIT:2> ) <EOL> return j <EOL> def __unicode__ ( self ) : <EOL> return self . name <EOL> class Batch ( models . Model ) : <EOL> name = models . CharField ( max_length = <NUM_LIT> , primary_key = True ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> validated_batch_file = models . CharField ( max_length = <NUM_LIT:100> ) <EOL> awardee = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' , null = True ) <EOL> released = models . DateTimeField ( null = True ) <EOL> source = models . CharField ( max_length = <NUM_LIT> , null = True ) <EOL> sitemap_indexed = models . DateTimeField ( auto_now_add = False , null = True ) <EOL> @ classmethod <EOL> def viewable_batches ( klass ) : <EOL> if settings . IS_PRODUCTION : <EOL> batches = Batch . objects . filter ( released__isnull = False ) <EOL> else : <EOL> batches = Batch . objects . all ( ) <EOL> return batches . order_by ( "<STR_LIT>" ) <EOL> @ property <EOL> def storage_url ( self ) : <EOL> """<STR_LIT>""" <EOL> source = self . source <EOL> if source : <EOL> u = urlparse . urljoin ( source , "<STR_LIT>" ) <EOL> else : <EOL> u = urlparse . urljoin ( "<STR_LIT>" , self . path ) <EOL> return u <EOL> @ property <EOL> def path ( self ) : <EOL> """<STR_LIT>""" <EOL> return os . path . join ( settings . BATCH_STORAGE , self . name , "<STR_LIT>" ) <EOL> @ property <EOL> def full_name ( self ) : <EOL> if self . awardee is None : <EOL> return "<STR_LIT:%s>" % self . name <EOL> else : <EOL> return "<STR_LIT>" % ( self . name , self . awardee . name ) <EOL> @ property <EOL> def validated_batch_url ( self ) : <EOL> return urlparse . urljoin ( self . storage_url , self . validated_batch_file ) <EOL> @ property <EOL> def page_count ( self ) : <EOL> return Page . objects . filter ( issue__batch__name = self . name ) . count ( ) <EOL> @ property <EOL> @ permalink <EOL> def url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . name } ) <EOL> @ property <EOL> @ permalink <EOL> def json_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . name } ) <EOL> @ property <EOL> def abstract_url ( self ) : <EOL> return self . url . rstrip ( '<STR_LIT:/>' ) + '<STR_LIT>' <EOL> def lccns ( self ) : <EOL> l = { } <EOL> for issue in self . issues . all ( ) : <EOL> l [ issue . title_id ] = <NUM_LIT:1> <EOL> return l . keys ( ) <EOL> def delete ( self , * args , ** kwargs ) : <EOL> try : <EOL> self . ocr_dump . delete ( ) <EOL> except OcrDump . DoesNotExist : <EOL> logging . warn ( "<STR_LIT>" , self ) <EOL> super ( Batch , self ) . delete ( * args , ** kwargs ) <EOL> def json ( self , include_issues = True , serialize = True , host = "<STR_LIT>" ) : <EOL> b = { } <EOL> b [ '<STR_LIT:name>' ] = self . name <EOL> b [ '<STR_LIT>' ] = rfc3339 ( self . created ) <EOL> b [ '<STR_LIT>' ] = self . page_count <EOL> b [ '<STR_LIT>' ] = self . lccns ( ) <EOL> b [ '<STR_LIT>' ] = { <EOL> "<STR_LIT:name>" : self . awardee . name , <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + self . awardee . json_url <EOL> } <EOL> b [ '<STR_LIT:url>' ] = "<STR_LIT>" + host + self . json_url <EOL> if include_issues : <EOL> b [ '<STR_LIT>' ] = [ ] <EOL> for issue in self . issues . all ( ) : <EOL> i = { <EOL> "<STR_LIT:title>" : { <EOL> "<STR_LIT:name>" : issue . title . display_name , <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + issue . title . json_url , <EOL> } , <EOL> "<STR_LIT>" : strftime ( issue . date_issued , "<STR_LIT>" ) , <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + issue . json_url <EOL> } <EOL> b [ '<STR_LIT>' ] . append ( i ) <EOL> if serialize : <EOL> return json . dumps ( b ) <EOL> else : <EOL> return b <EOL> def __unicode__ ( self ) : <EOL> return self . full_name <EOL> class LoadBatchEvent ( models . Model ) : <EOL> batch_name = models . CharField ( max_length = <NUM_LIT> ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> message = models . TextField ( null = True ) <EOL> def get_batch ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> return Batch . objects . get ( name = self . batch_name ) <EOL> except Title . DoesNotExist : <EOL> return None <EOL> def __unicode__ ( self ) : <EOL> return self . batch_name <EOL> class Title ( models . Model ) : <EOL> lccn = models . CharField ( primary_key = True , max_length = <NUM_LIT> ) <EOL> lccn_orig = models . CharField ( max_length = <NUM_LIT> ) <EOL> name = models . CharField ( max_length = <NUM_LIT> ) <EOL> name_normal = models . CharField ( max_length = <NUM_LIT> ) <EOL> edition = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> place_of_publication = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> publisher = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> frequency = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> frequency_date = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> medium = models . CharField ( null = True , max_length = <NUM_LIT:50> , help_text = "<STR_LIT>" ) <EOL> oclc = models . CharField ( null = True , max_length = <NUM_LIT> , db_index = True ) <EOL> issn = models . CharField ( null = True , max_length = <NUM_LIT:15> ) <EOL> start_year = models . CharField ( max_length = <NUM_LIT:10> ) <EOL> end_year = models . CharField ( max_length = <NUM_LIT:10> ) <EOL> country = models . ForeignKey ( '<STR_LIT>' ) <EOL> version = models . DateTimeField ( ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> has_issues = models . BooleanField ( default = False , db_index = True ) <EOL> uri = models . URLField ( null = True , max_length = <NUM_LIT> , help_text = "<STR_LIT>" ) <EOL> sitemap_indexed = models . DateTimeField ( auto_now_add = False , null = True ) <EOL> @ property <EOL> @ permalink <EOL> def url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . lccn } ) <EOL> @ property <EOL> @ permalink <EOL> def json_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . lccn } ) <EOL> @ property <EOL> def abstract_url ( self ) : <EOL> return self . url . rstrip ( '<STR_LIT:/>' ) + '<STR_LIT>' <EOL> @ property <EOL> def display_name ( self ) : <EOL> if self . medium : <EOL> return '<STR_LIT:U+0020>' . join ( [ self . name , self . medium ] ) <EOL> else : <EOL> return self . name <EOL> @ property <EOL> def first_issue ( self ) : <EOL> try : <EOL> return self . issues . order_by ( "<STR_LIT>" ) [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> return None <EOL> def has_essays ( self ) : <EOL> return self . essays . count ( ) > <NUM_LIT:0> <EOL> @ property <EOL> def first_essay ( self ) : <EOL> try : <EOL> return self . essays . all ( ) [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> return None <EOL> @ property <EOL> def last_issue ( self ) : <EOL> try : <EOL> return self . issues . order_by ( "<STR_LIT>" ) [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> return None <EOL> @ property <EOL> def last_issue_released ( self ) : <EOL> try : <EOL> return self . issues . order_by ( "<STR_LIT>" ) [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> return None <EOL> @ property <EOL> def holding_types ( self ) : <EOL> ht = [ h . type for h in self . holdings . all ( ) ] <EOL> if self . uri and not '<STR_LIT>' in ht : <EOL> ht . append ( '<STR_LIT>' ) <EOL> return ht <EOL> @ property <EOL> def solr_doc ( self ) : <EOL> doc = { <EOL> '<STR_LIT:id>' : self . url , <EOL> '<STR_LIT:type>' : '<STR_LIT:title>' , <EOL> '<STR_LIT:title>' : self . display_name , <EOL> '<STR_LIT>' : self . name_normal , <EOL> '<STR_LIT>' : self . lccn , <EOL> '<STR_LIT>' : self . edition , <EOL> '<STR_LIT>' : self . place_of_publication , <EOL> '<STR_LIT>' : self . frequency , <EOL> '<STR_LIT>' : self . publisher , <EOL> '<STR_LIT>' : self . start_year_int , <EOL> '<STR_LIT>' : self . end_year_int , <EOL> '<STR_LIT>' : [ l . name for l in self . languages . all ( ) ] , <EOL> '<STR_LIT>' : [ t . name for t in self . alt_titles . all ( ) ] , <EOL> '<STR_LIT>' : [ s . heading for s in self . subjects . all ( ) ] , <EOL> '<STR_LIT>' : [ n . text for n in self . notes . all ( ) ] , <EOL> '<STR_LIT>' : [ p . city for p in self . places . all ( ) ] , <EOL> '<STR_LIT>' : [ p . county for p in self . places . all ( ) ] , <EOL> '<STR_LIT>' : self . country . name , <EOL> '<STR_LIT:state>' : [ p . state for p in self . places . all ( ) ] , <EOL> '<STR_LIT>' : [ p . name for p in self . places . all ( ) ] , <EOL> '<STR_LIT>' : self . holding_types , <EOL> '<STR_LIT:url>' : [ u . value for u in self . urls . all ( ) ] , <EOL> '<STR_LIT>' : [ e . html for e in self . essays . all ( ) ] , <EOL> } <EOL> return doc <EOL> def json ( self , serialize = True , host = "<STR_LIT>" ) : <EOL> j = { <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + self . json_url , <EOL> "<STR_LIT>" : self . lccn , <EOL> "<STR_LIT:name>" : self . display_name , <EOL> "<STR_LIT>" : self . place_of_publication , <EOL> "<STR_LIT>" : self . publisher , <EOL> "<STR_LIT>" : self . start_year , <EOL> "<STR_LIT>" : self . end_year , <EOL> "<STR_LIT>" : [ s . heading for s in self . subjects . all ( ) ] , <EOL> "<STR_LIT>" : [ p . name for p in self . places . all ( ) ] , <EOL> "<STR_LIT>" : [ { <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + i . json_url , <EOL> "<STR_LIT>" : strftime ( i . date_issued , "<STR_LIT>" ) <EOL> } for i in self . issues . all ( ) ] <EOL> } <EOL> if serialize : <EOL> return json . dumps ( j , indent = <NUM_LIT:2> ) <EOL> return j <EOL> def has_non_english_language ( self ) : <EOL> for language in self . languages . all ( ) : <EOL> if language . code != '<STR_LIT>' : <EOL> return True <EOL> return False <EOL> def preceeding_titles ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _lookup_title_links ( self . preceeding_title_links . all ( ) ) <EOL> def succeeding_titles ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _lookup_title_links ( self . succeeding_title_links . all ( ) ) <EOL> def related_titles ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . _lookup_title_links ( self . related_title_links . all ( ) ) <EOL> @ property <EOL> def start_year_int ( self ) : <EOL> if self . start_year == '<STR_LIT>' : <EOL> return <NUM_LIT:0> <EOL> return int ( re . sub ( r'<STR_LIT>' , '<STR_LIT:0>' , self . start_year ) ) <EOL> @ property <EOL> def end_year_int ( self ) : <EOL> if self . end_year == '<STR_LIT>' : <EOL> return <NUM_LIT> <EOL> return int ( re . sub ( r'<STR_LIT>' , '<STR_LIT>' , self . end_year ) ) <EOL> def _lookup_title_links ( self , links ) : <EOL> titles = [ ] <EOL> for link in links : <EOL> if link . lccn : <EOL> try : <EOL> t = Title . objects . get ( lccn = link . lccn ) <EOL> titles . append ( t ) <EOL> continue <EOL> except Title . DoesNotExist : <EOL> pass <EOL> if link . oclc : <EOL> try : <EOL> t = Title . objects . get ( oclc = "<STR_LIT>" + link . oclc ) <EOL> titles . append ( t ) <EOL> except : <EOL> pass <EOL> return titles <EOL> def __unicode__ ( self ) : <EOL> return u'<STR_LIT>' % ( self . display_name , self . place_of_publication , <EOL> self . start_year , self . end_year ) <EOL> class Meta : <EOL> ordering = [ '<STR_LIT>' ] <EOL> class AltTitle ( models . Model ) : <EOL> name = models . CharField ( max_length = <NUM_LIT> ) <EOL> date = models . CharField ( max_length = <NUM_LIT> , null = True ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> class Meta : <EOL> ordering = [ '<STR_LIT:name>' ] <EOL> class MARC ( models . Model ) : <EOL> xml = models . TextField ( ) <EOL> title = models . OneToOneField ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> @ property <EOL> def html ( self ) : <EOL> """<STR_LIT>""" <EOL> doc = etree . fromstring ( self . xml ) <EOL> table = etree . Element ( '<STR_LIT>' ) <EOL> table . attrib [ '<STR_LIT:class>' ] = '<STR_LIT>' <EOL> tr = etree . SubElement ( table , '<STR_LIT>' ) <EOL> tr . attrib [ '<STR_LIT:class>' ] = '<STR_LIT>' <EOL> etree . SubElement ( tr , '<STR_LIT>' ) . text = '<STR_LIT>' <EOL> etree . SubElement ( tr , '<STR_LIT>' ) <EOL> etree . SubElement ( tr , '<STR_LIT>' ) . text = doc . find ( '<STR_LIT>' ) . text <EOL> for control_field in doc . findall ( '<STR_LIT>' ) : <EOL> tr = etree . SubElement ( table , '<STR_LIT>' ) <EOL> etree . SubElement ( tr , '<STR_LIT>' ) . text = control_field . attrib [ '<STR_LIT>' ] <EOL> etree . SubElement ( tr , '<STR_LIT>' ) <EOL> etree . SubElement ( tr , '<STR_LIT>' ) . text = control_field . text <EOL> for field in doc . findall ( '<STR_LIT>' ) : <EOL> tr = etree . SubElement ( table , '<STR_LIT>' ) <EOL> etree . SubElement ( tr , '<STR_LIT>' ) . text = field . attrib [ '<STR_LIT>' ] <EOL> indicators = etree . SubElement ( tr , '<STR_LIT>' ) <EOL> indicators . attrib [ '<STR_LIT:class>' ] = '<STR_LIT>' <EOL> indicators . text = field . attrib [ '<STR_LIT>' ] <EOL> indicators . text += field . attrib [ '<STR_LIT>' ] <EOL> td = etree . SubElement ( tr , '<STR_LIT>' ) <EOL> for subfield in field . findall ( '<STR_LIT>' ) : <EOL> code = etree . SubElement ( td , '<STR_LIT>' ) <EOL> code . attrib [ '<STR_LIT:class>' ] = '<STR_LIT>' <EOL> code . text = '<STR_LIT>' % subfield . attrib [ '<STR_LIT:code>' ] <EOL> value = etree . SubElement ( td , '<STR_LIT>' ) <EOL> value . attrib [ '<STR_LIT:class>' ] = '<STR_LIT>' <EOL> if field . attrib [ '<STR_LIT>' ] == '<STR_LIT>' and subfield . attrib [ '<STR_LIT:code>' ] == '<STR_LIT:u>' : <EOL> a = etree . SubElement ( value , '<STR_LIT:a>' ) <EOL> a . attrib [ '<STR_LIT>' ] = subfield . text <EOL> a . text = '<STR_LIT:U+0020>' . join ( textwrap . wrap ( subfield . text , <NUM_LIT> ) ) <EOL> else : <EOL> value . text = '<STR_LIT:U+0020>' . join ( textwrap . wrap ( subfield . text , <NUM_LIT> ) ) <EOL> return etree . tostring ( table , pretty_print = True ) <EOL> @ property <EOL> @ permalink <EOL> def url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . title . lccn } ) <EOL> class Issue ( models . Model ) : <EOL> date_issued = models . DateField ( db_index = True ) <EOL> volume = models . CharField ( null = True , max_length = <NUM_LIT:50> ) <EOL> number = models . CharField ( max_length = <NUM_LIT:50> ) <EOL> edition = models . IntegerField ( ) <EOL> edition_label = models . CharField ( max_length = <NUM_LIT:100> ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> batch = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> def __unicode__ ( self ) : <EOL> return "<STR_LIT>" % ( self . title . display_name , self . date_issued ) <EOL> @ property <EOL> @ permalink <EOL> def url ( self ) : <EOL> date = self . date_issued <EOL> return ( '<STR_LIT>' , ( ) , <EOL> { '<STR_LIT>' : self . title . lccn , <EOL> '<STR_LIT:date>' : "<STR_LIT>" % ( date . year , date . month , date . day ) , <EOL> '<STR_LIT>' : self . edition } ) <EOL> @ property <EOL> @ permalink <EOL> def json_url ( self ) : <EOL> date = self . date_issued <EOL> return ( '<STR_LIT>' , ( ) , <EOL> { '<STR_LIT>' : self . title . lccn , <EOL> '<STR_LIT:date>' : "<STR_LIT>" % ( date . year , date . month , date . day ) , <EOL> '<STR_LIT>' : self . edition } ) <EOL> @ property <EOL> def abstract_url ( self ) : <EOL> return self . url . rstrip ( '<STR_LIT:/>' ) + '<STR_LIT>' <EOL> @ property <EOL> def first_page ( self ) : <EOL> try : <EOL> return self . pages . all ( ) [ <NUM_LIT:0> ] <EOL> except Exception : <EOL> return None <EOL> @ property <EOL> def first_page_with_image ( self ) : <EOL> for page in self . pages . all ( ) : <EOL> if page . jp2_filename : <EOL> return page <EOL> return None <EOL> @ property <EOL> def _previous ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> previous_issue = self . get_previous_by_date_issued ( title = self . title ) <EOL> except Issue . DoesNotExist : <EOL> previous_issue = None <EOL> return previous_issue <EOL> @ property <EOL> def _next ( self ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> next_issue = self . get_next_by_date_issued ( title = self . title ) <EOL> except Issue . DoesNotExist : <EOL> next_issue = None <EOL> return next_issue <EOL> @ property <EOL> def previous ( self ) : <EOL> """<STR_LIT>""" <EOL> previous_issue = self . _previous <EOL> while previous_issue is not None and previous_issue . date_issued == self . date_issued and previous_issue . edition == self . edition : <EOL> previous_issue = previous_issue . _previous <EOL> return previous_issue <EOL> @ property <EOL> def next ( self ) : <EOL> """<STR_LIT>""" <EOL> next_issue = self . _next <EOL> while next_issue is not None and next_issue . date_issued == self . date_issued and next_issue . edition == self . edition : <EOL> next_issue = next_issue . _next <EOL> return next_issue <EOL> def save ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> self . title . has_issues = True <EOL> self . title . save ( ) <EOL> super ( Issue , self ) . save ( * args , ** kwargs ) <EOL> def delete ( self , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> super ( Issue , self ) . delete ( * args , ** kwargs ) <EOL> if self . title . issues . all ( ) . count ( ) == <NUM_LIT:0> : <EOL> self . title . has_issues = False <EOL> self . title . save ( ) <EOL> def json ( self , serialize = True , include_pages = True , host = '<STR_LIT>' ) : <EOL> j = { <EOL> '<STR_LIT:url>' : '<STR_LIT>' + host + self . json_url , <EOL> '<STR_LIT>' : strftime ( self . date_issued , "<STR_LIT>" ) , <EOL> '<STR_LIT>' : self . volume , <EOL> '<STR_LIT>' : self . number , <EOL> '<STR_LIT>' : self . edition , <EOL> '<STR_LIT:title>' : { "<STR_LIT:name>" : self . title . display_name , "<STR_LIT:url>" : '<STR_LIT>' + host + self . title . json_url } , <EOL> '<STR_LIT>' : { "<STR_LIT:name>" : self . batch . name , "<STR_LIT:url>" : '<STR_LIT>' + host + self . batch . json_url } , <EOL> } <EOL> j [ '<STR_LIT>' ] = [ { <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + p . json_url , <EOL> "<STR_LIT>" : p . sequence <EOL> } for p in self . pages . all ( ) ] <EOL> if serialize : <EOL> return json . dumps ( j , indent = <NUM_LIT:2> ) <EOL> return j <EOL> class Meta : <EOL> ordering = ( '<STR_LIT>' , ) <EOL> class Page ( models . Model ) : <EOL> sequence = models . IntegerField ( db_index = True ) <EOL> number = models . CharField ( max_length = <NUM_LIT:50> ) <EOL> section_label = models . CharField ( max_length = <NUM_LIT:100> ) <EOL> tiff_filename = models . CharField ( max_length = <NUM_LIT> ) <EOL> jp2_filename = models . CharField ( max_length = <NUM_LIT> , null = True ) <EOL> jp2_width = models . IntegerField ( null = True ) <EOL> jp2_length = models . IntegerField ( null = True ) <EOL> pdf_filename = models . CharField ( max_length = <NUM_LIT> , null = True ) <EOL> ocr_filename = models . CharField ( max_length = <NUM_LIT> , null = True ) <EOL> issue = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> reel = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' , null = True ) <EOL> indexed = models . BooleanField ( ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> def json ( self , serialize = True , host = "<STR_LIT>" ) : <EOL> j = { <EOL> "<STR_LIT>" : self . sequence , <EOL> "<STR_LIT>" : { <EOL> "<STR_LIT>" : strftime ( self . issue . date_issued , "<STR_LIT>" ) , <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + self . issue . json_url } , <EOL> "<STR_LIT>" : "<STR_LIT>" + host + self . jp2_url , <EOL> "<STR_LIT>" : "<STR_LIT>" + host + self . ocr_url , <EOL> "<STR_LIT:text>" : "<STR_LIT>" + host + self . txt_url , <EOL> "<STR_LIT>" : "<STR_LIT>" + host + self . pdf_url , <EOL> "<STR_LIT:title>" : { <EOL> "<STR_LIT:name>" : self . issue . title . display_name , <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + self . issue . title . json_url } <EOL> } <EOL> if serialize : <EOL> return json . dumps ( j , indent = <NUM_LIT:2> ) <EOL> return j <EOL> @ property <EOL> def jp2_abs_filename ( self ) : <EOL> return self . _abs_path ( self . jp2_filename ) <EOL> @ property <EOL> def tiff_abs_filename ( self ) : <EOL> return self . _abs_path ( self . tiff_filename ) <EOL> @ property <EOL> def pdf_abs_filename ( self ) : <EOL> return self . _abs_path ( self . pdf_filename ) <EOL> @ property <EOL> def ocr_abs_filename ( self ) : <EOL> return self . _abs_path ( self . ocr_filename ) <EOL> @ property <EOL> def noteAboutReproduction ( self ) : <EOL> try : <EOL> return self . notes . filter ( type = "<STR_LIT>" ) [ <NUM_LIT:0> ] <EOL> except IndexError : <EOL> return "<STR_LIT>" <EOL> def _abs_path ( self , rel_path ) : <EOL> if rel_path : <EOL> return os . path . join ( self . issue . batch . path , rel_path ) <EOL> else : <EOL> return None <EOL> def _url_parts ( self ) : <EOL> date = self . issue . date_issued <EOL> return { '<STR_LIT>' : self . issue . title . lccn , <EOL> '<STR_LIT:date>' : "<STR_LIT>" % ( date . year , date . month , date . day ) , <EOL> '<STR_LIT>' : self . issue . edition , <EOL> '<STR_LIT>' : self . sequence } <EOL> @ property <EOL> @ permalink <EOL> def url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> @ permalink <EOL> def json_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> def abstract_url ( self ) : <EOL> return self . url . rstrip ( '<STR_LIT:/>' ) + '<STR_LIT>' <EOL> @ property <EOL> @ permalink <EOL> def thumb_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> @ permalink <EOL> def medium_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> @ permalink <EOL> def jp2_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> @ permalink <EOL> def ocr_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> @ permalink <EOL> def txt_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> @ permalink <EOL> def pdf_url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , self . _url_parts ( ) ) <EOL> @ property <EOL> def solr_doc ( self ) : <EOL> date = self . issue . date_issued <EOL> month , day , year = '<STR_LIT>' % date . month , '<STR_LIT>' % date . day , '<STR_LIT>' % date . year <EOL> date = '<STR_LIT>' . join ( [ year , month , day ] ) <EOL> doc = self . issue . title . solr_doc <EOL> del doc [ '<STR_LIT>' ] <EOL> del doc [ '<STR_LIT:url>' ] <EOL> del doc [ '<STR_LIT>' ] <EOL> doc . update ( { <EOL> '<STR_LIT:id>' : self . url , <EOL> '<STR_LIT:type>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : self . issue . batch . name , <EOL> '<STR_LIT:date>' : date , <EOL> '<STR_LIT>' : month , <EOL> '<STR_LIT>' : year , <EOL> '<STR_LIT>' : day , <EOL> '<STR_LIT>' : self . number , <EOL> '<STR_LIT>' : self . sequence , <EOL> '<STR_LIT>' : self . section_label , <EOL> '<STR_LIT>' : self . issue . edition_label , <EOL> } ) <EOL> try : <EOL> ocr_texts = self . ocr . language_texts . select_related ( ) . values ( '<STR_LIT>' , '<STR_LIT:text>' ) <EOL> except OCR . DoesNotExist : <EOL> ocr_texts = None <EOL> for ocr_text in ocr_texts : <EOL> lang = ocr_text [ '<STR_LIT>' ] <EOL> if lang not in settings . SOLR_LANGUAGES : <EOL> lang = "<STR_LIT>" <EOL> doc [ '<STR_LIT>' % lang ] = ocr_text [ '<STR_LIT:text>' ] <EOL> return doc <EOL> def previous ( self ) : <EOL> """<STR_LIT>""" <EOL> previous_page = None <EOL> pages = list ( self . issue . pages . all ( ) . order_by ( "<STR_LIT>" ) ) <EOL> i = pages . index ( self ) <EOL> if i >= <NUM_LIT:0> : <EOL> i += <NUM_LIT:1> <EOL> if i < len ( pages ) : <EOL> previous_page = pages [ i ] <EOL> return previous_page <EOL> def next ( self ) : <EOL> """<STR_LIT>""" <EOL> next_page = None <EOL> pages = list ( self . issue . pages . all ( ) . order_by ( "<STR_LIT>" ) ) <EOL> i = pages . index ( self ) <EOL> if i >= <NUM_LIT:0> : <EOL> i += <NUM_LIT:1> <EOL> if i < len ( pages ) : <EOL> next_page = pages [ i ] <EOL> return next_page <EOL> @ classmethod <EOL> def lookup ( cls , page_id ) : <EOL> """<STR_LIT>""" <EOL> m = re . match ( r'<STR_LIT>' , page_id ) <EOL> if not m : <EOL> return None <EOL> lccn , date , edition , sequence = m . groups ( ) <EOL> q = Page . objects . filter ( issue__title__lccn = lccn , <EOL> issue__date_issued = date , <EOL> issue__edition = edition , <EOL> sequence = sequence ) <EOL> pages = q . order_by ( '<STR_LIT>' ) . all ( ) <EOL> if len ( pages ) == <NUM_LIT:0> : <EOL> return None <EOL> return pages [ <NUM_LIT:0> ] <EOL> def on_flickr ( self ) : <EOL> return len ( self . flickr_urls . all ( ) ) > <NUM_LIT:0> <EOL> def first_flickr_url ( self ) : <EOL> for flickr_url in self . flickr_urls . all ( ) : <EOL> return flickr_url . value <EOL> return None <EOL> def __unicode__ ( self ) : <EOL> parts = [ u'<STR_LIT:%s>' % self . issue . title ] <EOL> parts . append ( strftime ( self . issue . date_issued , '<STR_LIT>' ) ) <EOL> if self . issue . edition_label : <EOL> parts . append ( self . issue . edition_label ) <EOL> if self . section_label : <EOL> parts . append ( self . section_label ) <EOL> parts . append ( '<STR_LIT>' % self . sequence ) <EOL> return u'<STR_LIT:U+002CU+0020>' . join ( parts ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT>' , ) <EOL> class Admin : <EOL> pass <EOL> class LanguageText ( models . Model ) : <EOL> text = models . TextField ( ) <EOL> language = models . ForeignKey ( '<STR_LIT>' , null = True ) <EOL> ocr = models . ForeignKey ( '<STR_LIT>' , related_name = "<STR_LIT>" ) <EOL> class OCR ( models . Model ) : <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> page = models . OneToOneField ( '<STR_LIT>' , null = True , related_name = '<STR_LIT>' ) <EOL> @ property <EOL> def text ( self ) : <EOL> return ( '<STR_LIT:U+0020>' . join ( [ obj . text for obj in self . language_texts . all ( ) ] ) ) <EOL> class PublicationDate ( models . Model ) : <EOL> text = models . CharField ( max_length = <NUM_LIT> ) <EOL> titles = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> class Meta : <EOL> ordering = [ '<STR_LIT:text>' ] <EOL> class Place ( models . Model ) : <EOL> name = models . CharField ( primary_key = True , max_length = <NUM_LIT:100> ) <EOL> city = models . CharField ( null = True , max_length = <NUM_LIT:100> , db_index = True ) <EOL> county = models . CharField ( null = True , max_length = <NUM_LIT:100> , db_index = True ) <EOL> state = models . CharField ( null = True , max_length = <NUM_LIT:100> , db_index = True ) <EOL> country = models . CharField ( null = True , max_length = <NUM_LIT:100> ) <EOL> titles = models . ManyToManyField ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> dbpedia = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> geonames = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> latitude = models . FloatField ( null = True ) <EOL> longitude = models . FloatField ( null = True ) <EOL> def __unicode__ ( self ) : <EOL> return u"<STR_LIT>" % ( self . city , self . county , self . state ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class Subject ( models . Model ) : <EOL> heading = models . CharField ( max_length = <NUM_LIT> ) <EOL> type = models . CharField ( max_length = <NUM_LIT:1> ) <EOL> titles = models . ManyToManyField ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return self . heading <EOL> class Meta : <EOL> ordering = ( '<STR_LIT>' , ) <EOL> class Note ( models . Model ) : <EOL> text = models . TextField ( ) <EOL> type = models . CharField ( max_length = <NUM_LIT:3> ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return self . text <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:text>' , ) <EOL> class PageNote ( models . Model ) : <EOL> label = models . TextField ( ) <EOL> text = models . TextField ( ) <EOL> type = models . CharField ( max_length = <NUM_LIT:50> ) <EOL> page = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return u"<STR_LIT>" % ( self . type , self . label , self . text ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:text>' , ) <EOL> class IssueNote ( models . Model ) : <EOL> label = models . TextField ( ) <EOL> text = models . TextField ( ) <EOL> type = models . CharField ( max_length = <NUM_LIT:50> ) <EOL> issue = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return u"<STR_LIT>" % ( self . type , self . label , self . text ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:text>' , ) <EOL> class Essay ( models . Model ) : <EOL> title = models . TextField ( ) <EOL> created = models . DateTimeField ( ) <EOL> modified = models . DateTimeField ( ) <EOL> creator = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> essay_editor_url = models . TextField ( ) <EOL> html = models . TextField ( ) <EOL> loaded = models . DateTimeField ( auto_now_add = True ) <EOL> titles = models . ManyToManyField ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def first_title ( self ) : <EOL> return self . titles . all ( ) [ <NUM_LIT:0> ] <EOL> @ property <EOL> @ permalink <EOL> def url ( self ) : <EOL> return ( '<STR_LIT>' , ( ) , { '<STR_LIT>' : self . id } ) <EOL> class Meta : <EOL> ordering = [ '<STR_LIT:title>' ] <EOL> class Holding ( models . Model ) : <EOL> description = models . TextField ( null = True ) <EOL> type = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> institution = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> last_updated = models . CharField ( null = True , max_length = <NUM_LIT:10> ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> notes = models . TextField ( null = True , help_text = "<STR_LIT>" ) <EOL> def description_as_list ( self ) : <EOL> desc_list = [ ] <EOL> desc_txt = self . description <EOL> for d in desc_txt . split ( ) : <EOL> try : <EOL> if d [ <NUM_LIT:1> ] == '<STR_LIT:=>' and d . endswith ( '<STR_LIT:>>' ) : <EOL> desc_list . append ( d ) <EOL> desc_txt = desc_txt . replace ( d , '<STR_LIT>' ) <EOL> except IndexError : <EOL> continue <EOL> l = re . findall ( r'<STR_LIT>' , desc_txt ) <EOL> if l : <EOL> [ desc_list . append ( d ) for d in l ] <EOL> if desc_list : <EOL> return desc_list <EOL> else : <EOL> return [ self . description ] <EOL> def __unicode__ ( self ) : <EOL> return u"<STR_LIT>" % ( self . institution . name , self . type , self . description ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT>' , ) <EOL> class SucceedingTitleLink ( models . Model ) : <EOL> name = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> lccn = models . CharField ( null = True , max_length = <NUM_LIT:50> ) <EOL> oclc = models . CharField ( null = True , max_length = <NUM_LIT:50> ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class PreceedingTitleLink ( models . Model ) : <EOL> name = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> lccn = models . CharField ( null = True , max_length = <NUM_LIT:50> ) <EOL> oclc = models . CharField ( null = True , max_length = <NUM_LIT:50> ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return "<STR_LIT>" % ( self . name , self . lccn ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class RelatedTitleLink ( models . Model ) : <EOL> name = models . CharField ( null = True , max_length = <NUM_LIT> ) <EOL> lccn = models . CharField ( null = True , max_length = <NUM_LIT:50> ) <EOL> oclc = models . CharField ( null = True , max_length = <NUM_LIT:50> ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return "<STR_LIT>" % ( self . name , self . lccn ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class Ethnicity ( models . Model ) : <EOL> name = models . CharField ( null = False , max_length = <NUM_LIT> , primary_key = True ) <EOL> @ property <EOL> def has_issues ( self ) : <EOL> return self . titles_with_issues ( ) . count ( ) > <NUM_LIT:0> <EOL> def titles_with_issues ( self ) : <EOL> return self . titles ( ) . filter ( has_issues = True ) <EOL> def titles ( self ) : <EOL> f = Q ( subjects__heading__icontains = self . name ) <EOL> for s in self . synonyms . all ( ) : <EOL> f |= Q ( subjects__heading__icontains = s . synonym ) <EOL> return Title . objects . filter ( f ) . distinct ( ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class EthnicitySynonym ( models . Model ) : <EOL> synonym = models . CharField ( null = False , max_length = <NUM_LIT> ) <EOL> ethnicity = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT>' , ) <EOL> class Language ( models . Model ) : <EOL> code = models . CharField ( null = False , max_length = <NUM_LIT:3> , primary_key = True ) <EOL> name = models . CharField ( null = False , max_length = <NUM_LIT:100> ) <EOL> lingvoj = models . CharField ( null = True , max_length = <NUM_LIT:200> ) <EOL> titles = models . ManyToManyField ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return self . name <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class Country ( models . Model ) : <EOL> """<STR_LIT>""" <EOL> code = models . CharField ( null = False , max_length = <NUM_LIT:3> , primary_key = True ) <EOL> name = models . CharField ( null = False , max_length = <NUM_LIT:100> ) <EOL> region = models . CharField ( null = False , max_length = <NUM_LIT:100> ) <EOL> def __unicode__ ( self ) : <EOL> return "<STR_LIT>" % ( self . name , self . region ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class LaborPress ( models . Model ) : <EOL> name = models . CharField ( null = False , max_length = <NUM_LIT> , primary_key = True ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class MaterialType ( models . Model ) : <EOL> name = models . CharField ( null = False , max_length = <NUM_LIT> , primary_key = True ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class Institution ( models . Model ) : <EOL> code = models . CharField ( primary_key = True , max_length = <NUM_LIT:50> ) <EOL> name = models . CharField ( null = False , max_length = <NUM_LIT:255> ) <EOL> address1 = models . CharField ( null = True , max_length = <NUM_LIT:255> ) <EOL> address2 = models . CharField ( null = True , max_length = <NUM_LIT:255> ) <EOL> city = models . CharField ( null = False , max_length = <NUM_LIT:100> ) <EOL> state = models . CharField ( null = False , max_length = <NUM_LIT:2> ) <EOL> zip = models . CharField ( null = True , max_length = <NUM_LIT:20> ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> def __unicode__ ( self ) : <EOL> return u"<STR_LIT>" % ( self . name , self . city , self . state ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:name>' , ) <EOL> class PhysicalDescription ( models . Model ) : <EOL> text = models . TextField ( ) <EOL> type = models . CharField ( max_length = <NUM_LIT:3> ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> class Meta : <EOL> ordering = ( '<STR_LIT:type>' , ) <EOL> class Url ( models . Model ) : <EOL> value = models . TextField ( ) <EOL> type = models . CharField ( max_length = <NUM_LIT:1> , null = True ) <EOL> title = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> def __unicode__ ( self ) : <EOL> return self . value <EOL> class FlickrUrl ( models . Model ) : <EOL> value = models . TextField ( ) <EOL> page = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> def __unicode__ ( self ) : <EOL> return self . value <EOL> class Reel ( models . Model ) : <EOL> number = models . CharField ( max_length = <NUM_LIT:50> ) <EOL> batch = models . ForeignKey ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> implicit = models . BooleanField ( default = False ) <EOL> def titles ( self ) : <EOL> return Title . objects . filter ( issues__pages__reel = self ) . distinct ( ) <EOL> class OcrDump ( models . Model ) : <EOL> created = models . DateTimeField ( auto_now_add = True ) <EOL> sha1 = models . TextField ( ) <EOL> size = models . BigIntegerField ( ) <EOL> batch = models . OneToOneField ( '<STR_LIT>' , related_name = '<STR_LIT>' ) <EOL> @ classmethod <EOL> def new_from_batch ( klass , batch ) : <EOL> """<STR_LIT>""" <EOL> dump = OcrDump ( batch = batch ) <EOL> tar = tarfile . open ( dump . path , "<STR_LIT>" ) <EOL> for issue in batch . issues . all ( ) : <EOL> for page in issue . pages . filter ( ocr__isnull = False ) : <EOL> dump . _add_page ( page , tar ) <EOL> tar . close ( ) <EOL> dump . _calculate_size ( ) <EOL> dump . _calculate_sha1 ( ) <EOL> dump . save ( ) <EOL> return dump <EOL> @ property <EOL> def name ( self ) : <EOL> return self . batch . name + "<STR_LIT>" <EOL> @ property <EOL> def url ( self ) : <EOL> path = self . path . replace ( settings . STORAGE , "<STR_LIT>" ) <EOL> return os . path . join ( path ) <EOL> @ classmethod <EOL> def last ( klass ) : <EOL> dumps = OcrDump . objects . all ( ) . order_by ( "<STR_LIT>" ) <EOL> if dumps . count ( ) > <NUM_LIT:0> : <EOL> return dumps [ <NUM_LIT:0> ] <EOL> return None <EOL> @ property <EOL> def path ( self ) : <EOL> return os . path . join ( settings . OCR_DUMP_STORAGE , self . name ) <EOL> def json ( self , serialize = True , host = "<STR_LIT>" ) : <EOL> j = { <EOL> "<STR_LIT:name>" : self . name , <EOL> "<STR_LIT>" : rfc3339 ( self . created ) , <EOL> "<STR_LIT:size>" : self . size , <EOL> "<STR_LIT>" : self . sha1 , <EOL> "<STR_LIT:url>" : "<STR_LIT>" + host + self . url <EOL> } <EOL> if serialize : <EOL> return json . dumps ( i , indent = <NUM_LIT:2> ) <EOL> return j <EOL> def __unicode__ ( self ) : <EOL> return "<STR_LIT>" % ( self . path , self . size , self . sha1 ) <EOL> def _add_page ( self , page , tar ) : <EOL> d = page . issue . date_issued <EOL> relative_dir = "<STR_LIT>" % ( page . issue . title_id , d . year , d . month , d . day , page . issue . edition , page . sequence ) <EOL> txt_filename = relative_dir + "<STR_LIT>" <EOL> ocr_text = page . ocr . text . encode ( '<STR_LIT:utf-8>' ) <EOL> info = tarfile . TarInfo ( name = txt_filename ) <EOL> info . size = len ( ocr_text ) <EOL> info . mtime = time . time ( ) <EOL> tar . addfile ( info , StringIO ( ocr_text ) ) <EOL> xml_filename = relative_dir + "<STR_LIT>" <EOL> info = tarfile . TarInfo ( name = xml_filename ) <EOL> info . size = os . path . getsize ( page . ocr_abs_filename ) <EOL> info . mtime = time . time ( ) <EOL> tar . addfile ( info , open ( page . ocr_abs_filename ) ) <EOL> logging . info ( "<STR_LIT>" % ( page , tar . name ) ) <EOL> def delete ( self , * args , ** kwargs ) : <EOL> if os . path . isfile ( self . path ) : <EOL> os . remove ( self . path ) <EOL> return super ( OcrDump , self ) . delete ( * args , ** kwargs ) <EOL> def _calculate_size ( self ) : <EOL> self . size = os . path . getsize ( self . path ) <EOL> def _calculate_sha1 ( self ) : <EOL> """<STR_LIT>""" <EOL> f = open ( self . path ) <EOL> sha1 = hashlib . sha1 ( ) <EOL> while True : <EOL> buff = f . read ( <NUM_LIT:2> ** <NUM_LIT:16> ) <EOL> if not buff : <EOL> break <EOL> sha1 . update ( buff ) <EOL> self . sha1 = sha1 . hexdigest ( ) <EOL> return self . sha1 <EOL> def coordinates_path ( url_parts ) : <EOL> url = urlresolvers . reverse ( '<STR_LIT>' , kwargs = url_parts ) <EOL> path = url2pathname ( url ) <EOL> if path . startswith ( "<STR_LIT:/>" ) : <EOL> path = path [ <NUM_LIT:1> : ] <EOL> full_path = os . path . join ( settings . COORD_STORAGE , path ) <EOL> if not os . path . exists ( full_path ) : <EOL> os . makedirs ( full_path ) <EOL> return os . path . join ( full_path , "<STR_LIT>" ) <EOL> class TopicCategory ( models . Model ) : <EOL> name = models . CharField ( max_length = <NUM_LIT> ) <EOL> date_synced = models . DateTimeField ( auto_now_add = True ) <EOL> class Topic ( models . Model ) : <EOL> name = models . CharField ( max_length = <NUM_LIT> ) <EOL> category = models . ForeignKey ( '<STR_LIT>' ) <EOL> topic_start_year = models . IntegerField ( ) <EOL> topic_end_year = models . IntegerField ( ) <EOL> suggested_search_terms = models . TextField ( ) <EOL> intro_text = models . TextField ( ) <EOL> important_dates = models . TextField ( ) <EOL> class TopicPages ( models . Model ) : <EOL> page = models . ForeignKey ( '<STR_LIT>' , null = True ) <EOL> topic = models . ForeignKey ( '<STR_LIT>' ) <EOL> query_params = models . CharField ( max_length = <NUM_LIT> ) <EOL> title = models . CharField ( max_length = <NUM_LIT> ) <EOL> url = models . CharField ( max_length = <NUM_LIT:1000> ) <EOL> description = models . TextField ( ) </s>
