<s> import re <EOL> import math <EOL> import logging <EOL> import datetime <EOL> from urllib import urlencode , unquote <EOL> from solr import SolrConnection <EOL> from django . core import urlresolvers <EOL> from django . core . paginator import Paginator , Page <EOL> from django . db import connection , reset_queries <EOL> from django . http import QueryDict <EOL> from django . conf import settings <EOL> from chronam . core import models <EOL> from chronam . core . forms import _fulltext_range <EOL> from chronam . core . utils import utils <EOL> from chronam . core . title_loader import _normal_lccn <EOL> _log = logging . getLogger ( __name__ ) <EOL> PROX_DISTANCE_DEFAULT = <NUM_LIT:5> <EOL> ESCAPE_CHARS_RE = re . compile ( r'<STR_LIT>' ) <EOL> def solr_escape ( value ) : <EOL> """<STR_LIT>""" <EOL> return ESCAPE_CHARS_RE . sub ( r'<STR_LIT>' , value ) <EOL> def page_count ( ) : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> return solr . query ( '<STR_LIT>' , fields = [ '<STR_LIT:id>' ] ) . numFound <EOL> def title_count ( ) : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> return solr . query ( '<STR_LIT>' , fields = [ '<STR_LIT:id>' ] ) . numFound <EOL> class SolrPaginator ( Paginator ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , query ) : <EOL> self . query = query . copy ( ) <EOL> if '<STR_LIT>' in self . query : <EOL> del self . query [ '<STR_LIT>' ] <EOL> self . _q , self . facet_params = page_search ( self . query ) <EOL> try : <EOL> self . _cur_page = int ( self . query . get ( '<STR_LIT>' ) ) <EOL> except : <EOL> self . _cur_page = <NUM_LIT:1> <EOL> try : <EOL> self . _cur_index = int ( self . query . get ( '<STR_LIT:index>' ) ) <EOL> except : <EOL> self . _cur_index = <NUM_LIT:0> <EOL> try : <EOL> rows = int ( self . query . get ( '<STR_LIT>' ) ) <EOL> except : <EOL> rows = <NUM_LIT:10> <EOL> Paginator . __init__ ( self , None , per_page = rows , orphans = <NUM_LIT:0> ) <EOL> self . overall_index = ( self . _cur_page - <NUM_LIT:1> ) * self . per_page + self . _cur_index <EOL> self . _ocr_list = [ '<STR_LIT>' , ] <EOL> self . _ocr_list . extend ( [ '<STR_LIT>' % l for l in settings . SOLR_LANGUAGES ] ) <EOL> def _get_count ( self ) : <EOL> "<STR_LIT>" <EOL> if self . _count is None : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> solr_response = solr . query ( self . _q , fields = [ '<STR_LIT:id>' ] ) <EOL> self . _count = int ( solr_response . results . numFound ) <EOL> return self . _count <EOL> count = property ( _get_count ) <EOL> def highlight_url ( self , url , words , page , index ) : <EOL> q = self . query . copy ( ) <EOL> q [ "<STR_LIT>" ] = "<STR_LIT:U+0020>" . join ( words ) <EOL> q [ "<STR_LIT>" ] = page <EOL> q [ "<STR_LIT:index>" ] = index <EOL> return url + "<STR_LIT:#>" + q . urlencode ( ) <EOL> def _get_previous ( self ) : <EOL> previous_overall_index = self . overall_index - <NUM_LIT:1> <EOL> if previous_overall_index >= <NUM_LIT:0> : <EOL> p_page = previous_overall_index / self . per_page + <NUM_LIT:1> <EOL> p_index = previous_overall_index % self . per_page <EOL> o = self . page ( p_page ) . object_list [ p_index ] <EOL> q = self . query . copy ( ) <EOL> return self . highlight_url ( o . url , o . words , p_page , p_index ) <EOL> else : <EOL> return None <EOL> previous_result = property ( _get_previous ) <EOL> def _get_next ( self ) : <EOL> next_overall_index = self . overall_index + <NUM_LIT:1> <EOL> if next_overall_index < self . count : <EOL> n_page = next_overall_index / self . per_page + <NUM_LIT:1> <EOL> n_index = next_overall_index % self . per_page <EOL> o = self . page ( n_page ) . object_list [ n_index ] <EOL> return self . highlight_url ( o . url , o . words , n_page , n_index ) <EOL> else : <EOL> return None <EOL> next_result = property ( _get_next ) <EOL> def page ( self , number ) : <EOL> """<STR_LIT>""" <EOL> number = self . validate_number ( number ) <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> start = self . per_page * ( number - <NUM_LIT:1> ) <EOL> params = { "<STR_LIT>" : <NUM_LIT:100> , <EOL> "<STR_LIT>" : '<STR_LIT:true>' , <EOL> "<STR_LIT>" : '<STR_LIT>' , <EOL> } <EOL> params . update ( self . facet_params ) <EOL> sort_field , sort_order = _get_sort ( self . query . get ( '<STR_LIT>' ) , in_pages = True ) <EOL> solr_response = solr . query ( self . _q , <EOL> fields = [ '<STR_LIT:id>' , '<STR_LIT:title>' , '<STR_LIT:date>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] , <EOL> highlight = self . _ocr_list , <EOL> rows = self . per_page , <EOL> sort = sort_field , <EOL> sort_order = sort_order , <EOL> start = start , <EOL> ** params ) <EOL> solr_facets = solr_response . facet_counts <EOL> facets = { '<STR_LIT:state>' : sorted ( solr_facets . get ( '<STR_LIT>' ) [ '<STR_LIT:state>' ] . items ( ) , <EOL> lambda x , y : x - y , lambda k : k [ <NUM_LIT:1> ] , True ) , <EOL> '<STR_LIT>' : solr_facets [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : sorted ( solr_facets . get ( '<STR_LIT>' ) [ '<STR_LIT>' ] . items ( ) , <EOL> lambda x , y : x - y , lambda k : k [ <NUM_LIT:1> ] , True ) } <EOL> facets [ '<STR_LIT>' ] = sorted ( solr_facets [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ '<STR_LIT>' ] . items ( ) , <EOL> lambda x , y : int ( x ) - int ( y ) , lambda k : k [ <NUM_LIT:0> ] , True ) <EOL> facet_gap = self . facet_params [ '<STR_LIT>' ] <EOL> if facet_gap > <NUM_LIT:1> : <EOL> facets [ '<STR_LIT>' ] = [ ( '<STR_LIT>' % ( y [ <NUM_LIT:0> ] , int ( y [ <NUM_LIT:0> ] ) + facet_gap - <NUM_LIT:1> ) , y [ <NUM_LIT:1> ] ) <EOL> for y in facets [ '<STR_LIT>' ] ] <EOL> pages = [ ] <EOL> for result in solr_response . results : <EOL> page = models . Page . lookup ( result [ '<STR_LIT:id>' ] ) <EOL> if not page : <EOL> continue <EOL> words = set ( ) <EOL> coords = solr_response . highlighting [ result [ '<STR_LIT:id>' ] ] <EOL> for ocr in self . _ocr_list : <EOL> for s in coords . get ( ocr ) or [ ] : <EOL> words . update ( find_words ( s ) ) <EOL> page . words = sorted ( words , key = lambda v : v . lower ( ) ) <EOL> page . highlight_url = self . highlight_url ( page . url , <EOL> page . words , <EOL> number , len ( pages ) ) <EOL> pages . append ( page ) <EOL> solr_page = Page ( pages , number , self ) <EOL> solr_page . facets = facets <EOL> return solr_page <EOL> def pages ( self ) : <EOL> """<STR_LIT>""" <EOL> pages = [ ] <EOL> before = [ ] <EOL> middle = [ ] <EOL> end = [ ] <EOL> for p in self . page_range : <EOL> if p <= <NUM_LIT:3> : <EOL> before . append ( p ) <EOL> elif self . _num_pages - p <= <NUM_LIT:3> : <EOL> end . append ( p ) <EOL> elif abs ( p - self . _cur_page ) < <NUM_LIT:5> : <EOL> middle . append ( p ) <EOL> last = None <EOL> q = self . query . copy ( ) <EOL> for p in before + middle + end : <EOL> if last and p - last > <NUM_LIT:1> : <EOL> pages . append ( [ '<STR_LIT>' , None ] ) <EOL> else : <EOL> q [ '<STR_LIT>' ] = p <EOL> pages . append ( [ p , urlencode ( q ) ] ) <EOL> last = p <EOL> return pages <EOL> def englishify ( self ) : <EOL> """<STR_LIT>""" <EOL> d = self . query <EOL> parts = [ ] <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> parts . append ( '<STR_LIT>' . join ( d [ '<STR_LIT>' ] . split ( '<STR_LIT:U+0020>' ) ) ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> parts . append ( '<STR_LIT>' . join ( d [ '<STR_LIT>' ] . split ( '<STR_LIT:U+0020>' ) ) ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> parts . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> proxdistance = d . get ( '<STR_LIT>' , PROX_DISTANCE_DEFAULT ) <EOL> parts . append ( d [ '<STR_LIT>' ] ) <EOL> return parts <EOL> def results_table ( self ) : <EOL> """<STR_LIT>""" <EOL> table = [ ] <EOL> page = self . page ( self . _cur_page ) <EOL> objects = page . object_list <EOL> for i in range ( <NUM_LIT:0> , len ( objects ) , <NUM_LIT:2> ) : <EOL> h = objects [ i ] <EOL> row = [ h ] <EOL> if i + <NUM_LIT:1> < len ( objects ) : <EOL> h = objects [ i + <NUM_LIT:1> ] <EOL> row . append ( h ) <EOL> else : <EOL> row . append ( None ) <EOL> table . append ( row ) <EOL> return table <EOL> class SolrTitlesPaginator ( Paginator ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , query ) : <EOL> self . query = query . copy ( ) <EOL> q , fields , sort_field , sort_order , facets = get_solr_request_params_from_query ( self . query ) <EOL> year_facets = facets . pop ( '<STR_LIT>' ) <EOL> try : <EOL> page = int ( self . query . get ( '<STR_LIT>' ) ) <EOL> except : <EOL> page = <NUM_LIT:1> <EOL> try : <EOL> rows = int ( self . query . get ( '<STR_LIT>' ) ) <EOL> except : <EOL> rows = <NUM_LIT:50> <EOL> start = rows * ( page - <NUM_LIT:1> ) <EOL> query = q <EOL> solr_response = execute_solr_query ( q , fields , sort_field , sort_order , rows , start , facets ) <EOL> results = get_titles_from_solr_documents ( solr_response ) <EOL> Paginator . __init__ ( self , results , per_page = rows , orphans = <NUM_LIT:0> ) <EOL> self . _count = int ( solr_response . results . numFound ) <EOL> self . _num_pages = None <EOL> self . _cur_page = page <EOL> self . state_facets = sorted ( solr_response . facet_counts . get ( <EOL> '<STR_LIT>' ) [ '<STR_LIT:state>' ] . items ( ) , <EOL> lambda x , y : x - y , lambda k : k [ <NUM_LIT:1> ] , True ) <EOL> self . county_facets = sorted ( solr_response . facet_counts . get ( <EOL> '<STR_LIT>' ) [ '<STR_LIT>' ] . items ( ) , <EOL> lambda x , y : x - y , lambda k : k [ <NUM_LIT:1> ] , True ) <EOL> self . year_facets = year_facets <EOL> def page ( self , number ) : <EOL> """<STR_LIT>""" <EOL> number = self . validate_number ( number ) <EOL> return Page ( self . object_list , number , self ) <EOL> def get_titles_from_solr_documents ( solr_response ) : <EOL> """<STR_LIT>""" <EOL> lccns = [ d [ '<STR_LIT>' ] for d in solr_response . results ] <EOL> results = [ ] <EOL> for lccn in lccns : <EOL> try : <EOL> title = models . Title . objects . get ( lccn = lccn ) <EOL> results . append ( title ) <EOL> except models . Title . DoesNotExist , e : <EOL> pass <EOL> return results <EOL> def get_solr_request_params_from_query ( query ) : <EOL> q , facets = title_search ( query ) <EOL> fields = [ '<STR_LIT:id>' , '<STR_LIT:title>' , '<STR_LIT:date>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> sort_field , sort_order = _get_sort ( query . get ( '<STR_LIT>' ) ) <EOL> return q , fields , sort_field , sort_order , facets <EOL> def execute_solr_query ( query , fields , sort , sort_order , rows , start , facets ) : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> solr_response = solr . query ( query , <EOL> fields = [ '<STR_LIT>' , '<STR_LIT:title>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' ] , <EOL> rows = rows , <EOL> sort = sort , <EOL> sort_order = sort_order , <EOL> start = start , ** facets ) <EOL> return solr_response <EOL> def title_search ( d ) : <EOL> """<STR_LIT>""" <EOL> q = [ "<STR_LIT>" ] <EOL> if d . get ( '<STR_LIT:state>' ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT:state>' ] ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> for term in d . get ( '<STR_LIT>' , '<STR_LIT>' ) . replace ( '<STR_LIT:">' , '<STR_LIT>' ) . split ( ) : <EOL> q . append ( '<STR_LIT>' % ( term , term , term , term , term , term , term ) ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT:+>' + _expand_ethnicity ( d [ '<STR_LIT>' ] ) ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> year1 = d . get ( '<STR_LIT>' , None ) <EOL> if not year1 : <EOL> year1 = '<STR_LIT>' <EOL> year2 = d . get ( '<STR_LIT>' , None ) <EOL> if not year2 : <EOL> year2 = '<STR_LIT>' <EOL> if year1 != '<STR_LIT>' : <EOL> q . append ( '<STR_LIT>' % year1 ) <EOL> if year2 != '<STR_LIT>' : <EOL> q . append ( '<STR_LIT>' % year2 ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT>' % _normal_lccn ( d [ '<STR_LIT>' ] ) ) <EOL> if d . get ( '<STR_LIT>' ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> q = '<STR_LIT:U+0020>' . join ( q ) <EOL> range_gap = int ( math . ceil ( ( int ( year2 ) - int ( year1 ) ) / <NUM_LIT> ) ) * <NUM_LIT:5> <EOL> year_facets = range ( int ( year1 ) , int ( year2 ) , range_gap or <NUM_LIT:1> ) <EOL> facets = { '<STR_LIT>' : '<STR_LIT:true>' , '<STR_LIT>' : [ '<STR_LIT:state>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : <NUM_LIT:1> , '<STR_LIT>' : year_facets } <EOL> return q , facets <EOL> def page_search ( d ) : <EOL> """<STR_LIT>""" <EOL> q = [ '<STR_LIT>' ] <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> q . append ( query_join ( d . getlist ( '<STR_LIT>' ) , '<STR_LIT>' ) ) <EOL> if d . get ( '<STR_LIT:state>' , None ) : <EOL> q . append ( query_join ( d . getlist ( '<STR_LIT:state>' ) , '<STR_LIT:state>' ) ) <EOL> date_filter_type = d . get ( '<STR_LIT>' , None ) <EOL> date_boundaries = _fulltext_range ( ) <EOL> date1 = d . get ( '<STR_LIT>' , None ) <EOL> date2 = d . get ( '<STR_LIT>' , None ) <EOL> if not date1 : <EOL> date1 = date_boundaries [ <NUM_LIT:0> ] <EOL> if not date2 : <EOL> date2 = date_boundaries [ <NUM_LIT:1> ] <EOL> if date_filter_type == '<STR_LIT>' : <EOL> date1 = int ( date1 ) <EOL> date2 = int ( date2 ) <EOL> q . append ( '<STR_LIT>' % d ) <EOL> elif date_filter_type in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> d1 = _solrize_date ( str ( date1 ) ) <EOL> d2 = _solrize_date ( str ( date2 ) , is_start = False ) <EOL> if d1 and d2 : <EOL> date1 , date2 = map ( lambda d : int ( str ( d ) [ : <NUM_LIT:4> ] ) , ( d1 , d2 ) ) <EOL> q . append ( '<STR_LIT>' % ( d1 , d2 ) ) <EOL> gap = max ( <NUM_LIT:1> , int ( math . ceil ( ( date2 - date1 ) / <NUM_LIT:10> ) ) ) <EOL> ocrs = [ '<STR_LIT>' % l for l in settings . SOLR_LANGUAGES ] <EOL> lang = d . get ( '<STR_LIT>' , None ) <EOL> ocr_lang = '<STR_LIT>' + lang if lang else '<STR_LIT>' <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , "<STR_LIT>" ) ) <EOL> if lang : <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , ocr_lang ) ) <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , ocr_lang ) ) <EOL> else : <EOL> q . append ( '<STR_LIT:)>' ) <EOL> for ocr in ocrs : <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , ocr ) ) <EOL> q . append ( '<STR_LIT:)>' ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , "<STR_LIT>" , and_clause = True ) ) <EOL> if lang : <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , ocr_lang , and_clause = True ) ) <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , ocr_lang , and_clause = True ) ) <EOL> else : <EOL> q . append ( '<STR_LIT:)>' ) <EOL> for ocr in ocrs : <EOL> q . append ( '<STR_LIT>' + query_join ( solr_escape ( d [ '<STR_LIT>' ] ) . split ( '<STR_LIT:U+0020>' ) , ocr , and_clause = True ) ) <EOL> q . append ( '<STR_LIT:)>' ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> phrase = solr_escape ( d [ '<STR_LIT>' ] ) <EOL> q . append ( '<STR_LIT>' + '<STR_LIT>' + '<STR_LIT>' % ( phrase ) ) <EOL> if lang : <EOL> q . append ( '<STR_LIT>' + lang + '<STR_LIT>' % ( phrase ) ) <EOL> q . append ( '<STR_LIT>' + lang + '<STR_LIT>' % ( phrase ) ) <EOL> else : <EOL> q . append ( '<STR_LIT:)>' ) <EOL> for ocr in ocrs : <EOL> q . append ( '<STR_LIT>' + ocr + '<STR_LIT>' % ( phrase ) ) <EOL> q . append ( '<STR_LIT:)>' ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> distance = d . get ( '<STR_LIT>' , PROX_DISTANCE_DEFAULT ) <EOL> prox = solr_escape ( d [ '<STR_LIT>' ] ) <EOL> q . append ( '<STR_LIT>' + '<STR_LIT>' + '<STR_LIT>' % ( prox , distance ) ) <EOL> if lang : <EOL> q . append ( '<STR_LIT>' + lang + '<STR_LIT>' % ( prox , distance ) ) <EOL> q . append ( '<STR_LIT>' + lang + '<STR_LIT>' % ( prox , distance ) ) <EOL> else : <EOL> q . append ( '<STR_LIT:)>' ) <EOL> for ocr in ocrs : <EOL> q . append ( '<STR_LIT>' + ocr + '<STR_LIT>' % ( prox , distance ) ) <EOL> q . append ( '<STR_LIT:)>' ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> q . append ( '<STR_LIT>' % d [ '<STR_LIT>' ] ) <EOL> if d . get ( '<STR_LIT>' , None ) : <EOL> q . append ( '<STR_LIT>' % ( int ( d [ '<STR_LIT>' ] ) , int ( d [ '<STR_LIT>' ] ) ) ) <EOL> facet_params = { '<STR_LIT>' : '<STR_LIT:true>' , '<STR_LIT>' : [ '<STR_LIT:state>' , '<STR_LIT>' ] , <EOL> '<STR_LIT>' : '<STR_LIT>' , <EOL> '<STR_LIT>' : date1 , <EOL> '<STR_LIT>' : date2 , <EOL> '<STR_LIT>' : gap , '<STR_LIT>' : <NUM_LIT:1> } <EOL> return '<STR_LIT:U+0020>' . join ( q ) , facet_params <EOL> def query_join ( values , field , and_clause = False ) : <EOL> """<STR_LIT>""" <EOL> if not isinstance ( values , list ) : <EOL> values = [ values ] <EOL> values = [ solr_escape ( v ) for v in values ] <EOL> values = [ '<STR_LIT>' % v for v in values ] <EOL> if and_clause : <EOL> values = [ "<STR_LIT>" % v for v in values ] <EOL> if field . startswith ( "<STR_LIT>" ) : <EOL> if field == "<STR_LIT>" : <EOL> return "<STR_LIT>" % ( field , '<STR_LIT:U+0020>' . join ( values ) ) <EOL> else : <EOL> return "<STR_LIT>" % ( field , '<STR_LIT:U+0020>' . join ( values ) ) <EOL> else : <EOL> return "<STR_LIT>" % ( field , '<STR_LIT:U+0020>' . join ( values ) ) <EOL> def find_words ( s ) : <EOL> ems = re . findall ( '<STR_LIT>' , s ) <EOL> words = map ( lambda em : em [ <NUM_LIT:4> : - <NUM_LIT:5> ] , ems ) <EOL> return words <EOL> def index_titles ( since = None ) : <EOL> """<STR_LIT>""" <EOL> cursor = connection . cursor ( ) <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> if since : <EOL> cursor . execute ( "<STR_LIT>" % since ) <EOL> else : <EOL> solr . delete_query ( '<STR_LIT>' ) <EOL> cursor . execute ( "<STR_LIT>" ) <EOL> count = <NUM_LIT:0> <EOL> while True : <EOL> row = cursor . fetchone ( ) <EOL> if row == None : <EOL> break <EOL> title = models . Title . objects . get ( lccn = row [ <NUM_LIT:0> ] ) <EOL> index_title ( title , solr ) <EOL> count += <NUM_LIT:1> <EOL> if count % <NUM_LIT:100> == <NUM_LIT:0> : <EOL> _log . info ( "<STR_LIT>" % count ) <EOL> reset_queries ( ) <EOL> solr . commit ( ) <EOL> solr . commit ( ) <EOL> def index_title ( title , solr = None ) : <EOL> if solr == None : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> _log . info ( "<STR_LIT>" % title . lccn ) <EOL> try : <EOL> solr . add ( ** title . solr_doc ) <EOL> except Exception , e : <EOL> _log . exception ( e ) <EOL> def delete_title ( title ) : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> q = '<STR_LIT>' % title . solr_doc [ '<STR_LIT:id>' ] <EOL> r = solr . delete_query ( q ) <EOL> _log . info ( "<STR_LIT>" % title ) <EOL> def index_pages ( ) : <EOL> """<STR_LIT>""" <EOL> _log = logging . getLogger ( __name__ ) <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> solr . delete_query ( '<STR_LIT>' ) <EOL> cursor = connection . cursor ( ) <EOL> cursor . execute ( "<STR_LIT>" ) <EOL> count = <NUM_LIT:0> <EOL> while True : <EOL> row = cursor . fetchone ( ) <EOL> if row == None : <EOL> break <EOL> page = models . Page . objects . get ( id = row [ <NUM_LIT:0> ] ) <EOL> _log . info ( "<STR_LIT>" % ( count , page . url ) ) <EOL> solr . add ( ** page . solr_doc ) <EOL> count += <NUM_LIT:1> <EOL> if count % <NUM_LIT:100> == <NUM_LIT:0> : <EOL> reset_queries ( ) <EOL> solr . commit ( ) <EOL> def word_matches_for_page ( page_id , words ) : <EOL> """<STR_LIT>""" <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> if not isinstance ( page_id , str ) : <EOL> page_id = str ( page_id ) <EOL> ocr_list = [ '<STR_LIT>' , ] <EOL> ocr_list . extend ( [ '<STR_LIT>' % l for l in settings . SOLR_LANGUAGES ] ) <EOL> ocrs = '<STR_LIT>' . join ( [ query_join ( words , o ) for o in ocr_list ] ) <EOL> q = '<STR_LIT>' % ( page_id , ocrs ) <EOL> params = { "<STR_LIT>" : <NUM_LIT:100> , "<STR_LIT>" : '<STR_LIT:true>' , "<STR_LIT>" : '<STR_LIT>' } <EOL> response = solr . query ( q , fields = [ '<STR_LIT:id>' ] , highlight = ocr_list , ** params ) <EOL> if not response . highlighting . has_key ( page_id ) : <EOL> return [ ] <EOL> words = set ( ) <EOL> for ocr in ocr_list : <EOL> if ocr in response . highlighting [ page_id ] : <EOL> for context in response . highlighting [ page_id ] [ ocr ] : <EOL> words . update ( find_words ( context ) ) <EOL> return list ( words ) <EOL> def commit ( ) : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> solr . commit ( ) <EOL> def _get_sort ( sort , in_pages = False ) : <EOL> sort_field = sort_order = None <EOL> if sort == '<STR_LIT:state>' : <EOL> sort_field = '<STR_LIT>' <EOL> sort_order = '<STR_LIT>' <EOL> elif sort == '<STR_LIT:title>' : <EOL> sort_field = '<STR_LIT>' <EOL> sort_order = '<STR_LIT>' <EOL> elif sort == '<STR_LIT:date>' and in_pages : <EOL> sort_field = '<STR_LIT:date>' <EOL> sort_order = '<STR_LIT>' <EOL> elif sort == '<STR_LIT:date>' : <EOL> sort_field = '<STR_LIT>' <EOL> sort_order = '<STR_LIT>' <EOL> return sort_field , sort_order <EOL> def _expand_ethnicity ( e ) : <EOL> """<STR_LIT>""" <EOL> parts = [ '<STR_LIT>' % e ] <EOL> ethnicity = models . Ethnicity . objects . get ( name = e ) <EOL> for s in ethnicity . synonyms . all ( ) : <EOL> parts . append ( '<STR_LIT>' % s . synonym ) <EOL> q = '<STR_LIT>' . join ( parts ) <EOL> return "<STR_LIT:(>" + q + "<STR_LIT:)>" <EOL> def _solrize_date ( d , is_start = True ) : <EOL> """<STR_LIT>""" <EOL> d = d . strip ( ) <EOL> match = re . match ( r'<STR_LIT>' , d ) <EOL> if match : <EOL> m , d , y = match . groups ( ) <EOL> else : <EOL> match = re . match ( r'<STR_LIT>' , d ) <EOL> if match : <EOL> m , y = match . groups ( ) <EOL> if is_start : <EOL> d = '<STR_LIT>' <EOL> else : <EOL> d = '<STR_LIT>' <EOL> else : <EOL> match = re . match ( '<STR_LIT>' , d ) <EOL> if match : <EOL> y = match . group ( <NUM_LIT:1> ) <EOL> if is_start : <EOL> m , d = '<STR_LIT>' , '<STR_LIT>' <EOL> else : <EOL> m , d = '<STR_LIT>' , '<STR_LIT>' <EOL> else : <EOL> return None <EOL> if y and m and d : <EOL> return int ( y ) * <NUM_LIT> + int ( m ) * <NUM_LIT:100> + int ( d ) <EOL> else : <EOL> return None <EOL> def similar_pages ( page ) : <EOL> solr = SolrConnection ( settings . SOLR ) <EOL> d = page . issue . date_issued <EOL> year , month , day = '<STR_LIT>' . format ( d . year ) , '<STR_LIT>' . format ( d . month ) , '<STR_LIT>' . format ( d . day ) <EOL> date = '<STR_LIT>' . join ( map ( str , ( year , month , day ) ) ) <EOL> query = '<STR_LIT>' % ( date , query_join ( map ( lambda p : p . city , <EOL> page . issue . title . places . all ( ) ) , '<STR_LIT>' ) , page . issue . title . lccn ) <EOL> response = solr . query ( query , rows = <NUM_LIT> ) <EOL> results = response . results <EOL> return map ( lambda kwargs : utils . get_page ( ** kwargs ) , <EOL> map ( lambda r : urlresolvers . resolve ( r [ '<STR_LIT:id>' ] ) . kwargs , results ) ) </s>
