<s> import random <EOL> import sys <EOL> from collections import namedtuple , defaultdict <EOL> from datetime import datetime , date , time , timedelta <EOL> from logging import getLogger <EOL> from operator import itemgetter <EOL> from StringIO import StringIO <EOL> from urllib2 import urlopen <EOL> from zipfile import ZipFile <EOL> from django . http import Http404 <EOL> from django . db import transaction , reset_queries <EOL> from molly . apps . places import EntityCache <EOL> from molly . apps . places . models import ( Entity , Route , StopOnRoute , Source , <EOL> Journey , ScheduledStop ) <EOL> from molly . apps . places . providers import BaseMapsProvider , NaptanMapsProvider <EOL> from molly . conf . provider import task <EOL> from molly . utils . i18n import set_name_in_language <EOL> logger = getLogger ( __name__ ) <EOL> weekbool = namedtuple ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> class AtcoCifTimetableProvider ( BaseMapsProvider ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , url ) : <EOL> """<STR_LIT>""" <EOL> self . _url = url <EOL> self . _cache = EntityCache ( ) <EOL> self . _entity_type = NaptanMapsProvider ( None ) . _get_entity_types ( ) [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> @ task ( run_every = timedelta ( days = <NUM_LIT:7> ) ) <EOL> def import_data ( self , ** metadata ) : <EOL> deleted_routes = set ( Route . objects . filter ( external_ref__startswith = self . _url ) . values_list ( '<STR_LIT>' ) ) <EOL> archive = ZipFile ( StringIO ( urlopen ( self . _url ) . read ( ) ) ) <EOL> for file in archive . namelist ( ) : <EOL> logger . info ( file ) <EOL> routes = self . _import_cif ( archive . open ( file ) ) <EOL> logger . info ( '<STR_LIT>' % len ( routes ) ) <EOL> self . _import_routes ( routes ) <EOL> deleted_routes -= set ( self . _url + route [ '<STR_LIT:id>' ] for route in routes ) <EOL> archive . close ( ) <EOL> for route in deleted_routes : <EOL> Route . objects . filter ( external_ref = route ) . delete ( ) <EOL> def _parse_cif_date ( self , datestring ) : <EOL> return date ( int ( datestring [ <NUM_LIT:0> : <NUM_LIT:4> ] ) , int ( datestring [ <NUM_LIT:4> : <NUM_LIT:6> ] ) , <EOL> int ( datestring [ <NUM_LIT:6> : <NUM_LIT:8> ] ) ) <EOL> def _parse_cif_time ( self , timestring ) : <EOL> return time ( int ( timestring [ <NUM_LIT:0> : <NUM_LIT:2> ] ) % <NUM_LIT> , int ( timestring [ <NUM_LIT:2> : <NUM_LIT:4> ] ) ) <EOL> @ transaction . commit_on_success <EOL> def _import_cif ( self , cif ) : <EOL> """<STR_LIT>""" <EOL> self . _cache = EntityCache ( ) <EOL> reset_queries ( ) <EOL> routes = [ ] <EOL> this_journey = None <EOL> for line in cif : <EOL> if line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> if this_journey is not None : <EOL> routes [ - <NUM_LIT:1> ] [ '<STR_LIT>' ] . append ( this_journey ) <EOL> if line [ <NUM_LIT:2> ] == '<STR_LIT:D>' : <EOL> this_journey = None <EOL> continue <EOL> this_journey = { <EOL> '<STR_LIT>' : line [ <NUM_LIT:3> : <NUM_LIT:7> ] , <EOL> '<STR_LIT:id>' : line [ <NUM_LIT:7> : <NUM_LIT> ] , <EOL> '<STR_LIT>' : self . _parse_cif_date ( line [ <NUM_LIT> : <NUM_LIT> ] ) , <EOL> '<STR_LIT>' : self . _parse_cif_date ( line [ <NUM_LIT> : <NUM_LIT> ] ) , <EOL> '<STR_LIT>' : weekbool ( <EOL> line [ <NUM_LIT> ] == '<STR_LIT:1>' , <EOL> line [ <NUM_LIT:30> ] == '<STR_LIT:1>' , <EOL> line [ <NUM_LIT> ] == '<STR_LIT:1>' , <EOL> line [ <NUM_LIT:32> ] == '<STR_LIT:1>' , <EOL> line [ <NUM_LIT> ] == '<STR_LIT:1>' , <EOL> line [ <NUM_LIT> ] == '<STR_LIT:1>' , <EOL> line [ <NUM_LIT> ] == '<STR_LIT:1>' , <EOL> ) , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:S>' : '<STR_LIT>' , <EOL> '<STR_LIT:H>' : '<STR_LIT>' <EOL> } . get ( line [ <NUM_LIT> ] , '<STR_LIT:all>' ) , <EOL> '<STR_LIT>' : { <EOL> '<STR_LIT:A>' : '<STR_LIT>' , <EOL> '<STR_LIT:B>' : '<STR_LIT>' , <EOL> '<STR_LIT:X>' : '<STR_LIT>' <EOL> } . get ( line [ <NUM_LIT> ] , '<STR_LIT:all>' ) , <EOL> '<STR_LIT>' : line [ <NUM_LIT> : <NUM_LIT> ] , <EOL> '<STR_LIT>' : line [ <NUM_LIT> : <NUM_LIT> ] . strip ( ) , <EOL> '<STR_LIT>' : line [ <NUM_LIT:64> ] , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : [ ] , <EOL> } <EOL> elif line [ : <NUM_LIT:2> ] in ( '<STR_LIT>' , '<STR_LIT>' ) : <EOL> this_journey [ '<STR_LIT>' ] . append ( line [ <NUM_LIT:7> : ] ) <EOL> elif line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> try : <EOL> this_journey [ '<STR_LIT>' ] . append ( { <EOL> '<STR_LIT>' : self . _cache [ '<STR_LIT>' % line [ <NUM_LIT:2> : <NUM_LIT> ] . strip ( ) ] , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : self . _parse_cif_time ( line [ <NUM_LIT> : <NUM_LIT> ] ) , <EOL> '<STR_LIT>' : '<STR_LIT:O>' , <EOL> '<STR_LIT>' : line [ <NUM_LIT> ] == '<STR_LIT:0>' , <EOL> '<STR_LIT>' : line [ <NUM_LIT> ] == '<STR_LIT:1>' <EOL> } ) <EOL> except Http404 : <EOL> pass <EOL> elif line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> try : <EOL> this_journey [ '<STR_LIT>' ] . append ( { <EOL> '<STR_LIT>' : self . _cache [ '<STR_LIT>' % line [ <NUM_LIT:2> : <NUM_LIT> ] . strip ( ) ] , <EOL> '<STR_LIT>' : self . _parse_cif_time ( line [ <NUM_LIT> : <NUM_LIT> ] ) , <EOL> '<STR_LIT>' : self . _parse_cif_time ( line [ <NUM_LIT> : <NUM_LIT> ] ) , <EOL> '<STR_LIT>' : line [ <NUM_LIT> ] , <EOL> '<STR_LIT>' : line [ <NUM_LIT> ] == '<STR_LIT:0>' , <EOL> '<STR_LIT>' : line [ <NUM_LIT> ] == '<STR_LIT:1>' <EOL> } ) <EOL> except Http404 : <EOL> pass <EOL> elif line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> try : <EOL> this_journey [ '<STR_LIT>' ] . append ( { <EOL> '<STR_LIT>' : self . _cache [ '<STR_LIT>' % line [ <NUM_LIT:2> : <NUM_LIT> ] . strip ( ) ] , <EOL> '<STR_LIT>' : self . _parse_cif_time ( line [ <NUM_LIT> : <NUM_LIT> ] ) , <EOL> '<STR_LIT>' : None , <EOL> '<STR_LIT>' : '<STR_LIT:F>' , <EOL> '<STR_LIT>' : line [ <NUM_LIT> ] == '<STR_LIT:0>' , <EOL> '<STR_LIT>' : line [ <NUM_LIT> ] == '<STR_LIT:1>' <EOL> } ) <EOL> except Http404 : <EOL> pass <EOL> elif line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> route_id = line [ <NUM_LIT:2> : ] <EOL> elif line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> route_id += line [ <NUM_LIT> : - <NUM_LIT:1> ] <EOL> elif line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> if this_journey is not None : <EOL> routes [ - <NUM_LIT:1> ] [ '<STR_LIT>' ] . append ( this_journey ) <EOL> routes . append ( { <EOL> '<STR_LIT:id>' : route_id , <EOL> '<STR_LIT>' : line [ <NUM_LIT:10> : <NUM_LIT> ] . strip ( ) , <EOL> '<STR_LIT:description>' : line [ <NUM_LIT> : - <NUM_LIT:1> ] , <EOL> '<STR_LIT>' : [ ] , <EOL> '<STR_LIT>' : [ ] <EOL> } ) <EOL> elif line [ : <NUM_LIT:2> ] == '<STR_LIT>' : <EOL> stop_code = line [ <NUM_LIT:3> : <NUM_LIT:15> ] . strip ( ) <EOL> try : <EOL> entity = self . _cache [ '<STR_LIT>' % stop_code ] <EOL> if entity . source == self . _get_source ( ) : <EOL> raise Http404 ( ) <EOL> except Http404 : <EOL> try : <EOL> entity = Entity . objects . get ( source = self . _get_source ( ) , <EOL> _identifiers__scheme = '<STR_LIT>' , <EOL> _identifiers__value = stop_code ) <EOL> except Entity . DoesNotExist : <EOL> entity = Entity ( source = self . _get_source ( ) ) <EOL> identifiers = { '<STR_LIT>' : stop_code } <EOL> entity_type = self . _entity_type <EOL> entity . primary_type = entity_type <EOL> entity . save ( identifiers = identifiers ) <EOL> set_name_in_language ( entity , '<STR_LIT>' , title = line [ <NUM_LIT:15> : <NUM_LIT> ] . strip ( ) ) <EOL> entity . all_types = ( entity_type , ) <EOL> entity . update_all_types_completion ( ) <EOL> entity . save ( ) <EOL> routes [ - <NUM_LIT:1> ] [ '<STR_LIT>' ] . append ( entity ) <EOL> if this_journey is not None : <EOL> routes [ - <NUM_LIT:1> ] [ '<STR_LIT>' ] . append ( this_journey ) <EOL> return routes <EOL> def _import_routes ( self , routes ) : <EOL> for r in routes : <EOL> route , created = Route . objects . get_or_create ( <EOL> external_ref = self . _url + r [ '<STR_LIT:id>' ] , <EOL> defaults = { <EOL> '<STR_LIT>' : r [ '<STR_LIT>' ] , <EOL> '<STR_LIT>' : r [ '<STR_LIT:description>' ] , <EOL> } <EOL> ) <EOL> if not created : <EOL> route . service_id = r [ '<STR_LIT>' ] <EOL> route . service_name = r [ '<STR_LIT:description>' ] <EOL> route . save ( ) <EOL> route . stops . clear ( ) <EOL> for i , stop in enumerate ( r [ '<STR_LIT>' ] ) : <EOL> StopOnRoute . objects . create ( route = route , entity = stop , order = i ) <EOL> route . journey_set . all ( ) . delete ( ) <EOL> for journey in r [ '<STR_LIT>' ] : <EOL> self . _add_journey ( route , journey ) <EOL> def _add_journey ( self , route , journey ) : <EOL> j = Journey . objects . create ( <EOL> route = route , <EOL> external_ref = self . _url + journey [ '<STR_LIT:id>' ] , <EOL> notes = '<STR_LIT:\n>' . join ( journey [ '<STR_LIT>' ] ) , <EOL> runs_on_monday = journey [ '<STR_LIT>' ] . mon , <EOL> runs_on_tuesday = journey [ '<STR_LIT>' ] . tue , <EOL> runs_on_wednesday = journey [ '<STR_LIT>' ] . wed , <EOL> runs_on_thursday = journey [ '<STR_LIT>' ] . thu , <EOL> runs_on_friday = journey [ '<STR_LIT>' ] . fri , <EOL> runs_on_saturday = journey [ '<STR_LIT>' ] . sat , <EOL> runs_on_sunday = journey [ '<STR_LIT>' ] . sun , <EOL> runs_in_termtime = journey [ '<STR_LIT>' ] in ( '<STR_LIT:all>' , '<STR_LIT>' ) , <EOL> runs_in_school_holidays = journey [ '<STR_LIT>' ] in ( '<STR_LIT:all>' , '<STR_LIT>' ) , <EOL> runs_on_bank_holidays = journey [ '<STR_LIT>' ] in ( '<STR_LIT:all>' , '<STR_LIT>' , '<STR_LIT>' ) , <EOL> runs_on_non_bank_holidays = journey [ '<STR_LIT>' ] in ( '<STR_LIT:all>' , '<STR_LIT>' ) , <EOL> runs_from = journey [ '<STR_LIT>' ] , <EOL> runs_until = journey [ '<STR_LIT>' ] , <EOL> vehicle = journey [ '<STR_LIT>' ] ) <EOL> for i , stop in enumerate ( journey [ '<STR_LIT>' ] ) : <EOL> ScheduledStop . objects . create ( <EOL> journey = j , <EOL> entity = stop [ '<STR_LIT>' ] , <EOL> order = i , <EOL> sta = stop [ '<STR_LIT>' ] , <EOL> std = stop [ '<STR_LIT>' ] , <EOL> times_estimated = stop [ '<STR_LIT>' ] , <EOL> fare_stage = stop [ '<STR_LIT>' ] , <EOL> activity = stop [ '<STR_LIT>' ] <EOL> ) <EOL> def _get_source ( self ) : <EOL> source , created = Source . objects . get_or_create ( module_name = __name__ , <EOL> name = '<STR_LIT>' ) <EOL> return source <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> AtcoCifTimetableProvider ( '<STR_LIT>' ) . import_data ( { } ) </s>
