<s> import simplejson <EOL> import urllib <EOL> import random <EOL> import csv <EOL> import zipfile <EOL> import tempfile <EOL> import urllib2 <EOL> import os . path <EOL> import re <EOL> from datetime import timedelta <EOL> from django . db import transaction , reset_queries <EOL> from django . conf import settings <EOL> from django . contrib . gis . geos import Point <EOL> from django . utils . translation import ugettext_noop <EOL> from django . utils . translation import ugettext as _ <EOL> from molly . apps . places . providers import BaseMapsProvider <EOL> from molly . apps . places . models import Entity , EntityType , Source , EntityTypeCategory <EOL> from molly . utils . i18n import override , set_name_in_language <EOL> from molly . conf . provider import task <EOL> class PostcodesMapsProvider ( BaseMapsProvider ) : <EOL> def __init__ ( self , codepoint_path , import_areas = None ) : <EOL> self . codepoint_path = codepoint_path <EOL> self . import_areas = import_areas <EOL> CODEPOINT_OPEN_URL = '<STR_LIT>' <EOL> MODULE_NAME = "<STR_LIT>" <EOL> def _download_codepoint_open ( self ) : <EOL> archive_url = urllib2 . urlopen ( self . CODEPOINT_OPEN_URL ) <EOL> archive_file = open ( self . codepoint_path , '<STR_LIT:wb>' ) <EOL> archive_file . write ( archive_url . read ( ) ) <EOL> archive_file . close ( ) <EOL> @ task ( run_every = timedelta ( days = <NUM_LIT> ) ) <EOL> def import_data ( self , ** metadata ) : <EOL> entity_type , source = self . _get_entity_type ( ) , self . _get_source ( ) <EOL> if not os . path . exists ( self . codepoint_path ) : <EOL> self . _download_codepoint_open ( ) <EOL> try : <EOL> archive = zipfile . ZipFile ( self . codepoint_path ) <EOL> except zipfile . BadZipfile : <EOL> self . _download_codepoint_open ( ) <EOL> archive = zipfile . ZipFile ( self . codepoint_path ) <EOL> if self . import_areas : <EOL> filenames = [ '<STR_LIT>' % code . lower ( ) for code in self . import_areas ] <EOL> else : <EOL> filenames = [ path for path in archive . namelist ( ) if re . match ( r'<STR_LIT>' , path ) ] <EOL> for filename in filenames : <EOL> reset_queries ( ) <EOL> with transaction . commit_on_success ( ) : <EOL> if hasattr ( archive , '<STR_LIT>' ) : <EOL> f = archive . open ( filename ) <EOL> else : <EOL> f = tempfile . TemporaryFile ( ) <EOL> f . write ( archive . read ( filename ) ) <EOL> f . seek ( <NUM_LIT:0> ) <EOL> reader = csv . reader ( f ) <EOL> self . _load_from_csv ( reader , entity_type , source ) <EOL> del f <EOL> def _load_from_csv ( self , reader , entity_type , source ) : <EOL> j = <NUM_LIT:0> <EOL> for i , line in enumerate ( reader ) : <EOL> postcode_abbrev , ( easting , northing ) = line [ <NUM_LIT:0> ] , line [ <NUM_LIT:10> : <NUM_LIT:12> ] <EOL> postcode_abbrev = postcode_abbrev . replace ( '<STR_LIT:U+0020>' , '<STR_LIT>' ) <EOL> if re . match ( r'<STR_LIT>' , postcode_abbrev ) : <EOL> postcode = '<STR_LIT>' % ( postcode_abbrev [ : <NUM_LIT:2> ] , postcode_abbrev [ <NUM_LIT:2> : ] ) <EOL> elif re . match ( r'<STR_LIT>' , postcode_abbrev ) : <EOL> postcode = '<STR_LIT>' % ( postcode_abbrev [ : <NUM_LIT:3> ] , postcode_abbrev [ <NUM_LIT:3> : ] ) <EOL> elif re . match ( r'<STR_LIT>' , postcode_abbrev ) : <EOL> postcode = '<STR_LIT>' % ( postcode_abbrev [ : <NUM_LIT:3> ] , postcode_abbrev [ <NUM_LIT:3> : ] ) <EOL> elif re . match ( r'<STR_LIT>' , postcode_abbrev ) : <EOL> postcode = '<STR_LIT>' % ( postcode_abbrev [ : <NUM_LIT:4> ] , postcode_abbrev [ <NUM_LIT:4> : ] ) <EOL> elif re . match ( r'<STR_LIT>' , postcode_abbrev ) : <EOL> postcode = '<STR_LIT>' % ( postcode_abbrev [ : <NUM_LIT:3> ] , postcode_abbrev [ <NUM_LIT:3> : ] ) <EOL> elif re . match ( r'<STR_LIT>' , postcode_abbrev ) : <EOL> postcode = '<STR_LIT>' % ( postcode_abbrev [ : <NUM_LIT:4> ] , postcode_abbrev [ <NUM_LIT:4> : ] ) <EOL> else : <EOL> postcode = postcode_abbrev <EOL> try : <EOL> easting , northing = int ( easting ) , int ( northing ) <EOL> except ValueError : <EOL> continue <EOL> j += <NUM_LIT:1> <EOL> try : <EOL> entity = Entity . objects . get ( source = source , _identifiers__scheme = '<STR_LIT>' , _identifiers__value = postcode_abbrev ) <EOL> except Entity . DoesNotExist : <EOL> entity = Entity ( source = source ) <EOL> entity . location = Point ( easting , northing , srid = <NUM_LIT> ) <EOL> entity . geometry = entity . location <EOL> entity . primary_type = entity_type <EOL> identifiers = { <EOL> '<STR_LIT>' : postcode_abbrev , <EOL> '<STR_LIT>' : postcode , <EOL> } <EOL> entity . save ( identifiers = identifiers ) <EOL> set_name_in_language ( entity , '<STR_LIT>' , <EOL> title = postcode ) <EOL> entity . all_types . add ( entity_type ) <EOL> entity . update_all_types_completion ( ) <EOL> def _get_entity_type ( self ) : <EOL> category , created = EntityTypeCategory . objects . get_or_create ( name = ugettext_noop ( '<STR_LIT>' ) ) <EOL> verbose_names = { } <EOL> for lang_code , lang_name in settings . LANGUAGES : <EOL> verbose_names [ lang_code ] = ( _ ( '<STR_LIT>' ) , _ ( '<STR_LIT>' ) , _ ( '<STR_LIT>' ) ) <EOL> defaults = { '<STR_LIT>' : verbose_names } <EOL> entity_type , created = EntityType . objects . get_or_create ( <EOL> slug = '<STR_LIT>' , category = category , defaults = defaults ) <EOL> if created : <EOL> entity_type . show_in_nearby_list = False <EOL> entity_type . show_in_category_list = False <EOL> entity_type . save ( ) <EOL> return entity_type <EOL> def _get_source ( self ) : <EOL> try : <EOL> source = Source . objects . get ( module_name = self . MODULE_NAME ) <EOL> except Source . DoesNotExist : <EOL> source = Source ( module_name = self . MODULE_NAME ) <EOL> source . name = "<STR_LIT>" <EOL> source . save ( ) <EOL> return source </s>
