<s> from open_trails import app <EOL> from werkzeug . utils import secure_filename <EOL> from itertools import groupby , count <EOL> from operator import itemgetter <EOL> from StringIO import StringIO <EOL> from tempfile import mkdtemp <EOL> import os , os . path , json , subprocess , zipfile , csv , boto , tempfile , urlparse , urllib , zipfile <EOL> from boto . s3 . key import Key <EOL> from models import Dataset <EOL> from flask import make_response <EOL> def get_dataset ( datastore , id ) : <EOL> '''<STR_LIT>''' <EOL> try : <EOL> valid_path = '<STR_LIT>' . format ( id ) <EOL> valid_file = datastore . read ( valid_path ) <EOL> except AttributeError : <EOL> return None <EOL> if valid_file . read ( ) == id : <EOL> dataset = Dataset ( id ) <EOL> dataset . datastore = datastore <EOL> return dataset <EOL> def allowed_file ( filename ) : <EOL> return '<STR_LIT:.>' in filename and filename . rsplit ( '<STR_LIT:.>' , <NUM_LIT:1> ) [ <NUM_LIT:1> ] in set ( [ '<STR_LIT>' ] ) <EOL> def clean_name ( name ) : <EOL> '''<STR_LIT>''' <EOL> return secure_filename ( name ) . lower ( ) . replace ( "<STR_LIT:_>" , "<STR_LIT:->" ) <EOL> def make_id_from_url ( url ) : <EOL> '''<STR_LIT>''' <EOL> parsed = urlparse . urlparse ( url ) <EOL> steward_id = parsed . netloc . split ( '<STR_LIT:.>' ) [ <NUM_LIT:0> ] <EOL> return secure_filename ( steward_id ) . lower ( ) . replace ( "<STR_LIT:_>" , "<STR_LIT:->" ) <EOL> def unzip ( zipfile_path , search_ext = '<STR_LIT>' , other_exts = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> '''<STR_LIT>''' <EOL> zf = zipfile . ZipFile ( zipfile_path , '<STR_LIT:r>' ) <EOL> dirname = mkdtemp ( prefix = '<STR_LIT>' , suffix = search_ext ) <EOL> foundfile_path = None <EOL> for name in sorted ( zf . namelist ( ) ) : <EOL> base , ( _ , ext ) = os . path . basename ( name ) , os . path . splitext ( name ) <EOL> if base . startswith ( '<STR_LIT:.>' ) : <EOL> continue <EOL> if ext in [ search_ext ] + list ( other_exts ) : <EOL> unzipped_path = '<STR_LIT>' . format ( dirname , base ) <EOL> with open ( unzipped_path , '<STR_LIT:w>' ) as f : <EOL> f . write ( zf . open ( name ) . read ( ) ) <EOL> if ext == search_ext : <EOL> foundfile_path = unzipped_path <EOL> return foundfile_path <EOL> def zip_file ( destination , content , filename ) : <EOL> '''<STR_LIT>''' <EOL> with zipfile . ZipFile ( destination , '<STR_LIT:w>' , zipfile . ZIP_DEFLATED ) as zf : <EOL> zf . writestr ( filename , content ) <EOL> def get_sample_features ( dataset , zipped_geojson_name ) : <EOL> '''<STR_LIT:U+0020>''' <EOL> zip_path = '<STR_LIT>' . format ( dataset . id , zipped_geojson_name ) <EOL> zip_buffer = dataset . datastore . read ( zip_path ) <EOL> zf = zipfile . ZipFile ( zip_buffer , '<STR_LIT:r>' ) <EOL> for name in zf . namelist ( ) : <EOL> _ , ext = os . path . splitext ( name ) <EOL> if ext == '<STR_LIT>' : <EOL> gf = zf . open ( name , '<STR_LIT:r>' ) <EOL> geojson = json . load ( gf ) <EOL> return geojson [ '<STR_LIT>' ] [ : <NUM_LIT:3> ] <EOL> return [ ] <EOL> def get_sample_segment_features ( dataset ) : <EOL> return get_sample_features ( dataset , '<STR_LIT>' ) <EOL> def get_sample_trailhead_features ( dataset ) : <EOL> return get_sample_features ( dataset , '<STR_LIT>' ) <EOL> def get_sample_transformed_segments_features ( dataset ) : <EOL> return get_sample_features ( dataset , '<STR_LIT>' ) <EOL> def get_sample_transformed_trailhead_features ( dataset ) : <EOL> return get_sample_features ( dataset , '<STR_LIT>' ) <EOL> def encode_list ( items ) : <EOL> '''<STR_LIT:U+0020>''' <EOL> return '<STR_LIT>' . join ( map ( str , items ) ) <EOL> def make_named_trails ( segment_features ) : <EOL> '''<STR_LIT:U+0020>''' <EOL> names = [ ( f [ '<STR_LIT>' ] [ '<STR_LIT:name>' ] , f [ '<STR_LIT>' ] [ '<STR_LIT:id>' ] ) <EOL> for f in segment_features <EOL> if f [ '<STR_LIT>' ] [ '<STR_LIT:name>' ] ] <EOL> if not names : <EOL> return [ ] <EOL> names = sorted ( names , key = itemgetter ( <NUM_LIT:0> ) ) <EOL> groups = groupby ( names , itemgetter ( <NUM_LIT:0> ) ) <EOL> name_ids = [ ( name , encode_list ( map ( itemgetter ( <NUM_LIT:1> ) , names_ids ) ) ) <EOL> for ( name , names_ids ) in groups ] <EOL> id_counter = count ( <NUM_LIT:1> ) <EOL> return [ dict ( id = str ( id_counter . next ( ) ) , <EOL> name = name , segment_ids = ids , <EOL> description = None , part_of = None ) <EOL> for ( name , ids ) in name_ids ] <EOL> def package_opentrails_archive ( dataset ) : <EOL> '''<STR_LIT:U+0020>''' <EOL> buffer = StringIO ( ) <EOL> zf = zipfile . ZipFile ( buffer , '<STR_LIT:w>' ) <EOL> ot_prefix = '<STR_LIT>' . format ( dataset . id ) <EOL> try : <EOL> trailheads_zipname = '<STR_LIT>' . format ( ot_prefix ) <EOL> trailheads_zipfile = dataset . datastore . read ( trailheads_zipname ) <EOL> trailheads_path = unzip ( trailheads_zipfile , '<STR_LIT>' , [ ] ) <EOL> zf . writestr ( '<STR_LIT>' , open ( trailheads_path ) . read ( ) ) <EOL> except AttributeError : <EOL> pass <EOL> segments_zipname = '<STR_LIT>' . format ( ot_prefix ) <EOL> segments_zipfile = dataset . datastore . read ( segments_zipname ) <EOL> segments_path = unzip ( segments_zipfile , '<STR_LIT>' , [ ] ) <EOL> zf . writestr ( '<STR_LIT>' , open ( segments_path ) . read ( ) ) <EOL> named_trails_path = '<STR_LIT>' . format ( ot_prefix ) <EOL> named_trails_data = dataset . datastore . read ( named_trails_path ) <EOL> zf . writestr ( '<STR_LIT>' , named_trails_data . read ( ) ) <EOL> stewards_path = '<STR_LIT>' . format ( ot_prefix ) <EOL> stewards_data = dataset . datastore . read ( stewards_path ) <EOL> zf . writestr ( '<STR_LIT>' , stewards_data . read ( ) ) <EOL> zf . close ( ) <EOL> buffer . seek ( <NUM_LIT:0> ) <EOL> return buffer </s>
