<s> import json <EOL> import os <EOL> import shutil <EOL> import sys <EOL> import tempfile <EOL> import traceback <EOL> import copy <EOL> import math <EOL> import requests <EOL> from girder . constants import AccessType <EOL> from girder . utility import config <EOL> from girder . utility . model_importer import ModelImporter <EOL> from girder . plugins . jobs . constants import JobStatus <EOL> from girder . plugins . minerva . utility . bsve . bsve_utility import BsveUtility <EOL> from girder . plugins . minerva . utility . minerva_utility import mM <EOL> import girder_client <EOL> states_source = ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> ) <EOL> _states = None <EOL> """<STR_LIT>""" <EOL> ignored_properties = set ( [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] ) <EOL> def accumulate ( data ) : <EOL> """<STR_LIT>""" <EOL> global _states <EOL> if _states is None : <EOL> _states = requests . get ( states_source ) . json ( ) <EOL> geojson = copy . deepcopy ( _states ) <EOL> states = { } <EOL> for feature in geojson . get ( '<STR_LIT>' , [ ] ) : <EOL> abbr = feature [ '<STR_LIT>' ] [ '<STR_LIT>' ] . upper ( ) <EOL> name = feature [ '<STR_LIT>' ] [ '<STR_LIT:name>' ] . upper ( ) <EOL> feature [ '<STR_LIT>' ] . clear ( ) <EOL> feature [ '<STR_LIT>' ] [ '<STR_LIT>' ] = abbr <EOL> feature [ '<STR_LIT>' ] [ '<STR_LIT:name>' ] = name <EOL> states [ name ] = feature [ '<STR_LIT>' ] <EOL> all_props = set ( ) <EOL> for record in data : <EOL> m = ( record . get ( '<STR_LIT>' , { } ) . get ( '<STR_LIT>' , { } ) <EOL> . get ( '<STR_LIT:state>' , '<STR_LIT>' ) . upper ( ) ) <EOL> if m in states : <EOL> props = states [ m ] <EOL> props [ '<STR_LIT>' ] = props . get ( '<STR_LIT>' , <NUM_LIT:0> ) + <NUM_LIT:1> <EOL> for key , value in record . iteritems ( ) : <EOL> if key in ignored_properties : <EOL> continue <EOL> try : <EOL> v = float ( value ) <EOL> except Exception : <EOL> v = float ( '<STR_LIT>' ) <EOL> if math . isnan ( v ) or math . isinf ( v ) : <EOL> continue <EOL> if v < <NUM_LIT:0> : <EOL> print ( '<STR_LIT>' ) <EOL> props [ key ] = props . get ( key , <NUM_LIT:0> ) + v <EOL> all_props . add ( key ) <EOL> geojson [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : list ( all_props ) <EOL> } <EOL> return geojson <EOL> def run ( job ) : <EOL> job_model = ModelImporter . model ( '<STR_LIT>' , '<STR_LIT>' ) <EOL> job_model . updateJob ( job , status = JobStatus . RUNNING ) <EOL> try : <EOL> configFile = os . path . join ( os . path . dirname ( __file__ ) , "<STR_LIT>" ) <EOL> if os . path . exists ( configFile ) : <EOL> bsveConfig = json . load ( open ( configFile ) ) [ '<STR_LIT>' ] <EOL> else : <EOL> bsveConfig = { } <EOL> kwargs = job [ '<STR_LIT>' ] <EOL> datasetId = str ( kwargs [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> token = kwargs [ '<STR_LIT>' ] <EOL> params = kwargs [ '<STR_LIT>' ] <EOL> bsveUtility = BsveUtility ( <EOL> user = bsveConfig . get ( <EOL> '<STR_LIT>' , os . environ . get ( '<STR_LIT>' ) ) , <EOL> apikey = bsveConfig . get ( <EOL> '<STR_LIT>' , os . environ . get ( '<STR_LIT>' ) ) , <EOL> secret = bsveConfig . get ( <EOL> '<STR_LIT>' , os . environ . get ( '<STR_LIT>' ) ) , <EOL> base = bsveConfig . get ( '<STR_LIT>' ) <EOL> ) <EOL> data = accumulate ( bsveUtility . soda_dump ( count = params [ '<STR_LIT:count>' ] ) ) <EOL> tmpdir = tempfile . mkdtemp ( ) <EOL> outFilepath = tempfile . mkstemp ( suffix = '<STR_LIT>' , dir = tmpdir ) [ <NUM_LIT:1> ] <EOL> writer = open ( outFilepath , '<STR_LIT:w>' ) <EOL> writer . write ( json . dumps ( data ) ) <EOL> writer . close ( ) <EOL> outFilename = '<STR_LIT>' <EOL> humanFilepath = os . path . join ( tmpdir , outFilename ) <EOL> shutil . move ( outFilepath , humanFilepath ) <EOL> girderPort = config . getConfig ( ) [ '<STR_LIT>' ] <EOL> client = girder_client . GirderClient ( port = girderPort ) <EOL> client . token = token [ '<STR_LIT>' ] <EOL> client . uploadFileToItem ( datasetId , humanFilepath ) <EOL> user_model = ModelImporter . model ( '<STR_LIT:user>' ) <EOL> user = user_model . load ( job [ '<STR_LIT>' ] , force = True ) <EOL> item_model = ModelImporter . model ( '<STR_LIT>' ) <EOL> dataset = item_model . load ( datasetId , level = AccessType . WRITE , user = user ) <EOL> minerva_metadata = mM ( dataset ) <EOL> file_model = ModelImporter . model ( '<STR_LIT:file>' ) <EOL> existing = file_model . findOne ( { <EOL> '<STR_LIT>' : dataset [ '<STR_LIT>' ] , <EOL> '<STR_LIT:name>' : outFilename <EOL> } ) <EOL> if not existing : <EOL> raise ( Exception ( '<STR_LIT>' % <EOL> ( outFilename , datasetId ) ) ) <EOL> shutil . rmtree ( tmpdir ) <EOL> minerva_metadata [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> minerva_metadata [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> if data [ '<STR_LIT>' ] [ '<STR_LIT>' ] : <EOL> minerva_metadata [ '<STR_LIT>' ] = data [ '<STR_LIT>' ] [ '<STR_LIT>' ] [ <NUM_LIT:0> ] <EOL> minerva_metadata [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> mM ( dataset , minerva_metadata ) <EOL> existing = file_model . findOne ( { <EOL> '<STR_LIT>' : dataset [ '<STR_LIT>' ] , <EOL> '<STR_LIT:name>' : outFilename <EOL> } ) <EOL> if existing : <EOL> minerva_metadata [ '<STR_LIT>' ] = { <EOL> '<STR_LIT>' : existing [ '<STR_LIT>' ] , <EOL> '<STR_LIT:name>' : outFilename <EOL> } <EOL> else : <EOL> raise ( Exception ( '<STR_LIT>' % <EOL> ( outFilename , datasetId ) ) ) <EOL> mM ( dataset , minerva_metadata ) <EOL> job_model . updateJob ( job , status = JobStatus . SUCCESS ) <EOL> except Exception : <EOL> t , val , tb = sys . exc_info ( ) <EOL> log = '<STR_LIT>' % ( t . __name__ , repr ( val ) , traceback . extract_tb ( tb ) ) <EOL> job_model . updateJob ( job , status = JobStatus . ERROR , log = log ) <EOL> raise </s>
