<s> """<STR_LIT>""" <EOL> import dbio <EOL> from datetime import timedelta <EOL> import datasets <EOL> import modis <EOL> import requests <EOL> from requests . auth import HTTPDigestAuth <EOL> import tempfile <EOL> import lxml . html <EOL> import subprocess <EOL> import glob <EOL> import shutil <EOL> import numpy as np <EOL> table = "<STR_LIT>" <EOL> username = "<STR_LIT>" <EOL> password = "<STR_LIT>" <EOL> def dates ( dbname ) : <EOL> dts = datasets . dates ( dbname , table ) <EOL> return dts <EOL> def download ( dbname , dts , bbox ) : <EOL> """<STR_LIT>""" <EOL> res = <NUM_LIT> <EOL> tiles = modis . findTiles ( bbox ) <EOL> for dt in [ dts [ <NUM_LIT:0> ] + timedelta ( dti ) for dti in range ( ( dts [ <NUM_LIT:1> ] - dts [ <NUM_LIT:0> ] ) . days + <NUM_LIT:1> ) ] : <EOL> temppath = tempfile . mkdtemp ( ) <EOL> url = "<STR_LIT>" . format ( dt . year , dt . strftime ( "<STR_LIT>" ) ) <EOL> r = requests . get ( url , auth = HTTPDigestAuth ( username , password ) ) <EOL> if r . status_code == <NUM_LIT:200> : <EOL> dom = lxml . html . fromstring ( r . text ) <EOL> links = [ link for link in dom . xpath ( '<STR_LIT>' ) if link . find ( "<STR_LIT>" ) > <NUM_LIT:0> ] <EOL> for t in tiles : <EOL> filenames = filter ( lambda f : f . find ( "<STR_LIT>" . format ( t [ <NUM_LIT:1> ] , t [ <NUM_LIT:0> ] ) ) > <NUM_LIT:0> , links ) <EOL> if len ( filenames ) > <NUM_LIT:0> : <EOL> filename = filenames [ <NUM_LIT:0> ] <EOL> r = requests . get ( "<STR_LIT>" . format ( url , filename ) , auth = HTTPDigestAuth ( username , password ) ) <EOL> with open ( "<STR_LIT>" . format ( temppath , filename ) , '<STR_LIT:wb>' ) as fout : <EOL> fout . write ( r . content ) <EOL> tifs = glob . glob ( "<STR_LIT>" . format ( temppath ) ) <EOL> if len ( tifs ) > <NUM_LIT:0> : <EOL> subprocess . call ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" . format ( temppath ) ] + tifs ) <EOL> cmd = "<STR_LIT:U+0020>" . join ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" . format ( temppath ) , "<STR_LIT>" . format ( <EOL> temppath ) , "<STR_LIT>" , "<STR_LIT>" ] ) <EOL> subprocess . call ( cmd , shell = True ) <EOL> cmd = "<STR_LIT:U+0020>" . join ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" , str ( <EOL> res ) , str ( - res ) , "<STR_LIT>" . format ( temppath ) , "<STR_LIT>" . format ( temppath ) ] ) <EOL> subprocess . call ( cmd , shell = True ) <EOL> if bbox is None : <EOL> pstr = [ ] <EOL> else : <EOL> pstr = [ "<STR_LIT>" , str ( bbox [ <NUM_LIT:0> ] ) , str ( bbox [ <NUM_LIT:3> ] ) , str ( bbox [ <NUM_LIT:2> ] ) , str ( bbox [ <NUM_LIT:1> ] ) ] <EOL> subprocess . call ( [ "<STR_LIT>" , "<STR_LIT>" , "<STR_LIT>" ] + pstr + [ "<STR_LIT>" . format ( temppath ) , "<STR_LIT>" . format ( temppath ) ] ) <EOL> dbio . ingest ( dbname , "<STR_LIT>" . format ( temppath ) , dt , table , False ) <EOL> shutil . rmtree ( temppath ) <EOL> else : <EOL> print ( "<STR_LIT>" . format ( <EOL> dt . strftime ( "<STR_LIT>" ) ) ) <EOL> class Modscag ( object ) : <EOL> def __init__ ( self ) : <EOL> """<STR_LIT>""" <EOL> self . statevar = [ "<STR_LIT>" ] <EOL> self . obsvar = "<STR_LIT>" <EOL> self . res = <NUM_LIT> <EOL> self . stddev = <NUM_LIT> <EOL> self . tablename = table <EOL> def x ( self , dt , models ) : <EOL> """<STR_LIT>""" <EOL> data = { } <EOL> db = dbio . connect ( models . dbname ) <EOL> cur = db . cursor ( ) <EOL> for s in self . statevar : <EOL> sql = "<STR_LIT>" . format ( <EOL> models . name , s , dt . year , dt . month , dt . day ) <EOL> cur . execute ( sql ) <EOL> e , lon , lat , vals = zip ( * cur . fetchall ( ) ) <EOL> gid = [ models [ <NUM_LIT:0> ] . lgid [ ( l [ <NUM_LIT:0> ] , l [ <NUM_LIT:1> ] ) ] for l in zip ( lat , lon ) ] <EOL> nens = max ( e ) <EOL> data [ s ] = np . array ( vals ) . reshape ( ( len ( vals ) / nens , nens ) ) <EOL> lat = np . array ( lat ) . reshape ( ( len ( lat ) / nens , nens ) ) <EOL> lon = np . array ( lon ) . reshape ( ( len ( lon ) / nens , nens ) ) <EOL> gid = np . array ( gid ) . reshape ( ( len ( gid ) / nens , nens ) ) <EOL> cur . close ( ) <EOL> db . close ( ) <EOL> return data , lat , lon , gid <EOL> def get ( self , dt , models ) : <EOL> """<STR_LIT>""" <EOL> db = dbio . connect ( models . dbname ) <EOL> cur = db . cursor ( ) <EOL> sql = "<STR_LIT>" . format ( <EOL> self . tablename , models . name , dt . year , dt . month , dt . day ) <EOL> cur . execute ( sql ) <EOL> if bool ( cur . rowcount ) : <EOL> lon , lat , data = zip ( * cur . fetchall ( ) ) <EOL> data = np . array ( data ) . reshape ( ( len ( data ) , <NUM_LIT:1> ) ) <EOL> lat = np . array ( lat ) . reshape ( ( len ( lat ) , <NUM_LIT:1> ) ) <EOL> lon = np . array ( lon ) . reshape ( ( len ( lon ) , <NUM_LIT:1> ) ) <EOL> self . nobs = len ( data ) <EOL> else : <EOL> data = lat = lon = None <EOL> cur . close ( ) <EOL> db . close ( ) <EOL> return data , lat , lon </s>
