<s> '''<STR_LIT>''' <EOL> __author__ = "<STR_LIT>" <EOL> __copyright__ = "<STR_LIT>" <EOL> __license__ = "<STR_LIT>" <EOL> __maintainer__ = "<STR_LIT>" <EOL> __email__ = "<STR_LIT>" <EOL> __status__ = "<STR_LIT>" <EOL> import os <EOL> from pandas import DataFrame , Series , datetime , read_csv , concat <EOL> import numpy as np <EOL> from skspec . core . timespectra import TimeSpectra <EOL> from skspec . core . specindex import SpecIndex <EOL> from skspec . core . file_utils import get_files_in_dir , get_shortname <EOL> import logging <EOL> logger = logging . getLogger ( __name__ ) <EOL> spec_suite_months = { '<STR_LIT>' : <NUM_LIT:1> , <EOL> '<STR_LIT>' : <NUM_LIT:2> , <EOL> '<STR_LIT>' : <NUM_LIT:3> , <EOL> '<STR_LIT>' : <NUM_LIT:4> , <EOL> '<STR_LIT>' : <NUM_LIT:5> , <EOL> '<STR_LIT>' : <NUM_LIT:6> , <EOL> '<STR_LIT>' : <NUM_LIT:7> , <EOL> '<STR_LIT>' : <NUM_LIT:8> , <EOL> '<STR_LIT>' : <NUM_LIT:9> , <EOL> '<STR_LIT>' : <NUM_LIT:10> , <EOL> '<STR_LIT>' : <NUM_LIT:11> , <EOL> '<STR_LIT>' : <NUM_LIT:12> } <EOL> spec_dtype = np . dtype ( [ ( '<STR_LIT>' , float ) , ( '<STR_LIT>' , float ) ] ) <EOL> def get_shortname ( filepath , cut_extension = False ) : <EOL> '''<STR_LIT>''' <EOL> shortname = os . path . basename ( filepath ) <EOL> if cut_extension : <EOL> shortname = os . path . splitext ( shortname ) [ <NUM_LIT:0> ] <EOL> return shortname <EOL> def extract_darkfile ( filelist , return_null = True ) : <EOL> '''<STR_LIT>''' <EOL> darkfiles = [ ] <EOL> for infile in filelist : <EOL> if "<STR_LIT>" in infile . lower ( ) or "<STR_LIT>" in infile . lower ( ) : <EOL> darkfiles . append ( infile ) <EOL> if len ( darkfiles ) == <NUM_LIT:0> : <EOL> if return_null : <EOL> return None <EOL> else : <EOL> raise Warning ( "<STR_LIT>" % ( filelist [ <NUM_LIT:0> ] , filelist [ - <NUM_LIT:1> ] ) ) <EOL> elif len ( darkfiles ) > <NUM_LIT:1> : <EOL> raise Warning ( "<STR_LIT>" % ( filelist [ <NUM_LIT:0> ] , filelist [ - <NUM_LIT:1> ] ) ) <EOL> else : <EOL> return darkfiles [ <NUM_LIT:0> ] <EOL> def get_headermetadata_dataframe ( dataframe , time_file_dict , name = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> filecount = len ( dataframe . columns ) <EOL> timestart , timeend = dataframe . columns [ <NUM_LIT:0> ] , dataframe . columns [ - <NUM_LIT:1> ] <EOL> filestart , fileend = time_file_dict [ timestart ] , time_file_dict [ timeend ] <EOL> specstart , specend = dataframe . index [ <NUM_LIT:0> ] , dataframe . index [ - <NUM_LIT:1> ] <EOL> return { '<STR_LIT>' : filecount , <EOL> '<STR_LIT>' : timestart , <EOL> '<STR_LIT>' : timeend , <EOL> '<STR_LIT>' : filestart , <EOL> '<STR_LIT>' : fileend , <EOL> '<STR_LIT>' : specstart , <EOL> '<STR_LIT>' : specend } <EOL> def from_spec_files ( file_list , name = '<STR_LIT>' , skiphead = <NUM_LIT> , skipfoot = <NUM_LIT:1> , check_for_overlapping_time = True , extract_dark = True ) : <EOL> '''<STR_LIT>''' <EOL> dict_of_series = { } <EOL> time_file_dict = { } <EOL> _overlap_count = <NUM_LIT:0> <EOL> if extract_dark : <EOL> darkfile = extract_darkfile ( file_list , return_null = True ) <EOL> if darkfile : <EOL> with open ( darkfile ) as f : <EOL> header = [ f . next ( ) . strip ( ) for x in xrange ( skiphead ) ] <EOL> wavedata = np . genfromtxt ( darkfile , dtype = spec_dtype , skip_header = skiphead , skip_footer = skipfoot ) <EOL> darktime = _get_datetime_specsuite ( header ) <EOL> baseline = Series ( wavedata [ '<STR_LIT>' ] , index = wavedata [ '<STR_LIT>' ] , name = darkfile ) <EOL> file_list . remove ( darkfile ) <EOL> f . close ( ) <EOL> else : <EOL> baseline = None <EOL> file_list = [ f for f in file_list <EOL> if os . path . basename ( f ) != '<STR_LIT>' ] <EOL> for infile in file_list : <EOL> with open ( infile ) as f : <EOL> header = [ f . next ( ) . strip ( ) for x in xrange ( skiphead ) ] <EOL> wavedata = np . genfromtxt ( infile , dtype = spec_dtype , skip_header = skiphead , skip_footer = skipfoot ) <EOL> datetime = _get_datetime_specsuite ( header ) <EOL> if datetime in time_file_dict : <EOL> _overlap_count += <NUM_LIT:1> <EOL> if check_for_overlapping_time and _overlap_count : <EOL> raise IOError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> % ( datetime , infile , time_file_dict [ datetime ] ) ) <EOL> time_file_dict [ datetime ] = infile <EOL> dict_of_series [ datetime ] = Series ( wavedata [ '<STR_LIT>' ] , index = wavedata [ '<STR_LIT>' ] ) <EOL> f . close ( ) <EOL> df = DataFrame ( dict_of_series ) <EOL> print df . columns <EOL> timespec = TimeSpectra ( df , name = name ) <EOL> timespec . specunit = '<STR_LIT>' <EOL> timespec . filedict = time_file_dict <EOL> timespec . baseline = baseline <EOL> for infile in file_list : <EOL> if infile != darkfile : <EOL> with open ( infile ) as f : <EOL> header = [ f . next ( ) . strip ( ) for x in xrange ( skiphead ) ] <EOL> meta_partial = _get_metadata_fromheader ( header ) <EOL> break <EOL> meta_general = get_headermetadata_dataframe ( timespec , time_file_dict ) <EOL> meta_general . update ( meta_partial ) <EOL> timespec . metadata = meta_general <EOL> if _overlap_count : <EOL> logger . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' % ( _overlap_count , len ( file_list ) ) ) <EOL> return timespec <EOL> def _get_datetime_specsuite ( specsuiteheader ) : <EOL> '''<STR_LIT>''' <EOL> dateline = specsuiteheader [ <NUM_LIT:2> ] . split ( ) <EOL> year , month , day = int ( dateline [ <NUM_LIT:6> ] ) , dateline [ <NUM_LIT:2> ] , int ( dateline [ <NUM_LIT:3> ] ) <EOL> month = spec_suite_months [ month . lower ( ) ] <EOL> hrs , mins , secs = dateline [ <NUM_LIT:4> ] . split ( '<STR_LIT::>' ) <EOL> hrs = int ( hrs ) ; mins = int ( mins ) ; secs = int ( secs ) <EOL> return datetime ( year , month , day , hrs , mins , secs ) <EOL> def _get_metadata_fromheader ( specsuiteheader ) : <EOL> '''<STR_LIT>''' <EOL> sh = specsuiteheader <EOL> return { '<STR_LIT>' : sh [ <NUM_LIT:4> ] . split ( ) [ <NUM_LIT:3> ] , '<STR_LIT>' : sh [ <NUM_LIT:7> ] . split ( ) [ <NUM_LIT:1> ] , <EOL> '<STR_LIT>' : sh [ <NUM_LIT:8> ] . split ( ) [ <NUM_LIT:2> ] , '<STR_LIT>' : int ( sh [ <NUM_LIT:8> ] . split ( ) [ <NUM_LIT:3> ] ) , <EOL> '<STR_LIT>' : int ( sh [ <NUM_LIT:9> ] . split ( ) [ <NUM_LIT:2> ] ) , '<STR_LIT>' : int ( sh [ <NUM_LIT:10> ] . split ( ) [ <NUM_LIT:2> ] ) , <EOL> '<STR_LIT>' : sh [ <NUM_LIT:5> ] . split ( ) [ <NUM_LIT:3> ] , '<STR_LIT>' : sh [ <NUM_LIT:11> ] . split ( ) [ <NUM_LIT:4> ] , <EOL> '<STR_LIT>' : sh [ <NUM_LIT:12> ] . split ( ) [ <NUM_LIT:2> ] , '<STR_LIT>' : sh [ <NUM_LIT> ] . split ( ) [ <NUM_LIT:4> ] , <EOL> '<STR_LIT>' : sh [ <NUM_LIT> ] . split ( ) [ <NUM_LIT:4> ] , '<STR_LIT>' : int ( sh [ <NUM_LIT:15> ] . split ( ) [ <NUM_LIT:6> ] ) } <EOL> def from_timefile_datafile ( datafile , timefile , extract_dark = True , name = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> tlines = open ( timefile , '<STR_LIT:r>' ) . readlines ( ) <EOL> tlines = [ line . strip ( ) . split ( ) for line in tlines ] <EOL> tlines . pop ( <NUM_LIT:0> ) <EOL> time_file_dict = dict ( ( _get_datetime_timefile ( tline ) , tline [ <NUM_LIT:0> ] ) for tline in tlines ) <EOL> wavedata = np . genfromtxt ( datafile , dtype = '<STR_LIT:float>' , skip_header = <NUM_LIT:1> ) <EOL> data , wavelengths = wavedata [ : , <NUM_LIT:1> : : ] , wavedata [ : , <NUM_LIT:0> ] <EOL> sorted_tfd = sorted ( time_file_dict . items ( ) ) <EOL> sorted_times , sorted_files = zip ( * ( ( ( ( i [ <NUM_LIT:0> ] ) , ( i [ <NUM_LIT:1> ] ) ) for i in sorted_tfd ) ) ) <EOL> if extract_dark : <EOL> darkfile = extract_darkfile ( sorted_files , return_null = True ) <EOL> if darkfile : <EOL> darkindex = sorted_files . index ( darkfile ) <EOL> darktime = sorted_times [ darkindex ] <EOL> baseline = Series ( data [ : , darkindex ] , index = wavelengths , name = darkfile ) <EOL> del time_file_dict [ darktime ] <EOL> sorted_times = list ( sorted_times ) <EOL> sorted_times . remove ( darktime ) <EOL> data = np . delete ( data , darkindex , <NUM_LIT:1> ) <EOL> else : <EOL> baseline = None <EOL> dataframe = TimeSpectra ( data , columns = sorted_times , index = wavelengths ) <EOL> dataframe . baseline = baseline <EOL> dataframe . filedict = time_file_dict <EOL> if name : <EOL> dataframe . name = name <EOL> for line in tlines : <EOL> if line [ <NUM_LIT:0> ] == darkfile : <EOL> pass <EOL> else : <EOL> meta_partial = _get_headermetadata_timefile ( line [ <NUM_LIT:0> ] ) <EOL> break <EOL> meta_general = get_headermetadata_dataframe ( dataframe , time_file_dict ) <EOL> meta_general . update ( meta_partial ) <EOL> dataframe . metadata = meta_general <EOL> dataframe . specunit = '<STR_LIT>' <EOL> dataframe . sort ( axis = <NUM_LIT:1> , inplace = True ) <EOL> return dataframe <EOL> def from_gwu_chem_UVVIS ( filelist , sortnames = False , shortname = True , cut_extension = False , name = '<STR_LIT>' ) : <EOL> '''<STR_LIT>''' <EOL> if shortname : <EOL> fget = lambda x : get_shortname ( x , cut_extension = cut_extension ) <EOL> else : <EOL> fget = lambda x : x <EOL> working_names = [ fget ( afile ) for afile in filelist ] <EOL> dflist = [ read_csv ( afile , sep = '<STR_LIT:U+002C>' , header = None , index_col = <NUM_LIT:0> , skiprows = <NUM_LIT:2> , na_values = '<STR_LIT:U+0020>' , <EOL> names = [ fget ( afile ) ] ) for afile in filelist ] <EOL> dataframe = concat ( dflist , axis = <NUM_LIT:1> ) <EOL> if sortnames : <EOL> dataframe = dataframe . reindex ( columns = sorted ( working_names ) ) <EOL> dataframe = TimeSpectra ( dataframe ) <EOL> dataframe . metadata = None <EOL> dataframe . filedict = None <EOL> dataframe . baseline = None <EOL> dataframe . specunit = '<STR_LIT>' <EOL> if name : <EOL> dataframe . name = name <EOL> return dataframe <EOL> def _get_datetime_timefile ( splitline ) : <EOL> '''<STR_LIT>''' <EOL> hrs , mins , secs = splitline [ <NUM_LIT:4> ] . split ( '<STR_LIT::>' ) <EOL> hrs = int ( hrs ) ; mins = int ( mins ) ; secs = int ( secs ) <EOL> day , month , year = int ( splitline [ <NUM_LIT:3> ] ) , spec_suite_months [ splitline [ <NUM_LIT:2> ] . lower ( ) ] , int ( splitline [ <NUM_LIT:1> ] ) <EOL> return datetime ( year , month , day , hrs , mins , secs ) <EOL> def _get_headermetadata_timefile ( splitline ) : <EOL> '''<STR_LIT>''' <EOL> filldic = dict ( zip ( [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] , splitline [ <NUM_LIT:5> : ] ) ) <EOL> missing = ( '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ) <EOL> missingdic = dict ( ( item , None ) for item in missing ) <EOL> missingdic [ '<STR_LIT>' ] = '<STR_LIT>' <EOL> return missingdic <EOL> def from_oceanoptics ( directory , sample_by = None , sort = True , ** from_spec_files_kwds ) : <EOL> '''<STR_LIT>''' <EOL> files = get_files_in_dir ( directory , sort = sort ) <EOL> if sample_by : <EOL> files = files [ <NUM_LIT:0> : : sample_by ] <EOL> return from_spec_files ( files , ** from_spec_files_kwds ) <EOL> if __name__ == '<STR_LIT:__main__>' : <EOL> ts = from_oceanoptics ( '<STR_LIT>' , sample_by = <NUM_LIT:10> , sort = True , name = '<STR_LIT>' ) <EOL> files = get_files_in_dir ( '<STR_LIT>' ) <EOL> ts = from_spec_files ( files , name = '<STR_LIT>' ) <EOL> files = get_files_in_dir ( '<STR_LIT>' ) <EOL> ts3 = from_gwu_chem_UVVIS ( files , name = '<STR_LIT>' ) <EOL> print ts3 , type ( ts3 ) , ts3 [ ts3 . columns [ <NUM_LIT:0> ] ] </s>
