<gh_stars>1-10
# convert multifile datasets to singlefile
import pandas as pd
import os

class Files( object ):
    def __init__( self, base_dir, *args, **kwargs ):
        '''
        list the files from the nested directory structure
        generated by SYNDA application and access of the ESGF CMIP5 data holdings
        '''
        self.base_dir = base_dir
        self.files = self.list_files( )
        # self.df = self._to_dataframe( )

    def list_files( self ):
        return [ os.path.join( root, fn ) for root, subs, files in os.walk( self.base_dir ) \
                    if len( files ) > 0 for fn in files if fn.endswith( '.nc' ) ]
    @staticmethod
    def _split_fn( fn ):
        return os.path.basename( fn ).split( '.' )[0].split( '_' )
    @staticmethod
    def f( x ):
        '''
        take the files dataframe and split the years
        into begin year/month and end year/month and 
        add new columns to a new dataframe
        '''
        begin, end = x[ 'years' ].split( '-' )
        x['begin_month'] = begin[4:] 
        x['begin_year'] = begin[:4]
        x['end_month'] = end[4:]
        x['end_year'] = end[:4]
        return x
    def _to_dataframe( self ):
        import pandas as pd

        column_order = ['fn', 'variable', 'cmor_table', 'model', 'scenario', 'experiment', 'years']
        out = []
        for fn in self.files:
            variable, cmor_table, model, scenario, experiment, years = self._split_fn( fn )
            out.append( {'fn':fn, 'variable':variable, 'cmor_table':cmor_table, \
                        'model':model, 'scenario':scenario, 'experiment':experiment, 'years':years } )
        
        df = pd.DataFrame( out )
        df = df.loc[ :, column_order ]
        return df.apply( self.f, axis=1 )

if __name__ == '__main__':
    import os, shutil, itertools
    from nco import Nco

    nco = Nco() # so we can do this from Python...
    
    base_dir = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/cmip5_nwt/cmip5_raw_restructure'
    output_path = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/cmip5_nwt/cmip5_raw_ncrcat'

    variables = [ 'tas', 'pr' ]
    scenarios = [ 'historical', 'rcp45', 'rcp60', 'rcp85' ]
    models = [ 'GFDL-CM3','IPSL-CM5A-LR', 'GISS-E2-R', 'CCSM4', 'MRI-CGCM3' ]

    # lets get the darn data returned that we want:
    files_df = Files( base_dir )._to_dataframe( )
    
    for variable, model, scenario in itertools.product( variables, models, scenarios ):
        print( 'running {}-{}-{}'.format( variable, model, scenario ) )
        # get the files we want to work with for this run
        cur_df = files_df[ (files_df.variable == variable) & (files_df.model == model) & (files_df.scenario == scenario) ].copy()
        cur_df = cur_df.apply( pd.to_numeric, errors='ignore' ) # dtypes updating
        cur_df = cur_df.sort_values(['end_year']) # sort it based on the end_year
        cur_files = cur_df['fn'].tolist()
        if len(cur_files) > 0:
            begin, end = cur_df.iloc[0]['begin_year'], cur_df.iloc[-1]['end_year']
            begin = str(begin)+'01'
            end = str(end)+'12'
            dirname, basename = os.path.split(cur_files[0])
            basename = basename.split( '.' )[0]
            basename = '_'.join(basename.split( '_' )[0:-2])
            output_filename = os.path.join( output_path, model, scenario, variable, basename+'_'+begin+'-'+end+'.nc' )
            
            try:
                dirname, basename = os.path.split( output_filename )
                if not os.path.exists( dirname ):
                    os.makedirs( dirname )
            except:
                pass

            if len( cur_files ) > 1:
                _ = nco.ncrcat( input=cur_files, output=output_filename )
                # os.system( 'ncrcat ' + ' '.join([ fn for fn in files ]) + ' -o ' + output_filename
            elif len(cur_files) == 1:
                fn, = cur_files
                basename = os.path.basename( fn )
                _ = shutil.copy( fn, os.path.join( output_path, model, scenario, variable, basename ) )
            else:
                print( '___ --> NODATA for: {} {} {}'.format(model, scenario, variable) )

# convert multifile datasets to singlefile
import pandas as pd
import os

class Files( object ):
    def __init__( self, base_dir, *args, **kwargs ):
        '''
        list the files from the nested directory structure
        generated by SYNDA application and access of the ESGF CMIP5 data holdings
        '''
        self.base_dir = base_dir
        self.files = self.list_files( )
        # self.df = self._to_dataframe( )

    def list_files( self ):
        return [ os.path.join( root, fn ) for root, subs, files in os.walk( self.base_dir ) \
                    if len( files ) > 0 for fn in files if fn.endswith( '.nc' ) ]
    @staticmethod
    def _split_fn( fn ):
        return os.path.basename( fn ).split( '.' )[0].split( '_' )
    @staticmethod
    def f( x ):
        '''
        take the files dataframe and split the years
        into begin year/month and end year/month and 
        add new columns to a new dataframe
        '''
        begin, end = x[ 'years' ].split( '-' )
        x['begin_month'] = begin[4:] 
        x['begin_year'] = begin[:4]
        x['end_month'] = end[4:]
        x['end_year'] = end[:4]
        return x
    def _to_dataframe( self ):
        import pandas as pd

        column_order = ['fn', 'variable', 'cmor_table', 'model', 'scenario', 'experiment', 'years']
        out = []
        for fn in self.files:
            variable, cmor_table, model, scenario, experiment, years = self._split_fn( fn )
            out.append( {'fn':fn, 'variable':variable, 'cmor_table':cmor_table, \
                        'model':model, 'scenario':scenario, 'experiment':experiment, 'years':years } )
        
        df = pd.DataFrame( out )
        df = df.loc[ :, column_order ]
        return df.apply( self.f, axis=1 )

if __name__ == '__main__':
    import os, shutil, itertools
    from nco import Nco

    nco = Nco() # so we can do this from Python...
    
    base_dir = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/cmip5_nwt/cmip5_raw_restructure'
    output_path = '/workspace/Shared/Tech_Projects/DeltaDownscaling/project_data/cmip5_nwt/cmip5_raw_ncrcat'

    variables = [ 'tas', 'pr' ]
    scenarios = [ 'historical', 'rcp45', 'rcp60', 'rcp85' ]
    models = [ 'GFDL-CM3','IPSL-CM5A-LR', 'GISS-E2-R', 'CCSM4', 'MRI-CGCM3' ]

    # lets get the darn data returned that we want:
    files_df = Files( base_dir )._to_dataframe( )
    
    for variable, model, scenario in itertools.product( variables, models, scenarios ):
        print( 'running {}-{}-{}'.format( variable, model, scenario ) )
        # get the files we want to work with for this run
        cur_df = files_df[ (files_df.variable == variable) & (files_df.model == model) & (files_df.scenario == scenario) ].copy()
        cur_df = cur_df.apply( pd.to_numeric, errors='ignore' ) # dtypes updating
        cur_df = cur_df.sort_values(['end_year']) # sort it based on the end_year
        cur_files = cur_df['fn'].tolist()
        if len(cur_files) > 0:
            begin, end = cur_df.iloc[0]['begin_year'], cur_df.iloc[-1]['end_year']
            begin = str(begin)+'01'
            end = str(end)+'12'
            dirname, basename = os.path.split(cur_files[0])
            basename = basename.split( '.' )[0]
            basename = '_'.join(basename.split( '_' )[0:-2])
            output_filename = os.path.join( output_path, model, scenario, variable, basename+'_'+begin+'-'+end+'.nc' )
            
            try:
                dirname, basename = os.path.split( output_filename )
                if not os.path.exists( dirname ):
                    os.makedirs( dirname )
            except:
                pass

            if len( cur_files ) > 1:
                _ = nco.ncrcat( input=cur_files, output=output_filename )
                # os.system( 'ncrcat ' + ' '.join([ fn for fn in files ]) + ' -o ' + output_filename
            elif len(cur_files) == 1:
                fn, = cur_files
                basename = os.path.basename( fn )
                _ = shutil.copy( fn, os.path.join( output_path, model, scenario, variable, basename ) )
            else:
                print( '___ --> NODATA for: {} {} {}'.format(model, scenario, variable) )
# convert multifile datasets to singlefile list the files from the nested directory structure         generated by SYNDA application and access of the ESGF CMIP5 data holdings # self.df = self._to_dataframe( ) take the files dataframe and split the years         into begin year/month and end year/month and          add new columns to a new dataframe # so we can do this from Python... # lets get the darn data returned that we want: # get the files we want to work with for this run # dtypes updating # sort it based on the end_year # os.system( 'ncrcat ' + ' '.join([ fn for fn in files ]) + ' -o ' + output_filename