def generic_exiobase12_parser(exio_files, system=None):
    """ Generic EXIOBASE version 1 and 2 parser

    This is used internally by parse_exiobase1 / 2 functions to
    parse exiobase files. In most cases, these top-level functions
    should just work, but in case of archived exiobase versions
    it might be necessary to use low-level function here.

    Parameters
    ----------

    exio_files: dict of dict

    system: str (pxp or ixi)
        Only used for the metadata

    """

    version = ' & '.join({dd.get('version', '')
                          for dd in exio_files.values()
                          if dd.get('version', '')})

    meta_rec = MRIOMetaData(system=system,
                            name="EXIOBASE",
                            version=version)

    if len(version) == 0:
        meta_rec.note("No version information found, assuming exiobase 1")
        meta_rec.change_meta('version', 1)
        version = '1'

    core_components = ['A', 'Y', 'Z']

    core_data = dict()
    ext_data = dict()
    for tt, tpara in exio_files.items():
        full_file_path = os.path.join(tpara['root_repo'], tpara['file_path'])
        logging.debug("Parse {}".format(full_file_path))
        if tpara['root_repo'][-3:] == 'zip':
            with zipfile.ZipFile(tpara['root_repo'], 'r') as zz:
                raw_data = pd.read_table(
                    zz.open(tpara['file_path']),
                    index_col=list(range(tpara['index_col'])),
                    header=list(range(tpara['index_rows'])))
        else:
            raw_data = pd.read_table(
                full_file_path,
                index_col=list(range(tpara['index_col'])),
                header=list(range(tpara['index_rows'])))

        meta_rec._add_fileio('EXIOBASE data {} parsed from {}'.format(
            tt, full_file_path))
        if tt in core_components:
            core_data[tt] = raw_data
        else:
            ext_data[tt] = raw_data

    for table in core_data:
        core_data[table].index.names = ['region', 'sector', 'unit']
        if table == 'A' or table == 'Z':
            core_data[table].columns.names = ['region', 'sector']
            _unit = pd.DataFrame(
                    core_data[table].iloc[:, 0]).reset_index(
                        level='unit').unit
            _unit = pd.DataFrame(_unit)
            _unit.columns = ['unit']
        if table == 'Y':
            core_data[table].columns.names = ['region', 'category']
        core_data[table].reset_index(level='unit', drop=True, inplace=True)

    core_data['unit'] = _unit

    mon_unit = core_data['unit'].iloc[0, 0]
    if '/' in mon_unit:
        mon_unit = mon_unit.split('/')[0]
        core_data['unit'].unit = mon_unit

    extensions = dict()
    for tt, tpara in exio_files.items():
        if tt in core_components:
            continue
        ext_name = '_'.join(tt.split('_')[1:])
        table_type = tt.split('_')[0]

        if tpara['index_col'] == 3:
            ext_data[tt].index.names = [
                'stressor', 'compartment', 'unit']
        elif tpara['index_col'] == 2:
            ext_data[tt].index.names = [
                'stressor', 'unit']
        else:
            raise ParserError('Unknown EXIOBASE file structure')

        if table_type == 'FY':
            ext_data[tt].columns.names = [
                'region', 'category']
        else:
            ext_data[tt].columns.names = [
                'region', 'sector']
        try:
            _unit = pd.DataFrame(
                ext_data[tt].iloc[:, 0]
            ).reset_index(level='unit').unit
        except IndexError:
            _unit = pd.DataFrame(
                ext_data[tt].iloc[:, 0])
            _unit.columns = ['unit']
            _unit['unit'] = 'undef'
            _unit.reset_index(level='unit', drop=True, inplace=True)
            _unit = pd.DataFrame(_unit)
            _unit.columns = ['unit']
        _unit = pd.DataFrame(_unit)
        _unit.columns = ['unit']
        _new_unit = _unit.unit.str.replace('/'+mon_unit, '')
        _new_unit[_new_unit == ''] = _unit.unit[
            _new_unit == ''].str.replace('/', '')
        _unit.unit = _new_unit

        ext_data[tt].reset_index(level='unit', drop=True, inplace=True)
        ext_dict = extensions.get(ext_name, dict())
        ext_dict.update({table_type: ext_data[tt],
                         'unit': _unit,
                         'name': ext_name})
        extensions.update({ext_name: ext_dict})

    if version[0] == '1':
        year = 2000
    elif version[0] == '2':
        year = 2000
    elif version[0] == '3':
        raise ParserError(
            "This function can not be used to parse EXIOBASE 3")
    else:
        logging.warning("Unknown EXIOBASE version")
        year = None

    return IOSystem(version=version,
                    price='current',
                    year=year,
                    meta=meta_rec,
                    **dict(core_data, **extensions))
def generic_exiobase12_parser ( exio_files , system = None ) : version = ' & ' . join ( { dd . get ( 'version' , '' ) for dd in exio_files . values ( ) if dd . get ( 'version' , '' ) } ) meta_rec = MRIOMetaData ( system = system , name = "EXIOBASE" , version = version ) if len ( version ) == 0 : meta_rec . note ( "No version information found, assuming exiobase 1" ) meta_rec . change_meta ( 'version' , 1 ) version = '1' core_components = [ 'A' , 'Y' , 'Z' ] core_data = dict ( ) ext_data = dict ( ) for tt , tpara in exio_files . items ( ) : full_file_path = os . path . join ( tpara [ 'root_repo' ] , tpara [ 'file_path' ] ) logging . debug ( "Parse {}" . format ( full_file_path ) ) if tpara [ 'root_repo' ] [ - 3 : ] == 'zip' : with zipfile . ZipFile ( tpara [ 'root_repo' ] , 'r' ) as zz : raw_data = pd . read_table ( zz . open ( tpara [ 'file_path' ] ) , index_col = list ( range ( tpara [ 'index_col' ] ) ) , header = list ( range ( tpara [ 'index_rows' ] ) ) ) else : raw_data = pd . read_table ( full_file_path , index_col = list ( range ( tpara [ 'index_col' ] ) ) , header = list ( range ( tpara [ 'index_rows' ] ) ) ) meta_rec . _add_fileio ( 'EXIOBASE data {} parsed from {}' . format ( tt , full_file_path ) ) if tt in core_components : core_data [ tt ] = raw_data else : ext_data [ tt ] = raw_data for table in core_data : core_data [ table ] . index . names = [ 'region' , 'sector' , 'unit' ] if table == 'A' or table == 'Z' : core_data [ table ] . columns . names = [ 'region' , 'sector' ] _unit = pd . DataFrame ( core_data [ table ] . iloc [ : , 0 ] ) . reset_index ( level = 'unit' ) . unit _unit = pd . DataFrame ( _unit ) _unit . columns = [ 'unit' ] if table == 'Y' : core_data [ table ] . columns . names = [ 'region' , 'category' ] core_data [ table ] . reset_index ( level = 'unit' , drop = True , inplace = True ) core_data [ 'unit' ] = _unit mon_unit = core_data [ 'unit' ] . iloc [ 0 , 0 ] if '/' in mon_unit : mon_unit = mon_unit . split ( '/' ) [ 0 ] core_data [ 'unit' ] . unit = mon_unit extensions = dict ( ) for tt , tpara in exio_files . items ( ) : if tt in core_components : continue ext_name = '_' . join ( tt . split ( '_' ) [ 1 : ] ) table_type = tt . split ( '_' ) [ 0 ] if tpara [ 'index_col' ] == 3 : ext_data [ tt ] . index . names = [ 'stressor' , 'compartment' , 'unit' ] elif tpara [ 'index_col' ] == 2 : ext_data [ tt ] . index . names = [ 'stressor' , 'unit' ] else : raise ParserError ( 'Unknown EXIOBASE file structure' ) if table_type == 'FY' : ext_data [ tt ] . columns . names = [ 'region' , 'category' ] else : ext_data [ tt ] . columns . names = [ 'region' , 'sector' ] try : _unit = pd . DataFrame ( ext_data [ tt ] . iloc [ : , 0 ] ) . reset_index ( level = 'unit' ) . unit except IndexError : _unit = pd . DataFrame ( ext_data [ tt ] . iloc [ : , 0 ] ) _unit . columns = [ 'unit' ] _unit [ 'unit' ] = 'undef' _unit . reset_index ( level = 'unit' , drop = True , inplace = True ) _unit = pd . DataFrame ( _unit ) _unit . columns = [ 'unit' ] _unit = pd . DataFrame ( _unit ) _unit . columns = [ 'unit' ] _new_unit = _unit . unit . str . replace ( '/' + mon_unit , '' ) _new_unit [ _new_unit == '' ] = _unit . unit [ _new_unit == '' ] . str . replace ( '/' , '' ) _unit . unit = _new_unit ext_data [ tt ] . reset_index ( level = 'unit' , drop = True , inplace = True ) ext_dict = extensions . get ( ext_name , dict ( ) ) ext_dict . update ( { table_type : ext_data [ tt ] , 'unit' : _unit , 'name' : ext_name } ) extensions . update ( { ext_name : ext_dict } ) if version [ 0 ] == '1' : year = 2000 elif version [ 0 ] == '2' : year = 2000 elif version [ 0 ] == '3' : raise ParserError ( "This function can not be used to parse EXIOBASE 3" ) else : logging . warning ( "Unknown EXIOBASE version" ) year = None return IOSystem ( version = version , price = 'current' , year = year , meta = meta_rec , ** dict ( core_data , ** extensions ) )
Generic EXIOBASE version 1 and 2 parser

    This is used internally by parse_exiobase1 / 2 functions to
    parse exiobase files. In most cases, these top-level functions
    should just work, but in case of archived exiobase versions
    it might be necessary to use low-level function here.

    Parameters
    ----------

    exio_files: dict of dict

    system: str (pxp or ixi)
        Only used for the metadata