def test_dataformat(data_file, dataformat_file, name_mode='file_names'):
    """
    Test a data formatfile against a particular data file.

    This goes through all the steps of data import printing out
    the results of each step, so you can see where the import fails.

    Parameters
    ----------
    data_file : str
        Path to data file, including extension.
    dataformat : dict or str
        A dataformat dict, or path to file. See example below.
    name_mode : str
        How to identyfy sample names. If 'file_names' uses the
        input name of the file, stripped of the extension. If
        'metadata_names' uses the 'name' attribute of the 'meta'
        sub-dictionary in dataformat. If any other str, uses this
        str as the sample name.

    Example
    -------
    >>>
    {'genfromtext_args': {'delimiter': ',',
                          'skip_header': 4},  # passed directly to np.genfromtxt
     'column_id': {'name_row': 3,  # which row contains the column names
                   'delimiter': ',',  # delimeter between column names
                   'timecolumn': 0,  # which column contains the 'time' variable
                   'pattern': '([A-z]{1,2}[0-9]{1,3})'},  # a regex pattern which captures the column names
     'meta_regex': {  # a dict of (line_no: ([descriptors], [regexs])) pairs
                    0: (['path'], '(.*)'),
                    2: (['date', 'method'],  # MUST include date
                     '([A-Z][a-z]+ [0-9]+ [0-9]{4}[ ]+[0-9:]+ [amp]+).* ([A-z0-9]+\.m)')
                   }
    }

    Returns
    -------
    sample, analytes, data, meta : tuple
    """
    print('*************************************************\n' + 
          'Testing suitability of data format description...\n' + 
          '  Dataformat File: {:s}'.format(dataformat_file) + '\n' +  
          '  Data File: {:s}'.format(data_file) + '\n' +
          '*************************************************')
    
    print('  Test: open data file...')
    with open(data_file) as f:
        lines = f.readlines()    
    print('    Success!')
    
    print('  Test: read dataformat file...')
    # if dataformat is not a dict, load the json
    try:
        with open(dataformat_file) as f:
            dataformat = json.load(f)
        print('    Success!')
    except:
        print("        ***PROBLEM: The dataformat file isn't in a valid .json format")
        raise
    
    print("  Test: read metadata using 'metadata_regex'...")
    if 'meta_regex' in dataformat.keys():
        meta = Bunch()
        got = []
        for k, v in dataformat['meta_regex'].items():
            rep = '    Line {:s}: '.format(k)
            try:
                out = re.search(v[-1], lines[int(k)]).groups()
                for i in np.arange(len(v[0])):
                    meta[v[0][i]] = out[i]
                    print(rep + v[0][i] + ': ' + out[i])
                    got.append(v[0][i])
            except:
                print(rep + '***PROBLEM in meta_regex:\n'.format(k) + 
                      '        [' + ', '.join(v[0]) + '] cannot be derived from "' + v[1] + '".\n' + 
                      '        Test regex against: "{:s}"'.format(lines[int(k)].strip()) + '\n' + 
                      '        at https://regex101.com/.')
                raise
        print('    Finished - does the above look correct?')
        if 'date' not in got:
            print('        ***PROBLEM: ' + 
                  'meta_regex must identify data collection start time as "date".')
            raise ValueError('meta_regex must identify "date" attribute, containing data collection start time')
    else:
        print('        ***PROBLEM: ' + 
              'meta_regex not specified. At minimum, must identify data collection start time as "date".')
        raise ValueError('meta_regex must identify "date" attribute, containing data collection start time')
    
    # sample name
    print('  Test: Sample Name IDs...')
    if name_mode == 'file_names':
        sample = os.path.basename(data_file).split('.')[0]
        print('    Sample name grabbed from file: ' + sample)
    else:
        try:
            sample = meta['name']
            print('   Sample name from metadata_regex: ' + sample)
        except KeyError:
            print('       ***PROBLEM: Sample name not identified by metadata_regex - please include "name".')
            raise
    print('        ***Is the sample name correct?***')
    
    # column and analyte names
    print('  Test: Getting Column Names...')
    columns = np.array(lines[dataformat['column_id']['name_row']].strip().split(dataformat['column_id']['delimiter']))
    print('    Columns from line {:.0f}: '.format(dataformat['column_id']['name_row']) + ', '.join(columns))
    if 'pattern' in dataformat['column_id'].keys():
        print('    Cleaning up using column_id/pattern...')
        pr = re.compile(dataformat['column_id']['pattern'])
        analytes = [pr.match(c).groups()[0] for c in columns if pr.match(c)]
        if len(analytes) == 0:
            raise ValueError('no analyte names identified. Check pattern in column_id section.')
        print('    Cleaned Analyte Names: ' + ', '.join(analytes))
        print('        ***This should only contain analyte names... does it?***')

    # do any required pre-formatting
    if 'preformat_replace' in dataformat.keys():
        print('  Test: preformat_replace...')
        with open(data_file) as f:
            fbuffer = f.read()
        for k, v in dataformat['preformat_replace'].items():
            print('    replacing "' + k + '" with "' + v + '"')
            fbuffer = re.sub(k, v, fbuffer)
        print('    Done.')
        
        print('  Test: Reading Pre-Formatted Data...')
        # dead data
        try:
            read_data = np.genfromtxt(BytesIO(fbuffer.encode()),
                                    **dataformat['genfromtext_args']).T
        except:
            print('        ***PROBLEM during data read - check genfromtext_args\n' + 
                  '        and preformat_replace terms?')
            raise
    else:
        print('  Test: Reading Data...')
        # read data
        try:
            read_data = np.genfromtxt(data_file,
                                    **dataformat['genfromtext_args']).T
        except:
            print('        ***PROBLEM during data read - check genfromtext_args.\n' + 
                  '        If they look correct, think about including preformat_replace terms?')
            raise
    
    print('    checking dimensions...')
    if read_data.shape[0] != len(analytes) + 1:
        print('        ***PROBLEM: ' + 
              'There are {:.0f} data columns, but {:.0f} column names.\n'.format(read_data.shape[0], len(analytes) + 1) +
              '        Check your identification of column names, or your pre-formatting parameters')
        raise ValueError('Data - Column Name mismatch')
    else:
        print('    Success!')
    
    # data dict
    print('  Test: Combine data into dictionary...')    
    dind = np.ones(read_data.shape[0], dtype=bool)
    dind[dataformat['column_id']['timecolumn']] = False

    data = Bunch()
    print('    Time in column {:.0f} ({:s})'.format(dataformat['column_id']['timecolumn'],
                                                  columns[dataformat['column_id']['timecolumn']]))
    data['Time'] = read_data[dataformat['column_id']['timecolumn']]
    
    print('    Calculating total counts...')
    data['total_counts'] = read_data[dind].sum(0)
    print('    Success!')
    
    print('Tests completed successfully.\n' +
          '  **This does not necessarily mean everything has worked!**\n' +
          '    Look carefully through the output of this function, and\n' +
          '    make sure it looks right.\n\n' +
          'Outputs are: sample_name, analytes, data_dict, metadata_dict')

    return sample, analytes, data, meta
def test_dataformat ( data_file , dataformat_file , name_mode = 'file_names' ) : print ( '*************************************************\n' + 'Testing suitability of data format description...\n' + '  Dataformat File: {:s}' . format ( dataformat_file ) + '\n' + '  Data File: {:s}' . format ( data_file ) + '\n' + '*************************************************' ) print ( '  Test: open data file...' ) with open ( data_file ) as f : lines = f . readlines ( ) print ( '    Success!' ) print ( '  Test: read dataformat file...' ) try : with open ( dataformat_file ) as f : dataformat = json . load ( f ) print ( '    Success!' ) except : print ( "        ***PROBLEM: The dataformat file isn't in a valid .json format" ) raise print ( "  Test: read metadata using 'metadata_regex'..." ) if 'meta_regex' in dataformat . keys ( ) : meta = Bunch ( ) got = [ ] for k , v in dataformat [ 'meta_regex' ] . items ( ) : rep = '    Line {:s}: ' . format ( k ) try : out = re . search ( v [ - 1 ] , lines [ int ( k ) ] ) . groups ( ) for i in np . arange ( len ( v [ 0 ] ) ) : meta [ v [ 0 ] [ i ] ] = out [ i ] print ( rep + v [ 0 ] [ i ] + ': ' + out [ i ] ) got . append ( v [ 0 ] [ i ] ) except : print ( rep + '***PROBLEM in meta_regex:\n' . format ( k ) + '        [' + ', ' . join ( v [ 0 ] ) + '] cannot be derived from "' + v [ 1 ] + '".\n' + '        Test regex against: "{:s}"' . format ( lines [ int ( k ) ] . strip ( ) ) + '\n' + '        at https://regex101.com/.' ) raise print ( '    Finished - does the above look correct?' ) if 'date' not in got : print ( '        ***PROBLEM: ' + 'meta_regex must identify data collection start time as "date".' ) raise ValueError ( 'meta_regex must identify "date" attribute, containing data collection start time' ) else : print ( '        ***PROBLEM: ' + 'meta_regex not specified. At minimum, must identify data collection start time as "date".' ) raise ValueError ( 'meta_regex must identify "date" attribute, containing data collection start time' ) print ( '  Test: Sample Name IDs...' ) if name_mode == 'file_names' : sample = os . path . basename ( data_file ) . split ( '.' ) [ 0 ] print ( '    Sample name grabbed from file: ' + sample ) else : try : sample = meta [ 'name' ] print ( '   Sample name from metadata_regex: ' + sample ) except KeyError : print ( '       ***PROBLEM: Sample name not identified by metadata_regex - please include "name".' ) raise print ( '        ***Is the sample name correct?***' ) print ( '  Test: Getting Column Names...' ) columns = np . array ( lines [ dataformat [ 'column_id' ] [ 'name_row' ] ] . strip ( ) . split ( dataformat [ 'column_id' ] [ 'delimiter' ] ) ) print ( '    Columns from line {:.0f}: ' . format ( dataformat [ 'column_id' ] [ 'name_row' ] ) + ', ' . join ( columns ) ) if 'pattern' in dataformat [ 'column_id' ] . keys ( ) : print ( '    Cleaning up using column_id/pattern...' ) pr = re . compile ( dataformat [ 'column_id' ] [ 'pattern' ] ) analytes = [ pr . match ( c ) . groups ( ) [ 0 ] for c in columns if pr . match ( c ) ] if len ( analytes ) == 0 : raise ValueError ( 'no analyte names identified. Check pattern in column_id section.' ) print ( '    Cleaned Analyte Names: ' + ', ' . join ( analytes ) ) print ( '        ***This should only contain analyte names... does it?***' ) if 'preformat_replace' in dataformat . keys ( ) : print ( '  Test: preformat_replace...' ) with open ( data_file ) as f : fbuffer = f . read ( ) for k , v in dataformat [ 'preformat_replace' ] . items ( ) : print ( '    replacing "' + k + '" with "' + v + '"' ) fbuffer = re . sub ( k , v , fbuffer ) print ( '    Done.' ) print ( '  Test: Reading Pre-Formatted Data...' ) try : read_data = np . genfromtxt ( BytesIO ( fbuffer . encode ( ) ) , ** dataformat [ 'genfromtext_args' ] ) . T except : print ( '        ***PROBLEM during data read - check genfromtext_args\n' + '        and preformat_replace terms?' ) raise else : print ( '  Test: Reading Data...' ) try : read_data = np . genfromtxt ( data_file , ** dataformat [ 'genfromtext_args' ] ) . T except : print ( '        ***PROBLEM during data read - check genfromtext_args.\n' + '        If they look correct, think about including preformat_replace terms?' ) raise print ( '    checking dimensions...' ) if read_data . shape [ 0 ] != len ( analytes ) + 1 : print ( '        ***PROBLEM: ' + 'There are {:.0f} data columns, but {:.0f} column names.\n' . format ( read_data . shape [ 0 ] , len ( analytes ) + 1 ) + '        Check your identification of column names, or your pre-formatting parameters' ) raise ValueError ( 'Data - Column Name mismatch' ) else : print ( '    Success!' ) print ( '  Test: Combine data into dictionary...' ) dind = np . ones ( read_data . shape [ 0 ] , dtype = bool ) dind [ dataformat [ 'column_id' ] [ 'timecolumn' ] ] = False data = Bunch ( ) print ( '    Time in column {:.0f} ({:s})' . format ( dataformat [ 'column_id' ] [ 'timecolumn' ] , columns [ dataformat [ 'column_id' ] [ 'timecolumn' ] ] ) ) data [ 'Time' ] = read_data [ dataformat [ 'column_id' ] [ 'timecolumn' ] ] print ( '    Calculating total counts...' ) data [ 'total_counts' ] = read_data [ dind ] . sum ( 0 ) print ( '    Success!' ) print ( 'Tests completed successfully.\n' + '  **This does not necessarily mean everything has worked!**\n' + '    Look carefully through the output of this function, and\n' + '    make sure it looks right.\n\n' + 'Outputs are: sample_name, analytes, data_dict, metadata_dict' ) return sample , analytes , data , meta
Test a data formatfile against a particular data file.

    This goes through all the steps of data import printing out
    the results of each step, so you can see where the import fails.

    Parameters
    ----------
    data_file : str
        Path to data file, including extension.
    dataformat : dict or str
        A dataformat dict, or path to file. See example below.
    name_mode : str
        How to identyfy sample names. If 'file_names' uses the
        input name of the file, stripped of the extension. If
        'metadata_names' uses the 'name' attribute of the 'meta'
        sub-dictionary in dataformat. If any other str, uses this
        str as the sample name.

    Example
    -------
    >>>
    {'genfromtext_args': {'delimiter': ',',
                          'skip_header': 4},  # passed directly to np.genfromtxt
     'column_id': {'name_row': 3,  # which row contains the column names
                   'delimiter': ',',  # delimeter between column names
                   'timecolumn': 0,  # which column contains the 'time' variable
                   'pattern': '([A-z]{1,2}[0-9]{1,3})'},  # a regex pattern which captures the column names
     'meta_regex': {  # a dict of (line_no: ([descriptors], [regexs])) pairs
                    0: (['path'], '(.*)'),
                    2: (['date', 'method'],  # MUST include date
                     '([A-Z][a-z]+ [0-9]+ [0-9]{4}[ ]+[0-9:]+ [amp]+).* ([A-z0-9]+\.m)')
                   }
    }

    Returns
    -------
    sample, analytes, data, meta : tuple