def ascii2h5(dirname, output_fname):
    """
    Converts from a directory of tarballed ASCII ".samp" files to a single
    HDF5 file. Essentially, converts from the original release format to a
    single HDF5 file.
    """

    import tarfile
    import sys
    from glob import glob
    from contextlib import closing

    # The datatype that will be used to store extinction, A0
    A0_dtype = 'float16'

    def load_samp_file(f, fname):
        # Parse filename
        fname_chunks = os.path.split(fname)[1].split('_')
        l = float(fname_chunks[0])
        b = float(fname_chunks[1])

        # Load ASCII data
        data_raw = np.loadtxt(f, dtype='float64')

        n_samples = data_raw.shape[1] - 1
        n_dists = data_raw.shape[0]

        # Construct output
        dtype = [
            ('dist', 'int32'),
            ('A0', A0_dtype, (n_samples,))]

        data = np.empty(n_dists, dtype=dtype)
        data['dist'][:] = data_raw[:,0]
        data['A0'][:,:] = data_raw[:,1:]

        return (l,b), data

    def process_tarball(tarball_fname):
        # Write to the progress bar
        print('.', end='')
        sys.stdout.flush()

        with closing(tarfile.open(tarball_fname, mode='r:gz')) as f_tar:
            fnames = f_tar.getnames()

            f = f_tar.extractfile(fnames[0])
            (l,b), data = load_samp_file(f, fnames[0])
            n_dists, n_samples = data['A0'].shape
            n_coords = len(fnames)

            dtype = [
                ('l', 'float32'),
                ('b', 'float32'),
                ('dist', 'int32', (n_dists,)),
                ('A0', A0_dtype, (n_dists, n_samples))]
            data_combined = np.empty(n_coords, dtype=dtype)

            for k,fn in enumerate(fnames):
                # print('File {: >4d} of {:d}'.format(k+1, n_coords))
                f = f_tar.extractfile(fn)
                (l,b), data = load_samp_file(f, fn)
                data_combined['l'][k] = l
                data_combined['b'][k] = b
                data_combined['dist'][k] = data['dist']
                data_combined['A0'][k] = data['A0']

        return data_combined

    def save_data(data, fname):
        with closing(h5py.File(fname, 'w')) as f:
            f.create_dataset(
                'samples',
                data=data,
                chunks=True,
                compression='gzip',
                compression_opts=3)

    print('Progress: ', end='')
    sys.stdout.flush()

    tar_fname_list = glob(os.path.join(dirname, 'A_samp_*.tar.gz'))
    d = np.hstack([process_tarball(fn) for fn in tar_fname_list])

    print('+', end='')
    sys.stdout.flush()

    save_data(d, output_fname)

    print('')
def ascii2h5 ( dirname , output_fname ) : import tarfile import sys from glob import glob from contextlib import closing A0_dtype = 'float16' def load_samp_file ( f , fname ) : fname_chunks = os . path . split ( fname ) [ 1 ] . split ( '_' ) l = float ( fname_chunks [ 0 ] ) b = float ( fname_chunks [ 1 ] ) data_raw = np . loadtxt ( f , dtype = 'float64' ) n_samples = data_raw . shape [ 1 ] - 1 n_dists = data_raw . shape [ 0 ] dtype = [ ( 'dist' , 'int32' ) , ( 'A0' , A0_dtype , ( n_samples , ) ) ] data = np . empty ( n_dists , dtype = dtype ) data [ 'dist' ] [ : ] = data_raw [ : , 0 ] data [ 'A0' ] [ : , : ] = data_raw [ : , 1 : ] return ( l , b ) , data def process_tarball ( tarball_fname ) : print ( '.' , end = '' ) sys . stdout . flush ( ) with closing ( tarfile . open ( tarball_fname , mode = 'r:gz' ) ) as f_tar : fnames = f_tar . getnames ( ) f = f_tar . extractfile ( fnames [ 0 ] ) ( l , b ) , data = load_samp_file ( f , fnames [ 0 ] ) n_dists , n_samples = data [ 'A0' ] . shape n_coords = len ( fnames ) dtype = [ ( 'l' , 'float32' ) , ( 'b' , 'float32' ) , ( 'dist' , 'int32' , ( n_dists , ) ) , ( 'A0' , A0_dtype , ( n_dists , n_samples ) ) ] data_combined = np . empty ( n_coords , dtype = dtype ) for k , fn in enumerate ( fnames ) : f = f_tar . extractfile ( fn ) ( l , b ) , data = load_samp_file ( f , fn ) data_combined [ 'l' ] [ k ] = l data_combined [ 'b' ] [ k ] = b data_combined [ 'dist' ] [ k ] = data [ 'dist' ] data_combined [ 'A0' ] [ k ] = data [ 'A0' ] return data_combined def save_data ( data , fname ) : with closing ( h5py . File ( fname , 'w' ) ) as f : f . create_dataset ( 'samples' , data = data , chunks = True , compression = 'gzip' , compression_opts = 3 ) print ( 'Progress: ' , end = '' ) sys . stdout . flush ( ) tar_fname_list = glob ( os . path . join ( dirname , 'A_samp_*.tar.gz' ) ) d = np . hstack ( [ process_tarball ( fn ) for fn in tar_fname_list ] ) print ( '+' , end = '' ) sys . stdout . flush ( ) save_data ( d , output_fname ) print ( '' )
Converts from a directory of tarballed ASCII ".samp" files to a single
    HDF5 file. Essentially, converts from the original release format to a
    single HDF5 file.