<s> from __future__ import division , print_function , unicode_literals <EOL> from six . moves . urllib . request import urlretrieve <EOL> import numpy as np <EOL> import tarfile <EOL> import h5py <EOL> import os <EOL> bs_data_dir = os . environ . get ( '<STR_LIT>' , '<STR_LIT:.>' ) <EOL> url = '<STR_LIT>' <EOL> cifar10_file = os . path . join ( bs_data_dir , '<STR_LIT>' ) <EOL> hdf_file = os . path . join ( bs_data_dir , '<STR_LIT>' ) <EOL> print ( "<STR_LIT>" , bs_data_dir ) <EOL> if not os . path . exists ( cifar10_file ) : <EOL> print ( "<STR_LIT>" ) <EOL> urlretrieve ( url , cifar10_file ) <EOL> print ( "<STR_LIT>" ) <EOL> archive_paths = [ <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' <EOL> ] <EOL> print ( "<STR_LIT>" ) <EOL> with tarfile . open ( cifar10_file ) as f : <EOL> res = [ ] <EOL> for fn in archive_paths : <EOL> buf = f . extractfile ( fn ) . read ( ) <EOL> tmp = np . fromstring ( buf , dtype = np . uint8 ) . reshape ( - <NUM_LIT:1> , <NUM_LIT> * <NUM_LIT:3> + <NUM_LIT:1> ) <EOL> res . append ( tmp ) <EOL> print ( "<STR_LIT>" ) <EOL> ds = np . concatenate ( res ) <EOL> num_tr = <NUM_LIT> <EOL> x_tr = ds [ : num_tr , <NUM_LIT:1> : ] . reshape ( ( <NUM_LIT:1> , num_tr , <NUM_LIT:3> , <NUM_LIT:32> , <NUM_LIT:32> ) ) <EOL> x_tr = x_tr . transpose ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:2> ] ) <EOL> y_tr = ds [ : num_tr , <NUM_LIT:0> ] . reshape ( ( <NUM_LIT:1> , num_tr , <NUM_LIT:1> ) ) <EOL> x_va = ds [ num_tr : <NUM_LIT> , <NUM_LIT:1> : ] . reshape ( ( <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT:32> , <NUM_LIT:32> ) ) <EOL> x_va = x_va . transpose ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:2> ] ) <EOL> y_va = ds [ num_tr : <NUM_LIT> , <NUM_LIT:0> ] . reshape ( ( <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT:1> ) ) <EOL> x_te = ds [ <NUM_LIT> : , <NUM_LIT:1> : ] . reshape ( ( <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT:3> , <NUM_LIT:32> , <NUM_LIT:32> ) ) <EOL> x_te = x_te . transpose ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:2> ] ) <EOL> y_te = ds [ <NUM_LIT> : , <NUM_LIT:0> ] . reshape ( ( <NUM_LIT:1> , <NUM_LIT> , <NUM_LIT:1> ) ) <EOL> tr_mean = x_tr . squeeze ( ) . mean ( axis = <NUM_LIT:0> ) <EOL> tr_std = x_tr . squeeze ( ) . std ( axis = <NUM_LIT:0> ) <EOL> x_tr = ( x_tr - tr_mean ) / tr_std <EOL> x_va = ( x_va - tr_mean ) / tr_std <EOL> x_te = ( x_te - tr_mean ) / tr_std <EOL> print ( "<STR_LIT>" ) <EOL> f = h5py . File ( hdf_file , '<STR_LIT:w>' ) <EOL> description = """<STR_LIT>""" <EOL> f . attrs [ '<STR_LIT:description>' ] = description <EOL> f . attrs [ '<STR_LIT>' ] = tr_mean <EOL> f . attrs [ '<STR_LIT>' ] = tr_std <EOL> variant = f . create_group ( '<STR_LIT>' ) <EOL> group = variant . create_group ( '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT:default>' , data = x_tr , compression = '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT>' , data = y_tr , compression = '<STR_LIT>' ) <EOL> group = variant . create_group ( '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT:default>' , data = x_va , compression = '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT>' , data = y_va , compression = '<STR_LIT>' ) <EOL> group = variant . create_group ( '<STR_LIT:test>' ) <EOL> group . create_dataset ( name = '<STR_LIT:default>' , data = x_te , compression = '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT>' , data = y_te , compression = '<STR_LIT>' ) <EOL> nr_tr = <NUM_LIT> <EOL> x_tr = ds [ : nr_tr , <NUM_LIT:1> : ] . reshape ( ( <NUM_LIT:1> , nr_tr , <NUM_LIT:3> , <NUM_LIT:32> , <NUM_LIT:32> ) ) . transpose ( [ <NUM_LIT:0> , <NUM_LIT:1> , <NUM_LIT:3> , <NUM_LIT:4> , <NUM_LIT:2> ] ) <EOL> x_tr = ( x_tr - tr_mean ) / tr_std <EOL> y_tr = ds [ : nr_tr , <NUM_LIT:0> ] . reshape ( ( <NUM_LIT:1> , nr_tr , <NUM_LIT:1> ) ) <EOL> variant = f . create_group ( '<STR_LIT>' ) <EOL> group = variant . create_group ( '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT:default>' , data = x_tr , compression = '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT>' , data = y_tr , compression = '<STR_LIT>' ) <EOL> group = variant . create_group ( '<STR_LIT:test>' ) <EOL> group . create_dataset ( name = '<STR_LIT:default>' , data = x_te , compression = '<STR_LIT>' ) <EOL> group . create_dataset ( name = '<STR_LIT>' , data = y_te , compression = '<STR_LIT>' ) <EOL> f . close ( ) <EOL> print ( "<STR_LIT>" ) </s>
