<s> """<STR_LIT>""" <EOL> try : <EOL> from cPickle import BadPickleGet <EOL> except ImportError : <EOL> BadPickleGet = KeyError <EOL> import pickle <EOL> import logging <EOL> import numpy as np <EOL> from theano . compat import six <EOL> from theano . compat . six . moves import cPickle , xrange <EOL> import os <EOL> import time <EOL> import warnings <EOL> import sys <EOL> from pylearn2 . utils . string_utils import preprocess <EOL> from pylearn2 . utils . mem import improve_memory_error_message <EOL> io = None <EOL> hdf_reader = None <EOL> import struct <EOL> from pylearn2 . utils . exc import reraise_as <EOL> from pylearn2 . utils . string_utils import match <EOL> import shutil <EOL> logger = logging . getLogger ( __name__ ) <EOL> def load ( filepath , retry = True ) : <EOL> """<STR_LIT>""" <EOL> return _load ( filepath , recurse_depth = <NUM_LIT:0> , retry = True ) <EOL> def save ( filepath , obj , on_overwrite = '<STR_LIT:ignore>' ) : <EOL> """<STR_LIT>""" <EOL> filepath = preprocess ( filepath ) <EOL> if os . path . exists ( filepath ) : <EOL> if on_overwrite == '<STR_LIT>' : <EOL> backup = filepath + '<STR_LIT>' <EOL> shutil . move ( filepath , backup ) <EOL> save ( filepath , obj ) <EOL> try : <EOL> os . remove ( backup ) <EOL> except Exception as e : <EOL> warnings . warn ( "<STR_LIT>" + backup <EOL> + "<STR_LIT::>" + str ( e ) ) <EOL> return <EOL> else : <EOL> assert on_overwrite == '<STR_LIT:ignore>' <EOL> try : <EOL> _save ( filepath , obj ) <EOL> except RuntimeError as e : <EOL> """<STR_LIT>""" <EOL> if str ( e ) . find ( '<STR_LIT>' ) != - <NUM_LIT:1> : <EOL> logger . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' + str ( e ) + <EOL> '<STR_LIT>' + <EOL> '<STR_LIT>' ) <EOL> old_limit = sys . getrecursionlimit ( ) <EOL> try : <EOL> sys . setrecursionlimit ( <NUM_LIT> ) <EOL> _save ( filepath , obj ) <EOL> finally : <EOL> sys . setrecursionlimit ( old_limit ) <EOL> def get_pickle_protocol ( ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> protocol_str = os . environ [ '<STR_LIT>' ] <EOL> except KeyError : <EOL> protocol_str = '<STR_LIT:0>' <EOL> if protocol_str == '<STR_LIT>' : <EOL> return pickle . HIGHEST_PROTOCOL <EOL> return int ( protocol_str ) <EOL> def _save ( filepath , obj ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> import joblib <EOL> joblib_available = True <EOL> except ImportError : <EOL> joblib_available = False <EOL> if filepath . endswith ( '<STR_LIT>' ) : <EOL> np . save ( filepath , obj ) <EOL> return <EOL> save_dir = os . path . dirname ( filepath ) <EOL> if save_dir == '<STR_LIT>' : <EOL> save_dir = '<STR_LIT:.>' <EOL> if not os . path . exists ( save_dir ) : <EOL> os . makedirs ( save_dir ) <EOL> if os . path . exists ( save_dir ) and not os . path . isdir ( save_dir ) : <EOL> raise IOError ( "<STR_LIT>" % save_dir ) <EOL> elif not os . access ( save_dir , os . W_OK ) : <EOL> raise IOError ( "<STR_LIT>" % filepath ) <EOL> try : <EOL> if joblib_available and filepath . endswith ( '<STR_LIT>' ) : <EOL> joblib . dump ( obj , filepath ) <EOL> else : <EOL> if filepath . endswith ( '<STR_LIT>' ) : <EOL> warnings . warn ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> with open ( filepath , '<STR_LIT:wb>' ) as filehandle : <EOL> cPickle . dump ( obj , filehandle , get_pickle_protocol ( ) ) <EOL> except Exception as e : <EOL> logger . exception ( "<STR_LIT>" <EOL> "<STR_LIT>" . format ( filepath ) ) <EOL> if str ( e ) . find ( '<STR_LIT>' ) != - <NUM_LIT:1> : <EOL> raise <EOL> try : <EOL> logger . info ( '<STR_LIT>' ) <EOL> with open ( filepath , "<STR_LIT:wb>" ) as f : <EOL> pickle . dump ( obj , f ) <EOL> except Exception as e2 : <EOL> if str ( e ) == '<STR_LIT>' and str ( e2 ) == '<STR_LIT>' : <EOL> logger . exception ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( filepath ) ) <EOL> logger . exception ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> ) <EOL> logger . exception ( <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> ) <EOL> with open ( filepath , '<STR_LIT:wb>' ) as f : <EOL> cPickle . dump ( obj , f , get_pickle_protocol ( ) ) <EOL> logger . info ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> else : <EOL> if str ( e2 ) == '<STR_LIT>' : <EOL> raise <EOL> import pdb <EOL> tb = pdb . traceback . format_exc ( ) <EOL> reraise_as ( IOError ( str ( obj ) + <EOL> '<STR_LIT>' + <EOL> str ( filepath ) + <EOL> '<STR_LIT>' + str ( e ) + <EOL> '<STR_LIT>' + str ( e2 ) + <EOL> '<STR_LIT>' + tb ) ) <EOL> logger . warning ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' . format ( filepath , e ) ) <EOL> def clone_via_serialize ( obj ) : <EOL> """<STR_LIT>""" <EOL> s = cPickle . dumps ( obj , get_pickle_protocol ( ) ) <EOL> return cPickle . loads ( s ) <EOL> def to_string ( obj ) : <EOL> """<STR_LIT>""" <EOL> return cPickle . dumps ( obj , get_pickle_protocol ( ) ) <EOL> def from_string ( s ) : <EOL> """<STR_LIT>""" <EOL> return cPickle . loads ( s ) <EOL> def mkdir ( filepath ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> os . makedirs ( filepath ) <EOL> except OSError : <EOL> if not os . path . isdir ( filepath ) : <EOL> raise <EOL> def read_int ( fin , n = <NUM_LIT:1> ) : <EOL> """<STR_LIT>""" <EOL> if n == <NUM_LIT:1> : <EOL> s = fin . read ( <NUM_LIT:4> ) <EOL> if len ( s ) != <NUM_LIT:4> : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> return struct . unpack ( '<STR_LIT:i>' , s ) [ <NUM_LIT:0> ] <EOL> else : <EOL> rval = [ ] <EOL> for i in xrange ( n ) : <EOL> rval . append ( read_int ( fin ) ) <EOL> return rval <EOL> lush_magic = { <EOL> <NUM_LIT> : '<STR_LIT>' , <EOL> <NUM_LIT> : '<STR_LIT>' , <EOL> <NUM_LIT> : '<STR_LIT>' , <EOL> <NUM_LIT> : '<STR_LIT>' <EOL> } <EOL> def read_bin_lush_matrix ( filepath ) : <EOL> """<STR_LIT>""" <EOL> f = open ( filepath , '<STR_LIT:rb>' ) <EOL> try : <EOL> magic = read_int ( f ) <EOL> except ValueError : <EOL> reraise_as ( "<STR_LIT>" ) <EOL> ndim = read_int ( f ) <EOL> if ndim == <NUM_LIT:0> : <EOL> shape = ( ) <EOL> else : <EOL> shape = read_int ( f , max ( <NUM_LIT:3> , ndim ) ) <EOL> total_elems = <NUM_LIT:1> <EOL> for dim in shape : <EOL> total_elems *= dim <EOL> try : <EOL> dtype = lush_magic [ magic ] <EOL> except KeyError : <EOL> reraise_as ( ValueError ( '<STR_LIT>' + str ( magic ) ) ) <EOL> rval = np . fromfile ( file = f , dtype = dtype , count = total_elems ) <EOL> excess = f . read ( - <NUM_LIT:1> ) <EOL> if excess : <EOL> raise ValueError ( str ( len ( excess ) ) + <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> rval = rval . reshape ( * shape ) <EOL> f . close ( ) <EOL> return rval <EOL> def load_train_file ( config_file_path , environ = None ) : <EOL> """<STR_LIT>""" <EOL> from pylearn2 . config import yaml_parse <EOL> suffix_to_strip = '<STR_LIT>' <EOL> if config_file_path . endswith ( suffix_to_strip ) : <EOL> config_file_full_stem = config_file_path [ <NUM_LIT:0> : - len ( suffix_to_strip ) ] <EOL> else : <EOL> config_file_full_stem = config_file_path <EOL> os . environ [ "<STR_LIT>" ] = config_file_full_stem <EOL> directory = config_file_path . split ( '<STR_LIT:/>' ) [ : - <NUM_LIT:1> ] <EOL> directory = '<STR_LIT:/>' . join ( directory ) <EOL> if directory != '<STR_LIT>' : <EOL> directory += '<STR_LIT:/>' <EOL> os . environ [ "<STR_LIT>" ] = directory <EOL> os . environ [ "<STR_LIT>" ] = config_file_path . split ( '<STR_LIT:/>' ) [ - <NUM_LIT:1> ] <EOL> os . environ [ <EOL> "<STR_LIT>" ] = config_file_full_stem . split ( '<STR_LIT:/>' ) [ - <NUM_LIT:1> ] <EOL> return yaml_parse . load_path ( config_file_path , environ = environ ) <EOL> def _load ( filepath , recurse_depth = <NUM_LIT:0> , retry = True ) : <EOL> """<STR_LIT>""" <EOL> try : <EOL> import joblib <EOL> joblib_available = True <EOL> except ImportError : <EOL> joblib_available = False <EOL> if recurse_depth == <NUM_LIT:0> : <EOL> filepath = preprocess ( filepath ) <EOL> if filepath . endswith ( '<STR_LIT>' ) or filepath . endswith ( '<STR_LIT>' ) : <EOL> return np . load ( filepath ) <EOL> if filepath . endswith ( '<STR_LIT>' ) or filepath . endswith ( '<STR_LIT>' ) : <EOL> try : <EOL> return np . loadtxt ( filepath ) <EOL> except Exception : <EOL> reraise_as ( "<STR_LIT>" <EOL> "<STR_LIT>" . format ( filepath ) ) <EOL> if filepath . endswith ( '<STR_LIT>' ) : <EOL> global io <EOL> if io is None : <EOL> import scipy . io <EOL> io = scipy . io <EOL> try : <EOL> return io . loadmat ( filepath ) <EOL> except NotImplementedError as nei : <EOL> if str ( nei ) . find ( '<STR_LIT>' ) != - <NUM_LIT:1> : <EOL> global hdf_reader <EOL> if hdf_reader is None : <EOL> import h5py <EOL> hdf_reader = h5py <EOL> return hdf_reader . File ( filepath , '<STR_LIT:r>' ) <EOL> else : <EOL> raise <EOL> assert False <EOL> encoding = { '<STR_LIT>' : '<STR_LIT>' } if six . PY3 else { } <EOL> def exponential_backoff ( ) : <EOL> if recurse_depth > <NUM_LIT:9> : <EOL> logger . info ( '<STR_LIT>' <EOL> '<STR_LIT>' . format ( filepath ) ) <EOL> logger . info ( '<STR_LIT>' ) <EOL> with open ( filepath , '<STR_LIT:rb>' ) as f : <EOL> content = f . read ( ) <EOL> return cPickle . loads ( content , ** encoding ) <EOL> else : <EOL> nsec = <NUM_LIT:0.5> * ( <NUM_LIT> ** float ( recurse_depth ) ) <EOL> logger . info ( "<STR_LIT>" . format ( nsec ) ) <EOL> time . sleep ( nsec ) <EOL> return _load ( filepath , recurse_depth + <NUM_LIT:1> , retry ) <EOL> try : <EOL> if not joblib_available : <EOL> with open ( filepath , '<STR_LIT:rb>' ) as f : <EOL> obj = cPickle . load ( f , ** encoding ) <EOL> else : <EOL> try : <EOL> obj = joblib . load ( filepath ) <EOL> except Exception as e : <EOL> if os . path . exists ( filepath ) and not os . path . isdir ( filepath ) : <EOL> raise <EOL> raise_cannot_open ( filepath ) <EOL> except MemoryError as e : <EOL> if os . path . splitext ( filepath ) [ <NUM_LIT:1> ] == "<STR_LIT>" : <EOL> improve_memory_error_message ( e , <EOL> ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> % filepath ) <EOL> else : <EOL> improve_memory_error_message ( e , <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" % filepath ) <EOL> except ( BadPickleGet , EOFError , KeyError ) as e : <EOL> if not retry : <EOL> reraise_as ( e . __class__ ( '<STR_LIT>' . format ( filepath ) ) ) <EOL> obj = exponential_backoff ( ) <EOL> except ValueError : <EOL> logger . exception <EOL> if not retry : <EOL> reraise_as ( ValueError ( '<STR_LIT>' . format ( filepath ) ) ) <EOL> obj = exponential_backoff ( ) <EOL> except Exception : <EOL> reraise_as ( "<STR_LIT>" . format ( filepath ) ) <EOL> if not hasattr ( obj , '<STR_LIT>' ) : <EOL> try : <EOL> obj . yaml_src = '<STR_LIT>' + os . path . abspath ( filepath ) + '<STR_LIT:">' <EOL> except Exception : <EOL> pass <EOL> return obj <EOL> def raise_cannot_open ( path ) : <EOL> """<STR_LIT>""" <EOL> pieces = path . split ( '<STR_LIT:/>' ) <EOL> for i in xrange ( <NUM_LIT:1> , len ( pieces ) + <NUM_LIT:1> ) : <EOL> so_far = '<STR_LIT:/>' . join ( pieces [ <NUM_LIT:0> : i ] ) <EOL> if not os . path . exists ( so_far ) : <EOL> if i == <NUM_LIT:1> : <EOL> if so_far == '<STR_LIT>' : <EOL> continue <EOL> reraise_as ( IOError ( '<STR_LIT>' + path + '<STR_LIT>' + so_far + <EOL> '<STR_LIT>' ) ) <EOL> parent = '<STR_LIT:/>' . join ( pieces [ <NUM_LIT:0> : i - <NUM_LIT:1> ] ) <EOL> bad = pieces [ i - <NUM_LIT:1> ] <EOL> if not os . path . isdir ( parent ) : <EOL> reraise_as ( IOError ( "<STR_LIT>" + path + "<STR_LIT>" + <EOL> parent + "<STR_LIT>" ) ) <EOL> candidates = os . listdir ( parent ) <EOL> if len ( candidates ) == <NUM_LIT:0> : <EOL> reraise_as ( IOError ( "<STR_LIT>" + path + "<STR_LIT>" + <EOL> parent + "<STR_LIT>" ) ) <EOL> if len ( candidates ) > <NUM_LIT:100> : <EOL> reraise_as ( IOError ( "<STR_LIT>" + path + "<STR_LIT>" + <EOL> parent + "<STR_LIT:.>" ) ) <EOL> if os . path . islink ( path ) : <EOL> reraise_as ( IOError ( path + "<STR_LIT>" <EOL> "<STR_LIT>" ) ) <EOL> reraise_as ( IOError ( "<STR_LIT>" + path + "<STR_LIT>" + <EOL> parent + "<STR_LIT>" + match ( bad , candidates ) + <EOL> "<STR_LIT>" + bad + "<STR_LIT:?>" ) ) <EOL> assert False </s>
