<s> """<STR_LIT>""" <EOL> from __future__ import with_statement <EOL> import re <EOL> import copy <EOL> from itertools import groupby <EOL> import numpy as N <EOL> from pebl . util import * <EOL> from pebl import discretizer <EOL> from pebl import config <EOL> _pfilename = config . StringParameter ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> config . fileexists ( ) , <EOL> ) <EOL> _ptext = config . StringParameter ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' , <EOL> default = '<STR_LIT>' <EOL> ) <EOL> _pdiscretize = config . IntParameter ( <EOL> '<STR_LIT>' , <EOL> '<STR_LIT>' + '<STR_LIT>' , <EOL> default = <NUM_LIT:0> <EOL> ) <EOL> class ParsingError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class IncorrectArityError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , errors ) : <EOL> self . errors = errors <EOL> def __repr__ ( self ) : <EOL> msg = "<STR_LIT>" <EOL> for v , uniquevals in errors : <EOL> msg += "<STR_LIT>" % ( v . name , v . arity , uniquevals ) <EOL> class ClassVariableError ( Exception ) : <EOL> """<STR_LIT>""" <EOL> msg = """<STR_LIT>""" <EOL> class Annotation ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , * args ) : <EOL> self . name = str ( name ) <EOL> def __repr__ ( self ) : <EOL> return "<STR_LIT>" % ( self . __class__ . __name__ , self . name ) <EOL> class Sample ( Annotation ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> class Variable ( Annotation ) : <EOL> """<STR_LIT>""" <EOL> arity = - <NUM_LIT:1> <EOL> class ContinuousVariable ( Variable ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , param ) : <EOL> self . name = str ( name ) <EOL> class DiscreteVariable ( Variable ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , param ) : <EOL> self . name = str ( name ) <EOL> self . arity = int ( param ) <EOL> class ClassVariable ( DiscreteVariable ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , param ) : <EOL> self . name = str ( name ) <EOL> self . labels = [ l . strip ( ) for l in param . split ( '<STR_LIT:U+002C>' ) ] <EOL> self . label2int = dict ( ( l , i ) for i , l in enumerate ( self . labels ) ) <EOL> self . arity = len ( self . labels ) <EOL> class Dataset ( object ) : <EOL> def __init__ ( self , observations , missing = None , interventions = None , <EOL> variables = None , samples = None , skip_stats = False ) : <EOL> """<STR_LIT>""" <EOL> self . observations = observations <EOL> self . missing = missing <EOL> self . interventions = interventions <EOL> self . variables = variables <EOL> self . samples = samples <EOL> obs = observations <EOL> if missing is None : <EOL> self . missing = N . zeros ( obs . shape , dtype = bool ) <EOL> if interventions is None : <EOL> self . interventions = N . zeros ( obs . shape , dtype = bool ) <EOL> if variables is None : <EOL> self . variables = N . array ( [ Variable ( str ( i ) ) for i in xrange ( obs . shape [ <NUM_LIT:1> ] ) ] ) <EOL> self . _guess_arities ( ) <EOL> if samples is None : <EOL> self . samples = N . array ( [ Sample ( str ( i ) ) for i in xrange ( obs . shape [ <NUM_LIT:0> ] ) ] ) <EOL> if not skip_stats : <EOL> self . _calc_stats ( ) <EOL> def subset ( self , variables = None , samples = None ) : <EOL> """<STR_LIT>""" <EOL> variables = variables if variables is not None else range ( self . variables . size ) <EOL> samples = samples if samples is not None else range ( self . samples . size ) <EOL> skip_stats = not ( self . has_interventions or self . has_missing ) <EOL> d = Dataset ( <EOL> self . observations [ N . ix_ ( samples , variables ) ] , <EOL> self . missing [ N . ix_ ( samples , variables ) ] , <EOL> self . interventions [ N . ix_ ( samples , variables ) ] , <EOL> self . variables [ variables ] , <EOL> self . samples [ samples ] , <EOL> skip_stats = skip_stats <EOL> ) <EOL> if skip_stats : <EOL> d . _has_interventions = False <EOL> d . _has_missing = False <EOL> return d <EOL> def _subset_ni_fast ( self , variables ) : <EOL> ds = _FastDataset . __new__ ( _FastDataset ) <EOL> if not self . has_interventions : <EOL> ds . observations = self . observations [ : , variables ] <EOL> ds . samples = self . samples <EOL> else : <EOL> samples = N . where ( self . interventions [ : , variables [ <NUM_LIT:0> ] ] == False ) [ <NUM_LIT:0> ] <EOL> ds . observations = self . observations [ samples ] [ : , variables ] <EOL> ds . samples = self . samples [ samples ] <EOL> ds . variables = self . variables [ variables ] <EOL> return ds <EOL> def subset_byname ( self , variables = None , samples = None ) : <EOL> """<STR_LIT>""" <EOL> vardict = dict ( ( v . name , i ) for i , v in enumerate ( self . variables ) ) <EOL> sampledict = dict ( ( s . name , i ) for i , s in enumerate ( self . samples ) ) <EOL> variables = [ vardict [ v ] for v in variables ] if variables else variables <EOL> samples = [ sampledict [ s ] for s in samples ] if samples else samples <EOL> return self . subset ( variables , samples ) <EOL> def discretize ( self , includevars = None , excludevars = [ ] , numbins = <NUM_LIT:3> ) : <EOL> """<STR_LIT>""" <EOL> self . original_observations = self . observations . copy ( ) <EOL> self = discretizer . maximum_entropy_discretize ( <EOL> self , <EOL> includevars , excludevars , <EOL> numbins <EOL> ) <EOL> def tofile ( self , filename , * args , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> with file ( filename , '<STR_LIT:w>' ) as f : <EOL> f . write ( self . tostring ( * args , ** kwargs ) ) <EOL> def tostring ( self , linesep = '<STR_LIT:\n>' , variable_header = True , sample_header = True ) : <EOL> """<STR_LIT>""" <EOL> def dataitem ( row , col ) : <EOL> val = "<STR_LIT:X>" if self . missing [ row , col ] else str ( self . observations [ row , col ] ) <EOL> val += "<STR_LIT:!>" if self . interventions [ row , col ] else '<STR_LIT>' <EOL> return val <EOL> def variable ( v ) : <EOL> name = v . name <EOL> if isinstance ( v , ClassVariable ) : <EOL> return "<STR_LIT>" % ( name , '<STR_LIT:U+002C>' . join ( v . labels ) ) <EOL> elif isinstance ( v , DiscreteVariable ) : <EOL> return "<STR_LIT>" % ( name , v . arity ) <EOL> elif isinstance ( v , ContinuousVariable ) : <EOL> return "<STR_LIT>" % name <EOL> else : <EOL> return v . name <EOL> lines = [ ] <EOL> if sample_header : <EOL> lines . append ( "<STR_LIT:\t>" . join ( [ variable ( v ) for v in self . variables ] ) ) <EOL> nrows , ncols = self . shape <EOL> d = [ [ dataitem ( r , c ) for c in xrange ( ncols ) ] for r in xrange ( nrows ) ] <EOL> if sample_header and hasattr ( self . samples [ <NUM_LIT:0> ] , '<STR_LIT:name>' ) : <EOL> d = [ [ s . name ] + row for row , s in zip ( d , self . samples ) ] <EOL> lines . extend ( [ "<STR_LIT:\t>" . join ( row ) for row in d ] ) <EOL> return linesep . join ( lines ) <EOL> @ property <EOL> def shape ( self ) : <EOL> """<STR_LIT>""" <EOL> return self . observations . shape <EOL> @ property <EOL> def has_interventions ( self ) : <EOL> """<STR_LIT>""" <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> return self . _has_interventions <EOL> else : <EOL> self . _has_interventions = self . interventions . any ( ) <EOL> return self . _has_interventions <EOL> @ property <EOL> def has_missing ( self ) : <EOL> """<STR_LIT>""" <EOL> if hasattr ( self , '<STR_LIT>' ) : <EOL> return self . _has_missing <EOL> else : <EOL> self . _has_missing = self . missing . any ( ) <EOL> return self . _has_missing <EOL> def _calc_stats ( self ) : <EOL> self . _has_interventions = self . interventions . any ( ) <EOL> self . _has_missing = self . missing . any ( ) <EOL> def _guess_arities ( self ) : <EOL> """<STR_LIT>""" <EOL> for col , var in enumerate ( self . variables ) : <EOL> var . arity = N . unique ( self . observations [ : , col ] ) . size <EOL> var . __class__ = DiscreteVariable <EOL> def check_arities ( self ) : <EOL> """<STR_LIT>""" <EOL> errors = [ ] <EOL> for col , v in enumerate ( self . variables ) : <EOL> if isinstance ( v , DiscreteVariable ) : <EOL> uniquevals = N . unique ( self . observations [ : , col ] ) . size <EOL> if v . arity < uniquevals : <EOL> errors . append ( ( v , uniquevals ) ) <EOL> if errors : <EOL> raise IncorrectArityError ( errors ) <EOL> class _FastDataset ( Dataset ) : <EOL> """<STR_LIT>""" <EOL> pass <EOL> def fromfile ( filename ) : <EOL> """<STR_LIT>""" <EOL> with file ( filename ) as f : <EOL> return fromstring ( f . read ( ) ) <EOL> def fromstring ( stringrep , fieldsep = '<STR_LIT:\t>' ) : <EOL> """<STR_LIT>""" <EOL> def dataitem ( item , v ) : <EOL> item = item . strip ( ) <EOL> intervention = False <EOL> missing = False <EOL> if item [ <NUM_LIT:0> ] == "<STR_LIT:!>" : <EOL> intervention = True <EOL> item = item [ <NUM_LIT:1> : ] <EOL> elif item [ - <NUM_LIT:1> ] == "<STR_LIT:!>" : <EOL> intervention = True <EOL> item = item [ : - <NUM_LIT:1> ] <EOL> if item [ <NUM_LIT:0> ] in ( '<STR_LIT:x>' , '<STR_LIT:X>' ) or item [ - <NUM_LIT:1> ] in ( '<STR_LIT:x>' , '<STR_LIT:X>' ) : <EOL> missing = True <EOL> item = "<STR_LIT:0>" if not isinstance ( v , ClassVariable ) else v . labels [ <NUM_LIT:0> ] <EOL> val = item <EOL> if isinstance ( v , ClassVariable ) : <EOL> try : <EOL> val = v . label2int [ val ] <EOL> except KeyError : <EOL> raise ClassVariableError ( ) <EOL> elif isinstance ( v , DiscreteVariable ) : <EOL> try : <EOL> val = int ( val ) <EOL> except ValueError : <EOL> msg = "<STR_LIT>" % ( v . name , val ) <EOL> raise ParsingError ( msg ) <EOL> elif isinstance ( v , ContinuousVariable ) : <EOL> try : <EOL> val = float ( val ) <EOL> except ValueError : <EOL> msg = "<STR_LIT>" % ( v . name , val ) <EOL> raise ParsingError ( msg ) <EOL> else : <EOL> if '<STR_LIT:.>' in val : <EOL> try : <EOL> val = float ( val ) <EOL> except : <EOL> msg = "<STR_LIT>" % val <EOL> raise ParsingError ( msg ) <EOL> else : <EOL> try : <EOL> val = int ( val ) <EOL> except : <EOL> msg = "<STR_LIT>" % val <EOL> raise ParsingError ( msg ) <EOL> return ( val , missing , intervention ) <EOL> dtype_re = re . compile ( "<STR_LIT>" ) <EOL> def variable ( v ) : <EOL> v = v . strip ( "<STR_LIT>" ) <EOL> parts = v . split ( "<STR_LIT:U+002C>" , <NUM_LIT:1> ) <EOL> if len ( parts ) is <NUM_LIT:2> : <EOL> name , dtype = parts <EOL> match = dtype_re . match ( dtype ) <EOL> if not match : <EOL> raise ParsingError ( "<STR_LIT>" % v ) <EOL> dtype_name , dtype_param = match . groups ( ) <EOL> dtype_name = dtype_name . lower ( ) <EOL> else : <EOL> name = parts [ <NUM_LIT:0> ] <EOL> dtype_name , dtype_param = None , None <EOL> vartypes = { <EOL> None : Variable , <EOL> '<STR_LIT>' : ContinuousVariable , <EOL> '<STR_LIT>' : DiscreteVariable , <EOL> '<STR_LIT:class>' : ClassVariable <EOL> } <EOL> return vartypes [ dtype_name ] ( name , dtype_param ) <EOL> lines = ( l . strip ( ) for l in stringrep . splitlines ( ) if l ) <EOL> lines = ( l for l in lines if not l . startswith ( '<STR_LIT:#>' ) ) <EOL> variables = lines . next ( ) . split ( fieldsep ) <EOL> variables = N . array ( [ variable ( v ) for v in variables ] ) <EOL> d = [ [ c for c in row . split ( fieldsep ) ] for row in lines ] <EOL> samplenames = True if len ( d [ <NUM_LIT:0> ] ) == len ( variables ) + <NUM_LIT:1> else False <EOL> samples = None <EOL> if samplenames : <EOL> samples = N . array ( [ Sample ( row [ <NUM_LIT:0> ] ) for row in d ] ) <EOL> d = [ row [ <NUM_LIT:1> : ] for row in d ] <EOL> d = N . array ( [ [ dataitem ( c , v ) for c , v in zip ( row , variables ) ] for row in d ] ) <EOL> obs , missing , interventions = d . transpose ( <NUM_LIT:2> , <NUM_LIT:0> , <NUM_LIT:1> ) <EOL> dtype = '<STR_LIT:int>' if obs . dtype . kind is '<STR_LIT:i>' else obs . dtype <EOL> d = Dataset ( <EOL> obs . astype ( dtype ) , <EOL> missing . astype ( bool ) , <EOL> interventions . astype ( bool ) , <EOL> variables , <EOL> samples , <EOL> ) <EOL> d . check_arities ( ) <EOL> return d <EOL> def fromconfig ( ) : <EOL> """<STR_LIT>""" <EOL> fname = config . get ( '<STR_LIT>' ) <EOL> text = config . get ( '<STR_LIT>' ) <EOL> if text : <EOL> data_ = fromstring ( text ) <EOL> else : <EOL> if not fname : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> data_ = fromfile ( fname ) <EOL> numbins = config . get ( '<STR_LIT>' ) <EOL> if numbins > <NUM_LIT:0> : <EOL> data_ . discretize ( numbins = numbins ) <EOL> return data_ <EOL> def merge ( datasets , axis = None ) : <EOL> """<STR_LIT>""" <EOL> if axis == '<STR_LIT>' : <EOL> variables = N . hstack ( tuple ( d . variables for d in datasets ) ) <EOL> samples = datasets [ <NUM_LIT:0> ] . samples <EOL> stacker = N . hstack <EOL> else : <EOL> samples = N . hstack ( tuple ( d . samples for d in datasets ) ) <EOL> variables = datasets [ <NUM_LIT:0> ] . variables <EOL> stacker = N . vstack <EOL> missing = stacker ( tuple ( d . missing for d in datasets ) ) <EOL> interventions = stacker ( tuple ( d . interventions for d in datasets ) ) <EOL> observations = stacker ( tuple ( d . observations for d in datasets ) ) <EOL> return Dataset ( observations , missing , interventions , variables , samples ) </s>
