<s> from __future__ import print_function , division , absolute_import <EOL> from datetime import datetime , date , time , timedelta <EOL> from itertools import chain <EOL> import re <EOL> import sys <EOL> from textwrap import dedent <EOL> from warnings import warn <EOL> from dateutil . parser import parse as dateparse <EOL> import numpy as np <EOL> from . dispatch import dispatch <EOL> from . coretypes import ( int32 , int64 , float64 , bool_ , complex128 , datetime_ , <EOL> Option , var , from_numpy , Tuple , null , <EOL> Record , string , Null , DataShape , real , date_ , time_ , <EOL> Unit , timedelta_ , TimeDelta , object_ , String ) <EOL> from . predicates import isdimension , isrecord <EOL> from . py2help import _strtypes , _inttypes , MappingProxyType , OrderedDict <EOL> from . internal_utils import _toposort , groupby <EOL> from . util import subclasses <EOL> __all__ = [ '<STR_LIT>' ] <EOL> @ dispatch ( object ) <EOL> def discover ( obj , ** kwargs ) : <EOL> """<STR_LIT>""" <EOL> type_name = type ( obj ) . __name__ <EOL> if hasattr ( obj , '<STR_LIT>' ) and hasattr ( obj , '<STR_LIT>' ) : <EOL> warn ( <EOL> dedent ( <EOL> """<STR_LIT>""" % type_name , <EOL> ) , <EOL> DeprecationWarning , <EOL> ) <EOL> return from_numpy ( obj . shape , obj . dtype ) <EOL> raise NotImplementedError ( "<STR_LIT>" % type_name ) <EOL> @ dispatch ( _inttypes ) <EOL> def discover ( i ) : <EOL> return int64 <EOL> npinttypes = tuple ( chain . from_iterable ( ( x for x in subclasses ( icls ) <EOL> if x . __name__ . startswith ( ( '<STR_LIT:int>' , <EOL> '<STR_LIT>' ) ) ) <EOL> for icls in subclasses ( np . integer ) ) ) <EOL> if sys . version_info [ <NUM_LIT:0> ] == <NUM_LIT:3> : <EOL> @ dispatch ( bytes ) <EOL> def discover ( b ) : <EOL> return String ( '<STR_LIT:A>' ) <EOL> @ dispatch ( npinttypes ) <EOL> def discover ( n ) : <EOL> return from_numpy ( ( ) , n . dtype ) <EOL> @ dispatch ( float ) <EOL> def discover ( f ) : <EOL> return float64 <EOL> @ dispatch ( bool ) <EOL> def discover ( b ) : <EOL> return bool_ <EOL> @ dispatch ( complex ) <EOL> def discover ( z ) : <EOL> return complex128 <EOL> @ dispatch ( datetime ) <EOL> def discover ( dt ) : <EOL> return datetime_ <EOL> @ dispatch ( timedelta ) <EOL> def discover ( td ) : <EOL> return TimeDelta ( unit = '<STR_LIT>' ) <EOL> @ dispatch ( date ) <EOL> def discover ( dt ) : <EOL> return date_ <EOL> @ dispatch ( time ) <EOL> def discover ( t ) : <EOL> return time_ <EOL> @ dispatch ( ( type ( None ) , Null ) ) <EOL> def discover ( i ) : <EOL> return null <EOL> bools = { '<STR_LIT:False>' : False , <EOL> '<STR_LIT:false>' : False , <EOL> '<STR_LIT:True>' : True , <EOL> '<STR_LIT:true>' : True } <EOL> def timeparse ( x , formats = ( '<STR_LIT>' , '<STR_LIT>' ) ) : <EOL> msg = '<STR_LIT>' <EOL> for format in formats : <EOL> try : <EOL> return datetime . strptime ( x , format ) . time ( ) <EOL> except ValueError as e : <EOL> msg = str ( e ) <EOL> raise ValueError ( msg ) <EOL> def deltaparse ( x ) : <EOL> """<STR_LIT>""" <EOL> value , unit = re . split ( '<STR_LIT>' , x . strip ( ) ) <EOL> value = float ( value ) <EOL> if not value . is_integer ( ) : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> return np . timedelta64 ( int ( value ) , TimeDelta ( unit = unit ) . unit ) <EOL> string_coercions = int , float , bools . __getitem__ , deltaparse , timeparse <EOL> def is_zero_time ( t ) : <EOL> return not ( t . hour or t . minute or t . second or t . microsecond ) <EOL> @ dispatch ( _strtypes ) <EOL> def discover ( s ) : <EOL> if not s : <EOL> return null <EOL> for f in string_coercions : <EOL> try : <EOL> return discover ( f ( s ) ) <EOL> except ( ValueError , KeyError ) : <EOL> pass <EOL> if s . isalpha ( ) or s . isspace ( ) : <EOL> return string <EOL> try : <EOL> d = dateparse ( s ) <EOL> except ( ValueError , OverflowError ) : <EOL> pass <EOL> else : <EOL> return date_ if is_zero_time ( d . time ( ) ) else datetime_ <EOL> return string <EOL> @ dispatch ( ( tuple , list , set , frozenset ) ) <EOL> def discover ( seq ) : <EOL> if not seq : <EOL> return var * string <EOL> unite = do_one ( [ unite_identical , unite_base , unite_merge_dimensions ] ) <EOL> if ( all ( isinstance ( item , ( tuple , list ) ) for item in seq ) and <EOL> len ( set ( map ( len , seq ) ) ) == <NUM_LIT:1> ) : <EOL> columns = list ( zip ( * seq ) ) <EOL> try : <EOL> types = [ unite ( [ discover ( data ) for data in column ] ) . subshape [ <NUM_LIT:0> ] <EOL> for column in columns ] <EOL> unite = do_one ( [ unite_identical , unite_merge_dimensions , Tuple ] ) <EOL> return len ( seq ) * unite ( types ) <EOL> except AttributeError : <EOL> pass <EOL> if all ( isinstance ( item , dict ) for item in seq ) : <EOL> keys = sorted ( set . union ( * ( set ( d ) for d in seq ) ) ) <EOL> columns = [ [ item . get ( key ) for item in seq ] for key in keys ] <EOL> try : <EOL> types = [ unite ( [ discover ( data ) for data in column ] ) . subshape [ <NUM_LIT:0> ] <EOL> for column in columns ] <EOL> return len ( seq ) * Record ( list ( zip ( keys , types ) ) ) <EOL> except AttributeError : <EOL> pass <EOL> types = list ( map ( discover , seq ) ) <EOL> return do_one ( [ unite_identical , unite_merge_dimensions , Tuple ] ) ( types ) <EOL> def isnull ( ds ) : <EOL> return ds == null or ds == DataShape ( null ) <EOL> identity = lambda x : x <EOL> edges = [ <EOL> ( string , int64 ) , <EOL> ( string , real ) , <EOL> ( string , date_ ) , <EOL> ( string , datetime_ ) , <EOL> ( string , timedelta_ ) , <EOL> ( string , bool_ ) , <EOL> ( datetime_ , date_ ) , <EOL> ( int64 , int32 ) , <EOL> ( real , int64 ) , <EOL> ( string , null ) ] <EOL> numeric_edges = [ <EOL> ( int64 , int32 ) , <EOL> ( real , int64 ) , <EOL> ( string , null ) <EOL> ] <EOL> edges = groupby ( lambda x : x [ <NUM_LIT:1> ] , edges ) <EOL> edges = dict ( ( k , set ( a for a , b in v ) ) for k , v in edges . items ( ) ) <EOL> toposorted = _toposort ( edges ) <EOL> def lowest_common_dshape ( dshapes ) : <EOL> """<STR_LIT>""" <EOL> common = set . intersection ( * [ descendents ( edges , ds ) for ds in dshapes ] ) <EOL> if common and any ( c in toposorted for c in common ) : <EOL> return min ( common , key = toposorted . index ) <EOL> raise ValueError ( "<STR_LIT>" ) <EOL> def unite_base ( dshapes ) : <EOL> """<STR_LIT>""" <EOL> dshapes = [ unpack ( ds ) for ds in dshapes ] <EOL> bynull = groupby ( isnull , dshapes ) <EOL> try : <EOL> good_dshapes = bynull [ False ] <EOL> except KeyError : <EOL> return len ( dshapes ) * null <EOL> if all ( isinstance ( ds , Unit ) for ds in good_dshapes ) : <EOL> base = lowest_common_dshape ( good_dshapes ) <EOL> elif ( all ( isinstance ( ds , Record ) for ds in good_dshapes ) and <EOL> ds . names == dshapes [ <NUM_LIT:0> ] . names for ds in good_dshapes ) : <EOL> names = good_dshapes [ <NUM_LIT:0> ] . names <EOL> base = Record ( [ [ name , <EOL> unite_base ( [ ds . dict . get ( name , null ) for ds in good_dshapes ] ) . subshape [ <NUM_LIT:0> ] ] <EOL> for name in names ] ) <EOL> if base : <EOL> if bynull . get ( True ) : <EOL> base = Option ( base ) <EOL> return len ( dshapes ) * base <EOL> def unite_identical ( dshapes ) : <EOL> """<STR_LIT>""" <EOL> if len ( set ( dshapes ) ) == <NUM_LIT:1> : <EOL> return len ( dshapes ) * dshapes [ <NUM_LIT:0> ] <EOL> def unite_merge_dimensions ( dshapes , unite = unite_identical ) : <EOL> """<STR_LIT>""" <EOL> n = len ( dshapes ) <EOL> if all ( isinstance ( ds , DataShape ) and isdimension ( ds [ <NUM_LIT:0> ] ) for ds in dshapes ) : <EOL> dims = [ ds [ <NUM_LIT:0> ] for ds in dshapes ] <EOL> base = unite ( [ ds . subshape [ <NUM_LIT:0> ] for ds in dshapes ] ) <EOL> if base : <EOL> if len ( set ( dims ) ) == <NUM_LIT:1> : <EOL> return n * ( dims [ <NUM_LIT:0> ] * base . subshape [ <NUM_LIT:0> ] ) <EOL> else : <EOL> return n * ( var * base . subshape [ <NUM_LIT:0> ] ) <EOL> def do_one ( funcs ) : <EOL> def f ( inp ) : <EOL> for func in funcs : <EOL> result = func ( inp ) <EOL> if result : <EOL> return result <EOL> return inp <EOL> return f <EOL> def unpack ( ds ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( ds , DataShape ) and len ( ds ) == <NUM_LIT:1> : <EOL> return ds [ <NUM_LIT:0> ] <EOL> else : <EOL> return ds <EOL> @ discover . register ( dict ) <EOL> @ discover . register ( MappingProxyType ) <EOL> def _mapping_discover ( m ) : <EOL> return Record ( ( k , discover ( m [ k ] ) ) for k in sorted ( m ) ) <EOL> @ dispatch ( OrderedDict ) <EOL> def discover ( od ) : <EOL> return Record ( ( k , discover ( v ) ) for k , v in od . items ( ) ) <EOL> @ dispatch ( np . number ) <EOL> def discover ( n ) : <EOL> return from_numpy ( ( ) , type ( n ) ) <EOL> @ dispatch ( np . timedelta64 ) <EOL> def discover ( n ) : <EOL> return from_numpy ( ( ) , n ) <EOL> def is_string_array ( x ) : <EOL> """<STR_LIT>""" <EOL> return all ( isinstance ( i , _strtypes ) for i in x . flat [ : <NUM_LIT:5> ] . tolist ( ) ) <EOL> @ dispatch ( np . ndarray ) <EOL> def discover ( x ) : <EOL> ds = from_numpy ( x . shape , x . dtype ) <EOL> if ds . measure == object_ and is_string_array ( x ) : <EOL> return DataShape ( * ( ds . shape + ( string , ) ) ) <EOL> if isrecord ( ds . measure ) and object_ in ds . measure . types : <EOL> m = Record ( [ [ name , string if typ == object_ and is_string_array ( x [ name ] ) <EOL> else typ ] <EOL> for name , typ in ds . measure . parameters [ <NUM_LIT:0> ] ] ) <EOL> return DataShape ( * ( ds . shape + ( m , ) ) ) <EOL> else : <EOL> return ds <EOL> def descendents ( d , x ) : <EOL> """<STR_LIT>""" <EOL> desc = set ( [ x ] ) <EOL> children = d . get ( x , set ( ) ) <EOL> while children : <EOL> children = set . union ( * [ set ( d . get ( kid , ( ) ) ) for kid in desc ] ) <EOL> children -= desc <EOL> desc . update ( children ) <EOL> return desc <EOL> Mock = None <EOL> try : <EOL> from unittest . mock import Mock <EOL> except ImportError : <EOL> try : <EOL> from mock import Mock <EOL> except ImportError : <EOL> pass <EOL> if Mock is not None : <EOL> @ dispatch ( Mock ) <EOL> def discover ( m ) : <EOL> raise NotImplementedError ( "<STR_LIT>" ) <EOL> del Mock </s>
