<s> """<STR_LIT>""" <EOL> from __future__ import absolute_import , print_function , unicode_literals <EOL> from copy import deepcopy <EOL> import numpy as np <EOL> import scipy . sparse as sp <EOL> from six import iteritems <EOL> from six . moves import zip <EOL> from sklearn . feature_extraction import DictVectorizer , FeatureHasher <EOL> from skll . data . dict_vectorizer import DictVectorizer as NewDictVectorizer <EOL> class FeatureSet ( object ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , name , ids , labels = None , features = None , <EOL> vectorizer = None ) : <EOL> super ( FeatureSet , self ) . __init__ ( ) <EOL> self . name = name <EOL> if isinstance ( ids , list ) : <EOL> ids = np . array ( ids ) <EOL> self . ids = ids <EOL> if isinstance ( labels , list ) : <EOL> labels = np . array ( labels ) <EOL> self . labels = labels <EOL> self . features = features <EOL> self . vectorizer = vectorizer <EOL> if isinstance ( self . features , list ) : <EOL> if self . vectorizer is None : <EOL> self . vectorizer = NewDictVectorizer ( sparse = True ) <EOL> self . features = self . vectorizer . fit_transform ( self . features ) <EOL> if self . features is not None : <EOL> num_feats = self . features . shape [ <NUM_LIT:0> ] <EOL> if self . ids is None : <EOL> raise ValueError ( '<STR_LIT>' ) <EOL> num_ids = self . ids . shape [ <NUM_LIT:0> ] <EOL> if num_feats != num_ids : <EOL> raise ValueError ( ( '<STR_LIT>' <EOL> '<STR_LIT>' ) % ( num_ids , <EOL> num_feats ) ) <EOL> if self . labels is None : <EOL> self . labels = np . empty ( num_feats ) <EOL> self . labels . fill ( None ) <EOL> num_labels = self . labels . shape [ <NUM_LIT:0> ] <EOL> if num_feats != num_labels : <EOL> raise ValueError ( ( '<STR_LIT>' <EOL> '<STR_LIT>' ) % ( num_labels , <EOL> num_feats ) ) <EOL> def __contains__ ( self , value ) : <EOL> """<STR_LIT>""" <EOL> return value in self . ids <EOL> def __eq__ ( self , other ) : <EOL> """<STR_LIT>""" <EOL> if not self . features . has_sorted_indices : <EOL> self . features . sort_indices ( ) <EOL> if not other . features . has_sorted_indices : <EOL> other . features . sort_indices ( ) <EOL> return ( self . ids . shape == other . ids . shape and <EOL> self . labels . shape == other . labels . shape and <EOL> self . features . shape == other . features . shape and <EOL> ( self . ids == other . ids ) . all ( ) and <EOL> ( self . labels == other . labels ) . all ( ) and <EOL> np . allclose ( self . features . data , other . features . data , <EOL> rtol = <NUM_LIT> ) and <EOL> ( self . features . indices == other . features . indices ) . all ( ) and <EOL> ( self . features . indptr == other . features . indptr ) . all ( ) and <EOL> self . vectorizer == other . vectorizer ) <EOL> def __iter__ ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . features is not None : <EOL> if not isinstance ( self . vectorizer , DictVectorizer ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> for id_ , label_ , feats in zip ( self . ids , self . labels , <EOL> self . features ) : <EOL> yield ( id_ , label_ , <EOL> self . vectorizer . inverse_transform ( feats ) [ <NUM_LIT:0> ] ) <EOL> else : <EOL> return <EOL> def __len__ ( self ) : <EOL> return self . features . shape [ <NUM_LIT:0> ] <EOL> def __add__ ( self , other ) : <EOL> """<STR_LIT>""" <EOL> if set ( self . ids ) != set ( other . ids ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> ids_indices = dict ( ( y , x ) for x , y in enumerate ( other . ids ) ) <EOL> relative_order = [ ids_indices [ self_id ] for self_id in self . ids ] <EOL> new_set = FeatureSet ( '<STR_LIT:+>' . join ( sorted ( [ self . name , other . name ] ) ) , <EOL> deepcopy ( self . ids ) ) <EOL> if not isinstance ( self . vectorizer , type ( other . vectorizer ) ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> uses_feature_hasher = isinstance ( self . vectorizer , FeatureHasher ) <EOL> if uses_feature_hasher : <EOL> if ( self . vectorizer . n_features != <EOL> other . vectorizer . n_features ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> else : <EOL> if ( set ( self . vectorizer . feature_names_ ) & <EOL> set ( other . vectorizer . feature_names_ ) ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> num_feats = self . features . shape [ <NUM_LIT:1> ] <EOL> new_set . features = sp . hstack ( [ self . features , <EOL> other . features [ relative_order ] ] , <EOL> '<STR_LIT>' ) <EOL> new_set . vectorizer = deepcopy ( self . vectorizer ) <EOL> if not uses_feature_hasher : <EOL> for feat_name , index in other . vectorizer . vocabulary_ . items ( ) : <EOL> new_set . vectorizer . vocabulary_ [ feat_name ] = ( index + <EOL> num_feats ) <EOL> other_names = other . vectorizer . feature_names_ <EOL> new_set . vectorizer . feature_names_ . extend ( other_names ) <EOL> if self . has_labels : <EOL> if other . has_labels and not np . all ( self . labels == other . labels [ relative_order ] ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> new_set . labels = deepcopy ( self . labels ) <EOL> else : <EOL> new_set . labels = deepcopy ( other . labels [ relative_order ] ) <EOL> return new_set <EOL> def filter ( self , ids = None , labels = None , features = None , inverse = False ) : <EOL> """<STR_LIT>""" <EOL> mask = np . ones ( len ( self ) , dtype = bool ) <EOL> if ids is not None : <EOL> mask = np . logical_and ( mask , np . in1d ( self . ids , ids ) ) <EOL> if labels is not None : <EOL> mask = np . logical_and ( mask , np . in1d ( self . labels , labels ) ) <EOL> if inverse and ( labels is not None or ids is not None ) : <EOL> mask = np . logical_not ( mask ) <EOL> self . ids = self . ids [ mask ] <EOL> self . labels = self . labels [ mask ] <EOL> self . features = self . features [ mask , : ] <EOL> if features is not None : <EOL> if isinstance ( self . vectorizer , FeatureHasher ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> columns = np . array ( sorted ( { feat_num for feat_name , feat_num in <EOL> iteritems ( self . vectorizer . vocabulary_ ) <EOL> if ( feat_name in features or <EOL> feat_name . split ( '<STR_LIT:=>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] in <EOL> features ) } ) ) <EOL> if inverse : <EOL> all_columns = np . arange ( self . features . shape [ <NUM_LIT:1> ] ) <EOL> columns = all_columns [ np . logical_not ( np . in1d ( all_columns , <EOL> columns ) ) ] <EOL> self . features = self . features [ : , columns ] <EOL> self . vectorizer . restrict ( columns , indices = True ) <EOL> def filtered_iter ( self , ids = None , labels = None , features = None , <EOL> inverse = False ) : <EOL> """<STR_LIT>""" <EOL> if self . features is not None and not isinstance ( self . vectorizer , <EOL> DictVectorizer ) : <EOL> raise ValueError ( '<STR_LIT>' <EOL> '<STR_LIT>' <EOL> '<STR_LIT>' ) <EOL> for id_ , label_ , feats in zip ( self . ids , self . labels , self . features ) : <EOL> if ids is not None and ( id_ in ids ) == inverse : <EOL> continue <EOL> if labels is not None and ( label_ in labels ) == inverse : <EOL> continue <EOL> feat_dict = self . vectorizer . inverse_transform ( feats ) [ <NUM_LIT:0> ] <EOL> if features is not None : <EOL> feat_dict = { name : value for name , value in <EOL> iteritems ( feat_dict ) if <EOL> ( inverse != ( name in features or <EOL> name . split ( '<STR_LIT:=>' , <NUM_LIT:1> ) [ <NUM_LIT:0> ] in features ) ) } <EOL> elif not inverse : <EOL> feat_dict = { } <EOL> yield id_ , label_ , feat_dict <EOL> def __sub__ ( self , other ) : <EOL> """<STR_LIT>""" <EOL> new_set = deepcopy ( self ) <EOL> new_set . filter ( features = other . vectorizer . feature_names_ , <EOL> inverse = True ) <EOL> return new_set <EOL> @ property <EOL> def has_labels ( self ) : <EOL> """<STR_LIT>""" <EOL> if self . labels is not None : <EOL> return not ( np . issubdtype ( self . labels . dtype , float ) and <EOL> np . isnan ( np . min ( self . labels ) ) ) <EOL> else : <EOL> return False <EOL> def __str__ ( self ) : <EOL> """<STR_LIT>""" <EOL> return str ( self . __dict__ ) <EOL> def __repr__ ( self ) : <EOL> """<STR_LIT>""" <EOL> return repr ( self . __dict__ ) <EOL> def __getitem__ ( self , value ) : <EOL> """<STR_LIT>""" <EOL> if isinstance ( value , slice ) : <EOL> sliced_ids = self . ids [ value ] <EOL> sliced_feats = ( self . features [ value ] if self . features is not None <EOL> else None ) <EOL> sliced_labels = ( self . labels [ value ] if self . labels is not None <EOL> else None ) <EOL> return FeatureSet ( '<STR_LIT>' . format ( self . name , value ) , sliced_ids , <EOL> features = sliced_feats , labels = sliced_labels , <EOL> vectorizer = self . vectorizer ) <EOL> else : <EOL> label = self . labels [ value ] if self . labels is not None else None <EOL> feats = self . features [ value , : ] <EOL> features = ( self . vectorizer . inverse_transform ( feats ) [ <NUM_LIT:0> ] if <EOL> self . features is not None else { } ) <EOL> return self . ids [ value ] , label , features </s>
