<s> """<STR_LIT>""" <EOL> import logging <EOL> import sys <EOL> import math <EOL> import re <EOL> from bigml . api import FINISHED <EOL> from bigml . api import ( BigML , get_cluster_id , get_status ) <EOL> from bigml . util import cast , utf8 <EOL> from bigml . centroid import Centroid <EOL> from bigml . basemodel import retrieve_resource <EOL> from bigml . basemodel import ONLY_MODEL <EOL> from bigml . model import print_distribution <EOL> from bigml . model import STORAGE <EOL> from bigml . predicate import TM_TOKENS , TM_FULL_TERM <EOL> from bigml . modelfields import ModelFields <EOL> from bigml . io import UnicodeWriter <EOL> LOGGER = logging . getLogger ( '<STR_LIT>' ) <EOL> OPTIONAL_FIELDS = [ '<STR_LIT>' , '<STR_LIT:text>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> CSV_STATISTICS = [ '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' , <EOL> '<STR_LIT>' , '<STR_LIT>' , '<STR_LIT>' ] <EOL> INDENT = "<STR_LIT:U+0020>" * <NUM_LIT:4> <EOL> INTERCENTROID_MEASURES = [ ( '<STR_LIT>' , min ) , <EOL> ( '<STR_LIT>' , lambda ( x ) : sum ( x ) / float ( len ( x ) ) ) , <EOL> ( '<STR_LIT>' , max ) ] <EOL> def parse_terms ( text , case_sensitive = True ) : <EOL> """<STR_LIT>""" <EOL> if text is None : <EOL> return [ ] <EOL> expression = ur '<STR_LIT>' <EOL> pattern = re . compile ( expression ) <EOL> return [ match [ <NUM_LIT:1> ] if case_sensitive else match [ <NUM_LIT:1> ] . lower ( ) <EOL> for match in re . findall ( pattern , text ) ] <EOL> def parse_items ( text , regexp ) : <EOL> """<STR_LIT>""" <EOL> if text is None : <EOL> return [ ] <EOL> pattern = re . compile ( regexp , flags = re . U ) <EOL> return pattern . split ( text ) <EOL> def get_unique_terms ( terms , term_forms , tag_cloud ) : <EOL> """<STR_LIT>""" <EOL> extend_forms = { } <EOL> tag_cloud = tag_cloud . keys ( ) <EOL> for term , forms in term_forms . items ( ) : <EOL> for form in forms : <EOL> extend_forms [ form ] = term <EOL> extend_forms [ term ] = term <EOL> terms_set = set ( ) <EOL> for term in terms : <EOL> if term in tag_cloud : <EOL> terms_set . add ( term ) <EOL> elif term in extend_forms : <EOL> terms_set . add ( extend_forms [ term ] ) <EOL> return list ( terms_set ) <EOL> class Cluster ( ModelFields ) : <EOL> """<STR_LIT>""" <EOL> def __init__ ( self , cluster , api = None ) : <EOL> self . resource_id = None <EOL> self . centroids = None <EOL> self . scales = { } <EOL> self . term_forms = { } <EOL> self . tag_clouds = { } <EOL> self . term_analysis = { } <EOL> self . item_analysis = { } <EOL> self . items = { } <EOL> if not ( isinstance ( cluster , dict ) and '<STR_LIT>' in cluster and <EOL> cluster [ '<STR_LIT>' ] is not None ) : <EOL> if api is None : <EOL> api = BigML ( storage = STORAGE ) <EOL> self . resource_id = get_cluster_id ( cluster ) <EOL> if self . resource_id is None : <EOL> raise Exception ( api . error_message ( cluster , <EOL> resource_type = '<STR_LIT>' , <EOL> method = '<STR_LIT>' ) ) <EOL> query_string = ONLY_MODEL <EOL> cluster = retrieve_resource ( api , self . resource_id , <EOL> query_string = query_string ) <EOL> else : <EOL> self . resource_id = get_cluster_id ( cluster ) <EOL> if '<STR_LIT:object>' in cluster and isinstance ( cluster [ '<STR_LIT:object>' ] , dict ) : <EOL> cluster = cluster [ '<STR_LIT:object>' ] <EOL> if '<STR_LIT>' in cluster and isinstance ( cluster [ '<STR_LIT>' ] , dict ) : <EOL> status = get_status ( cluster ) <EOL> if '<STR_LIT:code>' in status and status [ '<STR_LIT:code>' ] == FINISHED : <EOL> clusters = cluster [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> self . centroids = [ Centroid ( centroid ) for centroid in clusters ] <EOL> self . scales = { } <EOL> self . scales . update ( cluster [ '<STR_LIT>' ] ) <EOL> self . term_forms = { } <EOL> self . tag_clouds = { } <EOL> self . term_analysis = { } <EOL> fields = cluster [ '<STR_LIT>' ] [ '<STR_LIT>' ] <EOL> summary_fields = cluster [ '<STR_LIT>' ] <EOL> for field_id in summary_fields : <EOL> del fields [ field_id ] <EOL> for field_id , field in fields . items ( ) : <EOL> if field [ '<STR_LIT>' ] == '<STR_LIT:text>' : <EOL> self . term_forms [ field_id ] = { } <EOL> self . term_forms [ field_id ] . update ( field [ <EOL> '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> self . tag_clouds [ field_id ] = { } <EOL> self . tag_clouds [ field_id ] . update ( field [ <EOL> '<STR_LIT>' ] [ '<STR_LIT>' ] ) <EOL> self . term_analysis [ field_id ] = { } <EOL> self . term_analysis [ field_id ] . update ( <EOL> field [ '<STR_LIT>' ] ) <EOL> if field [ '<STR_LIT>' ] == '<STR_LIT>' : <EOL> self . items [ field_id ] = { } <EOL> self . items [ field_id ] . update ( <EOL> dict ( field [ '<STR_LIT>' ] [ '<STR_LIT>' ] ) ) <EOL> self . item_analysis [ field_id ] = { } <EOL> self . item_analysis [ field_id ] . update ( <EOL> field [ '<STR_LIT>' ] ) <EOL> ModelFields . __init__ ( self , fields ) <EOL> if not all ( [ field_id in self . fields for <EOL> field_id in self . scales ] ) : <EOL> raise Exception ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> else : <EOL> raise Exception ( "<STR_LIT>" ) <EOL> else : <EOL> raise Exception ( "<STR_LIT>" <EOL> "<STR_LIT>" % <EOL> cluster ) <EOL> def centroid ( self , input_data , by_name = True ) : <EOL> """<STR_LIT>""" <EOL> input_data = self . filter_input_data ( input_data , by_name = by_name ) <EOL> for field_id , field in self . fields . items ( ) : <EOL> if ( field [ '<STR_LIT>' ] not in OPTIONAL_FIELDS and <EOL> field_id not in input_data ) : <EOL> raise Exception ( "<STR_LIT>" <EOL> "<STR_LIT>" <EOL> "<STR_LIT>" ) <EOL> cast ( input_data , self . fields ) <EOL> unique_terms = self . get_unique_terms ( input_data ) <EOL> nearest = { '<STR_LIT>' : None , '<STR_LIT>' : None , <EOL> '<STR_LIT>' : float ( '<STR_LIT>' ) } <EOL> for centroid in self . centroids : <EOL> distance2 = centroid . distance2 ( input_data , unique_terms , <EOL> self . scales , <EOL> stop_distance2 = nearest [ '<STR_LIT>' ] ) <EOL> if distance2 is not None : <EOL> nearest = { '<STR_LIT>' : centroid . centroid_id , <EOL> '<STR_LIT>' : centroid . name , <EOL> '<STR_LIT>' : distance2 } <EOL> nearest [ '<STR_LIT>' ] = math . sqrt ( nearest [ '<STR_LIT>' ] ) <EOL> return nearest <EOL> def get_unique_terms ( self , input_data ) : <EOL> """<STR_LIT>""" <EOL> unique_terms = { } <EOL> for field_id in self . term_forms : <EOL> if field_id in input_data : <EOL> input_data_field = input_data . get ( field_id , '<STR_LIT>' ) <EOL> if isinstance ( input_data_field , basestring ) : <EOL> case_sensitive = self . term_analysis [ field_id ] . get ( <EOL> '<STR_LIT>' , True ) <EOL> token_mode = self . term_analysis [ field_id ] . get ( <EOL> '<STR_LIT>' , '<STR_LIT:all>' ) <EOL> if token_mode != TM_FULL_TERM : <EOL> terms = parse_terms ( input_data_field , <EOL> case_sensitive = case_sensitive ) <EOL> else : <EOL> terms = [ ] <EOL> if token_mode != TM_TOKENS : <EOL> terms . append ( <EOL> input_data_field if case_sensitive <EOL> else input_data_field . lower ( ) ) <EOL> unique_terms [ field_id ] = get_unique_terms ( <EOL> terms , self . term_forms [ field_id ] , <EOL> self . tag_clouds . get ( field_id , [ ] ) ) <EOL> else : <EOL> unique_terms [ field_id ] = input_data_field <EOL> del input_data [ field_id ] <EOL> for field_id in self . item_analysis : <EOL> if field_id in input_data : <EOL> input_data_field = input_data . get ( field_id , '<STR_LIT>' ) <EOL> if isinstance ( input_data_field , basestring ) : <EOL> separator = self . item_analysis [ field_id ] . get ( <EOL> '<STR_LIT>' , '<STR_LIT:U+0020>' ) <EOL> regexp = self . item_analysis [ field_id ] . get ( <EOL> '<STR_LIT>' ) <EOL> if regexp is None : <EOL> regexp = ur '<STR_LIT:%s>' % re . escape ( separator ) <EOL> terms = parse_items ( input_data_field , regexp ) <EOL> unique_terms [ field_id ] = get_unique_terms ( <EOL> terms , { } , <EOL> self . items . get ( field_id , [ ] ) ) <EOL> else : <EOL> unique_terms [ field_id ] = input_data_field <EOL> del input_data [ field_id ] <EOL> return unique_terms <EOL> def centroids_distance ( self , to_centroid ) : <EOL> """<STR_LIT>""" <EOL> intercentroid_distance = [ ] <EOL> unique_terms = self . get_unique_terms ( to_centroid . center ) <EOL> distances = [ ] <EOL> for centroid in self . centroids : <EOL> if centroid . centroid_id != to_centroid . centroid_id : <EOL> distances . append ( <EOL> math . sqrt ( <EOL> centroid . distance2 ( to_centroid . center , unique_terms , <EOL> self . scales ) ) ) <EOL> for measure , function in INTERCENTROID_MEASURES : <EOL> result = function ( distances ) <EOL> intercentroid_distance . append ( [ measure , result ] ) <EOL> return intercentroid_distance <EOL> def centroid_features ( self , centroid , field_ids ) : <EOL> """<STR_LIT>""" <EOL> features = [ ] <EOL> for field_id in field_ids : <EOL> value = centroid . center [ field_id ] <EOL> if isinstance ( value , basestring ) : <EOL> value = value . encode ( '<STR_LIT:utf-8>' ) <EOL> features . append ( value ) <EOL> return features <EOL> def get_data_distribution ( self ) : <EOL> """<STR_LIT>""" <EOL> distribution = [ [ centroid . name , centroid . count ] for centroid in <EOL> self . centroids ] <EOL> return sorted ( distribution , key = lambda x : x [ <NUM_LIT:0> ] ) <EOL> def statistics_CSV ( self , file_name = None ) : <EOL> """<STR_LIT>""" <EOL> self . statistics_csv ( file_name = file_name ) <EOL> def statistics_csv ( self , file_name = None ) : <EOL> """<STR_LIT>""" <EOL> rows = [ ] <EOL> writer = None <EOL> field_ids = self . centroids [ <NUM_LIT:0> ] . center . keys ( ) <EOL> headers = [ u"<STR_LIT>" ] <EOL> headers . extend ( [ u"<STR_LIT:%s>" % self . fields [ field_id ] [ "<STR_LIT:name>" ] <EOL> for field_id in field_ids ] ) <EOL> headers . extend ( [ u"<STR_LIT>" ] ) <EOL> intercentroids = False <EOL> header_complete = False <EOL> for centroid in self . centroids : <EOL> row = [ centroid . name ] <EOL> row . extend ( self . centroid_features ( centroid , field_ids ) ) <EOL> row . append ( centroid . count ) <EOL> if len ( self . centroids ) > <NUM_LIT:1> : <EOL> for measure , result in self . centroids_distance ( centroid ) : <EOL> if not intercentroids : <EOL> headers . append ( u"<STR_LIT>" % measure . lower ( ) ) <EOL> row . append ( result ) <EOL> intercentroids = True <EOL> for measure , result in centroid . distance . items ( ) : <EOL> if measure in CSV_STATISTICS : <EOL> if not header_complete : <EOL> headers . append ( u"<STR_LIT>" % <EOL> measure . lower ( ) . replace ( "<STR_LIT:_>" , "<STR_LIT:U+0020>" ) ) <EOL> row . append ( result ) <EOL> if not header_complete : <EOL> rows . append ( headers ) <EOL> header_complete = True <EOL> rows . append ( row ) <EOL> if file_name is None : <EOL> return rows <EOL> with UnicodeWriter ( file_name ) as writer : <EOL> for row in rows : <EOL> writer . writerow ( [ item if not isinstance ( item , basestring ) <EOL> else item . encode ( "<STR_LIT:utf-8>" ) for item in row ] ) <EOL> def summarize ( self , out = sys . stdout ) : <EOL> """<STR_LIT>""" <EOL> out . write ( u"<STR_LIT>" % len ( self . centroids ) ) <EOL> out . write ( u"<STR_LIT>" ) <EOL> print_distribution ( self . get_data_distribution ( ) , out = out ) <EOL> out . write ( u"<STR_LIT>" ) <EOL> centroids_list = sorted ( self . centroids , key = lambda x : x . name ) <EOL> out . write ( u"<STR_LIT>" ) <EOL> for centroid in centroids_list : <EOL> out . write ( utf8 ( u"<STR_LIT>" % centroid . name ) ) <EOL> connector = "<STR_LIT>" <EOL> for field_id , value in centroid . center . items ( ) : <EOL> if isinstance ( value , basestring ) : <EOL> value = u"<STR_LIT>" % value <EOL> out . write ( utf8 ( u"<STR_LIT>" % ( connector , <EOL> self . fields [ field_id ] [ '<STR_LIT:name>' ] , <EOL> value ) ) ) <EOL> connector = "<STR_LIT:U+002CU+0020>" <EOL> out . write ( u"<STR_LIT>" ) <EOL> out . write ( u"<STR_LIT>" ) <EOL> for centroid in centroids_list : <EOL> centroid . print_statistics ( out = out ) <EOL> if len ( self . centroids ) > <NUM_LIT:1> : <EOL> out . write ( u"<STR_LIT>" ) <EOL> for centroid in centroids_list : <EOL> out . write ( utf8 ( u"<STR_LIT>" % centroid . name ) ) <EOL> for measure , result in self . centroids_distance ( centroid ) : <EOL> out . write ( u"<STR_LIT>" % ( INDENT , measure , result ) ) <EOL> out . write ( u"<STR_LIT:\n>" ) </s>
